webrequest pkg, web request bar

pkg/filter/image_process.go

@@ -18,6 +18,7 @@ import (
 	"gitbase.de/apairon/mark2web/pkg/jobm"
 	"gitbase.de/apairon/mark2web/pkg/logger"
 	"gitbase.de/apairon/mark2web/pkg/mark2web"
+	"gitbase.de/apairon/mark2web/pkg/webrequest"
 	"github.com/disintegration/imaging"
 	"github.com/flosch/pongo2"
 )
@@ -153,7 +154,7 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 				logger.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
 				if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
 					// remote file
-					img, p.Format, err = helper.ImageWebRequest(imgSource)
+					img, p.Format, err = webrequest.GetImage(imgSource)
 				} else {
 					img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
 				}

pkg/mark2web/collection.go

@@ -8,6 +8,7 @@ import (
 
 	"gitbase.de/apairon/mark2web/pkg/helper"
 	"gitbase.de/apairon/mark2web/pkg/logger"
+	"gitbase.de/apairon/mark2web/pkg/webrequest"
 	"github.com/davecgh/go-spew/spew"
 	"github.com/flosch/pongo2"
 )
@@ -57,7 +58,7 @@ func (node *TreeNode) handleCollections() {
 				cacheEntry.hit++
 			} else {
 				logger.Log.Noticef("reading collection from: %s", errSrcText)
-				colData = helper.JSONWebRequest(url)
+				colData = webrequest.GetJSON(url)
 				colCache[url] = &colCacheEntry{
 					data:     colData,
 					navnames: make([]string, 0),

pkg/mark2web/context_fn.go

@@ -1,14 +1,14 @@
 package mark2web
 
 import (
-	"gitbase.de/apairon/mark2web/pkg/helper"
+	"gitbase.de/apairon/mark2web/pkg/webrequest"
 	"github.com/flosch/pongo2"
 )
 
 // RequestFn will make a web request and returns map[string]interface form pongo2
 func RequestFn(url *pongo2.Value, args ...*pongo2.Value) *pongo2.Value {
 	u := url.String()
-	return pongo2.AsValue(helper.JSONWebRequest(u))
+	return pongo2.AsValue(webrequest.GetJSON(u))
 }
 
 // RenderFn renders a pongo2 template with additional context

pkg/progress/bar.go

@@ -17,6 +17,12 @@ var bars = make(map[string]*bar)
 var initialized = false
 var terminalWidth = 80
 
+// OverallTotal is number of total jobs
+var OverallTotal = 0
+
+// OverallDone is number of done jobs
+var OverallDone = 0
+
 // Init initializes the bar drawing
 func Init() {
 	if t, err := tty.Open(); err == nil && t != nil {
@@ -29,6 +35,7 @@ func Init() {
 
 // IncrTotal increases the total jobs for the bar
 func IncrTotal(barname string) {
+	OverallTotal++
 	if initialized {
 		_bar := bars[barname]
 		if _bar == nil {
@@ -52,6 +59,7 @@ func IncrTotal(barname string) {
 
 // IncrDone increases to done jobs counter
 func IncrDone(barname string) {
+	OverallDone++
 	if initialized {
 		bars[barname].Bar.Incr()
 		bars[barname].Description = ""

pkg/webrequest/request.go

@@ -1,4 +1,4 @@
-package helper
+package webrequest
 
 import (
 	"encoding/json"
@@ -8,6 +8,8 @@ import (
 	"net/http"
 	"strings"
 
+	"gitbase.de/apairon/mark2web/pkg/progress"
+
 	"gitbase.de/apairon/mark2web/pkg/logger"
 )
 
@@ -23,18 +25,22 @@ type wrJSONEntry struct {
 var wrImageCache = make(map[string]*wrImageEntry)
 var wrJSONCache = make(map[string]*wrJSONEntry)
 
-// WebRequest will fetch an url and returns reponse
+// Get will fetch an url and returns reponse
-func WebRequest(url string, opts interface{}) (*http.Response, error) {
+func Get(url string, opts interface{}) (resp *http.Response, err error) {
 	logger.Log.Noticef("requesting url via GET %s", url)
 
-	return http.Get(url)
+	progress.IncrTotal("web request")
+	progress.DescribeCurrent("web request", url)
+	resp, err = http.Get(url)
+	progress.IncrDone("web request")
+	return resp, err
 }
 
-// JSONWebRequest will GET a json object/array from a given URL
+// GetJSON will GET a json object/array from a given URL
-func JSONWebRequest(url string) interface{} {
+func GetJSON(url string) interface{} {
 	cached := wrJSONCache[url]
 	if cached == nil {
-		resp, err := WebRequest(url, nil)
+		resp, err := Get(url, nil)
 		if err != nil {
 			logger.Log.Panicf("could not get url '%s': %s", url, err)
 		}
@@ -81,11 +87,11 @@ func JSONWebRequest(url string) interface{} {
 	return cached.data
 }
 
-// ImageWebRequest gets an image from an url
+// GetImage gets an image from an url
-func ImageWebRequest(url string) (image.Image, string, error) {
+func GetImage(url string) (image.Image, string, error) {
 	cached := wrImageCache[url]
 	if cached == nil {
-		resp, err := WebRequest(url, nil)
+		resp, err := Get(url, nil)
 		if err != nil {
 			return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
 		}