diff --git a/pkg/filter/image_process.go b/pkg/filter/image_process.go index a26bdaa..43f3012 100644 --- a/pkg/filter/image_process.go +++ b/pkg/filter/image_process.go @@ -18,6 +18,7 @@ import ( "gitbase.de/apairon/mark2web/pkg/jobm" "gitbase.de/apairon/mark2web/pkg/logger" "gitbase.de/apairon/mark2web/pkg/mark2web" + "gitbase.de/apairon/mark2web/pkg/webrequest" "github.com/disintegration/imaging" "github.com/flosch/pongo2" ) @@ -153,7 +154,7 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, * logger.Log.Noticef("processing image from %s to %s", imgSource, imgTarget) if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") { // remote file - img, p.Format, err = helper.ImageWebRequest(imgSource) + img, p.Format, err = webrequest.GetImage(imgSource) } else { img, err = imaging.Open(imgSource, imaging.AutoOrientation(true)) } diff --git a/pkg/mark2web/collection.go b/pkg/mark2web/collection.go index 688776a..e588e37 100644 --- a/pkg/mark2web/collection.go +++ b/pkg/mark2web/collection.go @@ -8,6 +8,7 @@ import ( "gitbase.de/apairon/mark2web/pkg/helper" "gitbase.de/apairon/mark2web/pkg/logger" + "gitbase.de/apairon/mark2web/pkg/webrequest" "github.com/davecgh/go-spew/spew" "github.com/flosch/pongo2" ) @@ -57,7 +58,7 @@ func (node *TreeNode) handleCollections() { cacheEntry.hit++ } else { logger.Log.Noticef("reading collection from: %s", errSrcText) - colData = helper.JSONWebRequest(url) + colData = webrequest.GetJSON(url) colCache[url] = &colCacheEntry{ data: colData, navnames: make([]string, 0), diff --git a/pkg/mark2web/context_fn.go b/pkg/mark2web/context_fn.go index c66f8df..a55ae77 100644 --- a/pkg/mark2web/context_fn.go +++ b/pkg/mark2web/context_fn.go @@ -1,14 +1,14 @@ package mark2web import ( - "gitbase.de/apairon/mark2web/pkg/helper" + "gitbase.de/apairon/mark2web/pkg/webrequest" "github.com/flosch/pongo2" ) // RequestFn will make a web request and returns map[string]interface form pongo2 func RequestFn(url *pongo2.Value, args ...*pongo2.Value) *pongo2.Value { u := url.String() - return pongo2.AsValue(helper.JSONWebRequest(u)) + return pongo2.AsValue(webrequest.GetJSON(u)) } // RenderFn renders a pongo2 template with additional context diff --git a/pkg/progress/bar.go b/pkg/progress/bar.go index c57fe58..72e6130 100644 --- a/pkg/progress/bar.go +++ b/pkg/progress/bar.go @@ -17,6 +17,12 @@ var bars = make(map[string]*bar) var initialized = false var terminalWidth = 80 +// OverallTotal is number of total jobs +var OverallTotal = 0 + +// OverallDone is number of done jobs +var OverallDone = 0 + // Init initializes the bar drawing func Init() { if t, err := tty.Open(); err == nil && t != nil { @@ -29,6 +35,7 @@ func Init() { // IncrTotal increases the total jobs for the bar func IncrTotal(barname string) { + OverallTotal++ if initialized { _bar := bars[barname] if _bar == nil { @@ -52,6 +59,7 @@ func IncrTotal(barname string) { // IncrDone increases to done jobs counter func IncrDone(barname string) { + OverallDone++ if initialized { bars[barname].Bar.Incr() bars[barname].Description = "" diff --git a/pkg/helper/webrequest.go b/pkg/webrequest/request.go similarity index 76% rename from pkg/helper/webrequest.go rename to pkg/webrequest/request.go index 53a2ab8..ce86f1d 100644 --- a/pkg/helper/webrequest.go +++ b/pkg/webrequest/request.go @@ -1,4 +1,4 @@ -package helper +package webrequest import ( "encoding/json" @@ -8,6 +8,8 @@ import ( "net/http" "strings" + "gitbase.de/apairon/mark2web/pkg/progress" + "gitbase.de/apairon/mark2web/pkg/logger" ) @@ -23,18 +25,22 @@ type wrJSONEntry struct { var wrImageCache = make(map[string]*wrImageEntry) var wrJSONCache = make(map[string]*wrJSONEntry) -// WebRequest will fetch an url and returns reponse -func WebRequest(url string, opts interface{}) (*http.Response, error) { +// Get will fetch an url and returns reponse +func Get(url string, opts interface{}) (resp *http.Response, err error) { logger.Log.Noticef("requesting url via GET %s", url) - return http.Get(url) + progress.IncrTotal("web request") + progress.DescribeCurrent("web request", url) + resp, err = http.Get(url) + progress.IncrDone("web request") + return resp, err } -// JSONWebRequest will GET a json object/array from a given URL -func JSONWebRequest(url string) interface{} { +// GetJSON will GET a json object/array from a given URL +func GetJSON(url string) interface{} { cached := wrJSONCache[url] if cached == nil { - resp, err := WebRequest(url, nil) + resp, err := Get(url, nil) if err != nil { logger.Log.Panicf("could not get url '%s': %s", url, err) } @@ -81,11 +87,11 @@ func JSONWebRequest(url string) interface{} { return cached.data } -// ImageWebRequest gets an image from an url -func ImageWebRequest(url string) (image.Image, string, error) { +// GetImage gets an image from an url +func GetImage(url string) (image.Image, string, error) { cached := wrImageCache[url] if cached == nil { - resp, err := WebRequest(url, nil) + resp, err := Get(url, nil) if err != nil { return nil, "", fmt.Errorf("could not get url '%s': %s", url, err) }