diff --git a/.vscode/settings.json b/.vscode/settings.json index 023c838..943fc59 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -6,7 +6,7 @@ "commands": [ { "match": "website/.*", - "cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create", + "cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create -logLevel warning -progress", "silent": false } ] diff --git a/pkg/filter/image_process.go b/pkg/filter/image_process.go index d466f39..a26bdaa 100644 --- a/pkg/filter/image_process.go +++ b/pkg/filter/image_process.go @@ -8,7 +8,6 @@ import ( "image/gif" "image/jpeg" "image/png" - "net/http" "net/url" "os" "path" @@ -66,20 +65,6 @@ func parseImageParams(str string) (*mark2web.ImagingConfig, error) { return &p, nil } -func getImageFromURL(url string) (image.Image, string, error) { - resp, err := http.Get(url) - if err != nil { - return nil, "", fmt.Errorf("could not get url '%s': %s", url, err) - } - - img, format, err := image.Decode(resp.Body) - if err != nil { - return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err) - } - - return img, format, nil -} - // ImageProcessFilter read the image url and process parameters and saves the resized/processed image // param: w=WITDH,h=HEIGHT func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) { @@ -168,7 +153,7 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, * logger.Log.Noticef("processing image from %s to %s", imgSource, imgTarget) if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") { // remote file - img, p.Format, err = getImageFromURL(imgSource) + img, p.Format, err = helper.ImageWebRequest(imgSource) } else { img, err = imaging.Open(imgSource, imaging.AutoOrientation(true)) } diff --git a/pkg/helper/webrequest.go b/pkg/helper/webrequest.go index f33f6f8..53a2ab8 100644 --- a/pkg/helper/webrequest.go +++ b/pkg/helper/webrequest.go @@ -2,6 +2,8 @@ package helper import ( "encoding/json" + "fmt" + "image" "io/ioutil" "net/http" "strings" @@ -9,47 +11,97 @@ import ( "gitbase.de/apairon/mark2web/pkg/logger" ) -// JSONWebRequest will GET a json object/array from a given URL -func JSONWebRequest(url string) interface{} { +type wrImageEntry struct { + img image.Image + format string +} + +type wrJSONEntry struct { + data interface{} +} + +var wrImageCache = make(map[string]*wrImageEntry) +var wrJSONCache = make(map[string]*wrJSONEntry) + +// WebRequest will fetch an url and returns reponse +func WebRequest(url string, opts interface{}) (*http.Response, error) { logger.Log.Noticef("requesting url via GET %s", url) - resp, err := http.Get(url) - if err != nil { - logger.Log.Panicf("could not get url '%s': %s", url, err) - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - logger.Log.Panicf("could not read body from url '%s': %s", url, err) - } - - logger.Log.Debugf("output from url '%s':\n%s", url, string(body)) - - if resp.StatusCode >= 400 { - logger.Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url) - } - - contentType := resp.Header.Get("Content-Type") - - if strings.Contains(contentType, "json") { - - } else { - logger.Log.Panicf("is not json '%s' from url '%s'", contentType, url) - } - - jsonMap := make(map[string]interface{}) - err = json.Unmarshal(body, &jsonMap) - if err == nil { - return jsonMap - } - - jsonArrayMap := make([]map[string]interface{}, 0) - err = json.Unmarshal(body, &jsonArrayMap) - if err == nil { - return jsonArrayMap - } - - logger.Log.Panicf("could not read json from '%s': invalid type", url) - return nil + return http.Get(url) +} + +// JSONWebRequest will GET a json object/array from a given URL +func JSONWebRequest(url string) interface{} { + cached := wrJSONCache[url] + if cached == nil { + resp, err := WebRequest(url, nil) + if err != nil { + logger.Log.Panicf("could not get url '%s': %s", url, err) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + logger.Log.Panicf("could not read body from url '%s': %s", url, err) + } + + logger.Log.Debugf("output from url '%s':\n%s", url, string(body)) + + if resp.StatusCode >= 400 { + logger.Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url) + } + + contentType := resp.Header.Get("Content-Type") + + if strings.Contains(contentType, "json") { + + } else { + logger.Log.Panicf("is not json '%s' from url '%s'", contentType, url) + } + + cached = new(wrJSONEntry) + + jsonMap := make(map[string]interface{}) + err = json.Unmarshal(body, &jsonMap) + if err == nil { + cached.data = jsonMap + } else { + jsonArrayMap := make([]map[string]interface{}, 0) + err = json.Unmarshal(body, &jsonArrayMap) + if err == nil { + cached.data = jsonArrayMap + } else { + logger.P("could not read json from '%s': invalid type", url) + } + } + + wrJSONCache[url] = cached + + } + return cached.data +} + +// ImageWebRequest gets an image from an url +func ImageWebRequest(url string) (image.Image, string, error) { + cached := wrImageCache[url] + if cached == nil { + resp, err := WebRequest(url, nil) + if err != nil { + return nil, "", fmt.Errorf("could not get url '%s': %s", url, err) + } + + img, format, err := image.Decode(resp.Body) + if err != nil { + return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err) + } + + cached = &wrImageEntry{ + img: img, + format: format, + } + + wrImageCache[url] = cached + } + + return cached.img, cached.format, nil } diff --git a/pkg/jobm/jobmanager.go b/pkg/jobm/jobmanager.go index de1fd73..d02dcb9 100644 --- a/pkg/jobm/jobmanager.go +++ b/pkg/jobm/jobmanager.go @@ -51,8 +51,8 @@ func Enqueue(jobs ...Job) { // Wait will wait for all jobs to finish func Wait() { - time.Sleep(time.Millisecond * 300) close(jobChan) + time.Sleep(time.Millisecond * 500) wg.Wait() } diff --git a/pkg/mark2web/content.go b/pkg/mark2web/content.go index 5d935a7..8458150 100644 --- a/pkg/mark2web/content.go +++ b/pkg/mark2web/content.go @@ -7,6 +7,8 @@ import ( "regexp" "strings" + "gitbase.de/apairon/mark2web/pkg/progress" + "gitbase.de/apairon/mark2web/pkg/helper" "gitbase.de/apairon/mark2web/pkg/logger" "github.com/davecgh/go-spew/spew" @@ -17,6 +19,9 @@ import ( // ReadContentDir walks through content directory and builds the tree of configurations func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string, conf *PathConfig) { + progress.IncrTotal("content dir") + progress.DescribeCurrent("content dir", "found "+inBase) + if node.root == nil { // first node is root node.root = node @@ -155,6 +160,8 @@ func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*Pa // ProcessContent walks recursivly through the input paths and processes all files for output func (node *TreeNode) ProcessContent() { + progress.DescribeCurrent("content dir", "processing "+node.InputPath) + helper.CreateDirectory(node.OutputPath) if node.root != node { @@ -293,6 +300,8 @@ func (node *TreeNode) ProcessContent() { } } + progress.IncrDone("content dir") + i := 0 // sub can dynamically increase, so no for range for i < len(node.Sub) { diff --git a/website/templates/base.html b/website/templates/base.html index ad9ff60..e3e2c79 100755 --- a/website/templates/base.html +++ b/website/templates/base.html @@ -31,11 +31,11 @@