cached webrequests

.vscode/settings.json

@@ -6,7 +6,7 @@
         "commands": [
             {
                 "match": "website/.*",
-                "cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create",
+                "cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create -logLevel warning -progress",
                 "silent": false
             }
         ]

pkg/filter/image_process.go

@@ -8,7 +8,6 @@ import (
 	"image/gif"
 	"image/jpeg"
 	"image/png"
-	"net/http"
 	"net/url"
 	"os"
 	"path"
@@ -66,20 +65,6 @@ func parseImageParams(str string) (*mark2web.ImagingConfig, error) {
 	return &p, nil
 }
 
-func getImageFromURL(url string) (image.Image, string, error) {
-	resp, err := http.Get(url)
-	if err != nil {
-		return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
-	}
-
-	img, format, err := image.Decode(resp.Body)
-	if err != nil {
-		return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
-	}
-
-	return img, format, nil
-}
-
 // ImageProcessFilter read the image url and process parameters and saves the resized/processed image
 // param: w=WITDH,h=HEIGHT
 func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
@@ -168,7 +153,7 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 				logger.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
 				if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
 					// remote file
-					img, p.Format, err = getImageFromURL(imgSource)
+					img, p.Format, err = helper.ImageWebRequest(imgSource)
 				} else {
 					img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
 				}

pkg/helper/webrequest.go

@@ -2,6 +2,8 @@ package helper
 
 import (
 	"encoding/json"
+	"fmt"
+	"image"
 	"io/ioutil"
 	"net/http"
 	"strings"
@@ -9,47 +11,97 @@ import (
 	"gitbase.de/apairon/mark2web/pkg/logger"
 )
 
-// JSONWebRequest will GET a json object/array from a given URL
+type wrImageEntry struct {
-func JSONWebRequest(url string) interface{} {
+	img    image.Image
+	format string
+}
+
+type wrJSONEntry struct {
+	data interface{}
+}
+
+var wrImageCache = make(map[string]*wrImageEntry)
+var wrJSONCache = make(map[string]*wrJSONEntry)
+
+// WebRequest will fetch an url and returns reponse
+func WebRequest(url string, opts interface{}) (*http.Response, error) {
 	logger.Log.Noticef("requesting url via GET %s", url)
 
-	resp, err := http.Get(url)
+	return http.Get(url)
-	if err != nil {
+}
-		logger.Log.Panicf("could not get url '%s': %s", url, err)
+
-	}
+// JSONWebRequest will GET a json object/array from a given URL
-	defer resp.Body.Close()
+func JSONWebRequest(url string) interface{} {
-
+	cached := wrJSONCache[url]
-	body, err := ioutil.ReadAll(resp.Body)
+	if cached == nil {
-	if err != nil {
+		resp, err := WebRequest(url, nil)
-		logger.Log.Panicf("could not read body from url '%s': %s", url, err)
+		if err != nil {
-	}
+			logger.Log.Panicf("could not get url '%s': %s", url, err)
-
+		}
-	logger.Log.Debugf("output from url '%s':\n%s", url, string(body))
+		defer resp.Body.Close()
-
+
-	if resp.StatusCode >= 400 {
+		body, err := ioutil.ReadAll(resp.Body)
-		logger.Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
+		if err != nil {
-	}
+			logger.Log.Panicf("could not read body from url '%s': %s", url, err)
-
+		}
-	contentType := resp.Header.Get("Content-Type")
+
-
+		logger.Log.Debugf("output from url '%s':\n%s", url, string(body))
-	if strings.Contains(contentType, "json") {
+
-
+		if resp.StatusCode >= 400 {
-	} else {
+			logger.Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
-		logger.Log.Panicf("is not json '%s' from url '%s'", contentType, url)
+		}
-	}
+
-
+		contentType := resp.Header.Get("Content-Type")
-	jsonMap := make(map[string]interface{})
+
-	err = json.Unmarshal(body, &jsonMap)
+		if strings.Contains(contentType, "json") {
-	if err == nil {
+
-		return jsonMap
+		} else {
-	}
+			logger.Log.Panicf("is not json '%s' from url '%s'", contentType, url)
-
+		}
-	jsonArrayMap := make([]map[string]interface{}, 0)
+
-	err = json.Unmarshal(body, &jsonArrayMap)
+		cached = new(wrJSONEntry)
-	if err == nil {
+
-		return jsonArrayMap
+		jsonMap := make(map[string]interface{})
-	}
+		err = json.Unmarshal(body, &jsonMap)
-
+		if err == nil {
-	logger.Log.Panicf("could not read json from '%s': invalid type", url)
+			cached.data = jsonMap
-	return nil
+		} else {
+			jsonArrayMap := make([]map[string]interface{}, 0)
+			err = json.Unmarshal(body, &jsonArrayMap)
+			if err == nil {
+				cached.data = jsonArrayMap
+			} else {
+				logger.P("could not read json from '%s': invalid type", url)
+			}
+		}
+
+		wrJSONCache[url] = cached
+
+	}
+	return cached.data
+}
+
+// ImageWebRequest gets an image from an url
+func ImageWebRequest(url string) (image.Image, string, error) {
+	cached := wrImageCache[url]
+	if cached == nil {
+		resp, err := WebRequest(url, nil)
+		if err != nil {
+			return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
+		}
+
+		img, format, err := image.Decode(resp.Body)
+		if err != nil {
+			return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
+		}
+
+		cached = &wrImageEntry{
+			img:    img,
+			format: format,
+		}
+
+		wrImageCache[url] = cached
+	}
+
+	return cached.img, cached.format, nil
 }

pkg/jobm/jobmanager.go

@@ -51,8 +51,8 @@ func Enqueue(jobs ...Job) {
 
 // Wait will wait for all jobs to finish
 func Wait() {
-	time.Sleep(time.Millisecond * 300)
 	close(jobChan)
+	time.Sleep(time.Millisecond * 500)
 	wg.Wait()
 }
 

pkg/mark2web/content.go

@@ -7,6 +7,8 @@ import (
 	"regexp"
 	"strings"
 
+	"gitbase.de/apairon/mark2web/pkg/progress"
+
 	"gitbase.de/apairon/mark2web/pkg/helper"
 	"gitbase.de/apairon/mark2web/pkg/logger"
 	"github.com/davecgh/go-spew/spew"
@@ -17,6 +19,9 @@ import (
 
 // ReadContentDir walks through content directory and builds the tree of configurations
 func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string, conf *PathConfig) {
+	progress.IncrTotal("content dir")
+	progress.DescribeCurrent("content dir", "found "+inBase)
+
 	if node.root == nil {
 		// first node is root
 		node.root = node
@@ -155,6 +160,8 @@ func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*Pa
 
 // ProcessContent walks recursivly through the input paths and processes all files for output
 func (node *TreeNode) ProcessContent() {
+	progress.DescribeCurrent("content dir", "processing "+node.InputPath)
+
 	helper.CreateDirectory(node.OutputPath)
 
 	if node.root != node {
@@ -293,6 +300,8 @@ func (node *TreeNode) ProcessContent() {
 		}
 	}
 
+	progress.IncrDone("content dir")
+
 	i := 0
 	// sub can dynamically increase, so no for range
 	for i < len(node.Sub) {

website/templates/base.html

@@ -31,11 +31,11 @@
 
 	<header id="header">
 		<div class="container">
-			<div class="logo" {% if not Data.slider and not Data.details.slider %}style="max-width: 400px;"{% endif %}>
+			<div class="logo" {% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}style="max-width: 400px;"{% endif %}>
 				<a href="{{ "/"|relative_path }}">
 					<picture>
 						<source media="(max-width: 768px)" srcset="project-files/img/logo_text.png">
-						<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider %}_text{% endif %}.png" alt="mark2web Logo">
+						<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}_text{% endif %}.png" alt="mark2web Logo">
 					</picture>
 				</a>
 			</div>