apairon/mark2web
1
1
Fork 0
Code Issues 8 Pull Requests Releases 5 Activity

cached webrequests

Browse Source
This commit is contained in:
Sebastian Frank 2019-03-25 15:07:02 +01:00
parent 267d1010bb
commit 234137f22f
Signed by: apairon
GPG Key ID: 7270D06DDA7FE8C3
6 changed files with 107 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.vscode/settings.json vendored
View File

@ -6,7 +6,7 @@
"commands": [
{
"match": "website/.*",
"cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create",
"cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create -logLevel warning -progress",
"silent": false
}
]

17
pkg/filter/image_process.go
View File

@ -8,7 +8,6 @@ import (
"image/gif"
"image/jpeg"
"image/png"
"net/http"
"net/url"
"os"
"path"
@ -66,20 +65,6 @@ func parseImageParams(str string) (*mark2web.ImagingConfig, error) {
return &p, nil
}
func getImageFromURL(url string) (image.Image, string, error) {
resp, err := http.Get(url)
if err != nil {
return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
}
img, format, err := image.Decode(resp.Body)
if err != nil {
return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
}
return img, format, nil
}
// ImageProcessFilter read the image url and process parameters and saves the resized/processed image
// param: w=WITDH,h=HEIGHT
func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
@ -168,7 +153,7 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
logger.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
// remote file
img, p.Format, err = getImageFromURL(imgSource)
img, p.Format, err = helper.ImageWebRequest(imgSource)
} else {
img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
}

134
pkg/helper/webrequest.go
View File

@ -2,6 +2,8 @@ package helper
import (
"encoding/json"
"fmt"
"image"
"io/ioutil"
"net/http"
"strings"
@ -9,47 +11,97 @@ import (
"gitbase.de/apairon/mark2web/pkg/logger"
)
// JSONWebRequest will GET a json object/array from a given URL
func JSONWebRequest(url string) interface{} {
type wrImageEntry struct {
img image.Image
format string
}
type wrJSONEntry struct {
data interface{}
}
var wrImageCache = make(map[string]*wrImageEntry)
var wrJSONCache = make(map[string]*wrJSONEntry)
// WebRequest will fetch an url and returns reponse
func WebRequest(url string, opts interface{}) (*http.Response, error) {
logger.Log.Noticef("requesting url via GET %s", url)
resp, err := http.Get(url)
if err != nil {
logger.Log.Panicf("could not get url '%s': %s", url, err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logger.Log.Panicf("could not read body from url '%s': %s", url, err)
}
logger.Log.Debugf("output from url '%s':\n%s", url, string(body))
if resp.StatusCode >= 400 {
logger.Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
}
contentType := resp.Header.Get("Content-Type")
if strings.Contains(contentType, "json") {
} else {
logger.Log.Panicf("is not json '%s' from url '%s'", contentType, url)
}
jsonMap := make(map[string]interface{})
err = json.Unmarshal(body, &jsonMap)
if err == nil {
return jsonMap
}
jsonArrayMap := make([]map[string]interface{}, 0)
err = json.Unmarshal(body, &jsonArrayMap)
if err == nil {
return jsonArrayMap
}
logger.Log.Panicf("could not read json from '%s': invalid type", url)
return nil
return http.Get(url)
}
// JSONWebRequest will GET a json object/array from a given URL
func JSONWebRequest(url string) interface{} {
cached := wrJSONCache[url]
if cached == nil {
resp, err := WebRequest(url, nil)
if err != nil {
logger.Log.Panicf("could not get url '%s': %s", url, err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logger.Log.Panicf("could not read body from url '%s': %s", url, err)
}
logger.Log.Debugf("output from url '%s':\n%s", url, string(body))
if resp.StatusCode >= 400 {
logger.Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
}
contentType := resp.Header.Get("Content-Type")
if strings.Contains(contentType, "json") {
} else {
logger.Log.Panicf("is not json '%s' from url '%s'", contentType, url)
}
cached = new(wrJSONEntry)
jsonMap := make(map[string]interface{})
err = json.Unmarshal(body, &jsonMap)
if err == nil {
cached.data = jsonMap
} else {
jsonArrayMap := make([]map[string]interface{}, 0)
err = json.Unmarshal(body, &jsonArrayMap)
if err == nil {
cached.data = jsonArrayMap
} else {
logger.P("could not read json from '%s': invalid type", url)
}
}
wrJSONCache[url] = cached
}
return cached.data
}
// ImageWebRequest gets an image from an url
func ImageWebRequest(url string) (image.Image, string, error) {
cached := wrImageCache[url]
if cached == nil {
resp, err := WebRequest(url, nil)
if err != nil {
return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
}
img, format, err := image.Decode(resp.Body)
if err != nil {
return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
}
cached = &wrImageEntry{
img: img,
format: format,
}
wrImageCache[url] = cached
}
return cached.img, cached.format, nil
}

2
pkg/jobm/jobmanager.go
View File

@ -51,8 +51,8 @@ func Enqueue(jobs ...Job) {
// Wait will wait for all jobs to finish
func Wait() {
time.Sleep(time.Millisecond * 300)
close(jobChan)
time.Sleep(time.Millisecond * 500)
wg.Wait()
}

9
pkg/mark2web/content.go
View File

@ -7,6 +7,8 @@ import (
"regexp"
"strings"
"gitbase.de/apairon/mark2web/pkg/progress"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"github.com/davecgh/go-spew/spew"
@ -17,6 +19,9 @@ import (
// ReadContentDir walks through content directory and builds the tree of configurations
func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string, conf *PathConfig) {
progress.IncrTotal("content dir")
progress.DescribeCurrent("content dir", "found "+inBase)
if node.root == nil {
// first node is root
node.root = node
@ -155,6 +160,8 @@ func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*Pa
// ProcessContent walks recursivly through the input paths and processes all files for output
func (node *TreeNode) ProcessContent() {
progress.DescribeCurrent("content dir", "processing "+node.InputPath)
helper.CreateDirectory(node.OutputPath)
if node.root != node {
@ -293,6 +300,8 @@ func (node *TreeNode) ProcessContent() {
}
}
progress.IncrDone("content dir")
i := 0
// sub can dynamically increase, so no for range
for i < len(node.Sub) {

4
website/templates/base.html
View File

@ -31,11 +31,11 @@
<header id="header">
<div class="container">
<div class="logo" {% if not Data.slider and not Data.details.slider %}style="max-width: 400px;"{% endif %}>
<div class="logo" {% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}style="max-width: 400px;"{% endif %}>
<a href="{{ "/"|relative_path }}">
<picture>
<source media="(max-width: 768px)" srcset="project-files/img/logo_text.png">
<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider %}_text{% endif %}.png" alt="mark2web Logo">
<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}_text{% endif %}.png" alt="mark2web Logo">
</picture>
</a>
</div>