gzip pre compression
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Sebastian Frank 2019-03-21 14:55:40 +01:00
parent 50139c6f51
commit 23fd5fe1d4
Signed by: apairon
GPG Key ID: 7270D06DDA7FE8C3
16 changed files with 388 additions and 153 deletions

View File

@ -1,3 +1,8 @@
NEUERUNGEN:
- `t=ZIEL_VERZEICHNIS` Parameter im `image_process` Filter
- Cached Collection Webrequests
- recursive Collections
- markdown-Filter `s=SYNTAX_HIGHLIGHT_SHEMA` Parameter
- image_process nutzt alle CPU-Kerne
- GZIP Vor-Komprimierung der Inhalte und Assets
- Code neu organisiert

View File

@ -1 +1 @@
1.1.1
1.2.0-pre

View File

@ -27,6 +27,10 @@ func ProcessAssets() {
if err != nil {
helper.Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
}
if Config.Assets.Compress {
compressFilesInDir(to)
}
}
}

160
pkg/mark2web/collection.go Normal file
View File

@ -0,0 +1,160 @@
package mark2web
import (
"path"
"strings"
"gitbase.de/apairon/mark2web/pkg/helper"
"github.com/davecgh/go-spew/spew"
"github.com/flosch/pongo2"
)
type colCacheEntry struct {
data interface{}
hit int
navnames []string
}
var colCache = make(map[string]*colCacheEntry)
func (node *TreeNode) handleCollections() {
collections := append(node.Config.Collections, node.Config.This.Collections...)
for _, colConfig := range collections {
if colConfig != nil {
if colConfig.Name == nil || *colConfig.Name == "" {
helper.Log.Panicf("missing Name in collection config in '%s'", node.InputPath)
}
if colConfig.URL == nil || *colConfig.URL == "" {
helper.Log.Panicf("missing EntriesJSON in collection config in '%s'", node.InputPath)
}
}
if node.ColMap == nil {
node.ColMap = make(helper.MapString)
}
ctx := NewContext()
ctx["This"] = node.Config.This
ctx["Data"] = node.Config.Data
url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
if err != nil {
helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", node.InputPath, err)
}
var colData interface{}
if cacheEntry, ok := colCache[url]; ok {
colData = cacheEntry.data
cacheEntry.hit++
} else {
colData = helper.JSONWebRequest(url)
colCache[url] = &colCacheEntry{
data: colData,
navnames: make([]string, 0),
}
}
node.ColMap[*colConfig.Name] = colData
if navT := colConfig.NavTemplate; navT != nil {
var entries []interface{}
var ok bool
if navT.EntriesAttribute != "" {
var colDataMap map[string]interface{}
if colDataMap, ok = colData.(map[string]interface{}); ok {
entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
if !ok {
helper.Log.Debug(spew.Sdump(colDataMap))
helper.Log.Panicf("invalid json data in [%s] from url '%s' for entries", navT.EntriesAttribute, url)
}
}
} else {
entries, ok = colData.([]interface{})
}
if !ok {
helper.Log.Debug(spew.Sdump(colData))
helper.Log.Panicf("invalid json data from url '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", url)
}
// build navigation with detail sites
for idx, colEl := range entries {
ctxE := make(pongo2.Context)
err := helper.Merge(&ctxE, ctx)
if err != nil {
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
}
var jsonCtx map[string]interface{}
if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
helper.Log.Debug(spew.Sdump(colEl))
helper.Log.Panicf("no json object for entry index %d from url '%s'", idx, url)
}
err = helper.Merge(&ctxE, pongo2.Context(jsonCtx))
if err != nil {
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
}
tpl := ""
if navT.Template != "" {
tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", node.InputPath, err)
}
}
if tpl == "" {
tpl = *node.Config.Template
}
dataKey := ""
if navT.DataKey != "" {
dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", node.InputPath, err)
}
}
goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", node.InputPath, err)
}
goTo = strings.Trim(goTo, "/")
goTo = path.Clean(goTo)
if strings.Contains(goTo, "..") {
helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
}
if goTo == "." {
helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
}
if goTo == "" {
helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
}
navname := ""
if navT.Navname != "" {
navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", node.InputPath, err)
}
}
body := ""
if navT.Body != "" {
body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", node.InputPath, err)
}
}
if l := len(colCache[url].navnames); colCache[url].hit > 1 &&
l > 0 &&
navname == colCache[url].navnames[l-1] {
// navname before used same url, so recursion loop
helper.Log.Panicf("collection request loop detected for in '%s' for url: %s", node.InputPath, url)
}
colCache[url].navnames = append(colCache[url].navnames, navname)
node.addSubNode(tpl, goTo, navname, colEl, dataKey, body, navT.Hidden)
}
}
}
}

89
pkg/mark2web/compress.go Normal file
View File

@ -0,0 +1,89 @@
package mark2web
import (
"bytes"
"compress/gzip"
"io"
"io/ioutil"
"os"
"path"
"gitbase.de/apairon/mark2web/pkg/helper"
)
func handleCompression(filename string, content []byte) {
ThreadStart(func() {
if _, ok := Config.Compress.Extensions[path.Ext(filename)]; ok {
if Config.Compress.GZIP {
gzFilename := filename + ".gz"
helper.Log.Infof("writing to compressed output file: %s", gzFilename)
var buf bytes.Buffer
zw, err := gzip.NewWriterLevel(&buf, gzip.BestCompression)
if err != nil {
helper.Log.Panicf("could not initialize gzip writer for '%s': %s", filename, err)
}
if content != nil {
// content given
_, err = zw.Write(content)
if err != nil {
helper.Log.Panicf("could not write gziped content for '%s': %s", filename, err)
}
} else {
// read file
f, err := os.Open(filename)
if err != nil {
helper.Log.Panicf("could not open file '%s': %s", filename, err)
}
defer f.Close()
_, err = io.Copy(zw, f)
if err != nil {
helper.Log.Panicf("could not gzip file '%s': %s", filename, err)
}
}
err = zw.Close()
if err != nil {
helper.Log.Panicf("could not close gziped content for '%s': %s", filename, err)
}
f, err := os.Create(gzFilename)
if err != nil {
helper.Log.Panicf("could not create file '%s': %s", gzFilename, err)
}
defer f.Close()
_, err = buf.WriteTo(f)
if err != nil {
helper.Log.Panicf("could not write to file '%s': %s", gzFilename, err)
}
}
}
})
}
func compressFilesInDir(dir string) {
helper.Log.Noticef("compressing configured files in: %s", dir)
var _processDir func(string)
_processDir = func(d string) {
entries, err := ioutil.ReadDir(d)
if err != nil {
helper.Log.Panicf("could not read dir '%s': %s", d, err)
}
for _, entry := range entries {
if entry.IsDir() {
_processDir(d + "/" + entry.Name())
} else {
handleCompression(d+"/"+entry.Name(), nil)
}
}
}
_processDir(dir)
}

View File

@ -13,6 +13,7 @@ type GlobalConfig struct {
} `yaml:"Webserver"`
Assets struct {
Compress bool `yaml:"Compress"`
FromPath string `yaml:"FromPath"`
ToPath string `yaml:"ToPath"`
Action string `yaml:"Action"`
@ -26,6 +27,11 @@ type GlobalConfig struct {
Action string `yaml:"Action"`
} `yaml:"OtherFiles"`
Compress struct {
GZIP bool `yaml:"GZIP"`
Extensions map[string]string `yaml:"Extensions"`
} `yaml:"Compress"`
Directories struct {
Input string
Output string

View File

@ -84,4 +84,7 @@ type PathConfig struct {
Imaging *ImagingConfig `yaml:"Imaging"`
Data helper.MapString `yaml:"Data"`
// Collections here are recursive if saved as nav, so request should be filtered
Collections []*CollectionConfig `yaml:"Collections"`
}

View File

@ -84,18 +84,7 @@ func (node *TreeNode) ProcessContent() {
}
goToFixed = path.Clean(goToFixed)
switch Config.Webserver.Type {
case "apache":
htaccessFile := node.OutputPath + "/.htaccess"
helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed)
err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
`), 0644)
if err != nil {
helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
}
break
}
htaccessRedirect(node.OutputPath, goToFixed)
}
for _, file := range node.InputFiles {
@ -274,6 +263,8 @@ RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
helper.Log.Panicf("could not write to output file '%s': %s", outFile, err)
}
handleCompression(outFile, []byte(result))
//fmt.Println(string(html))
}
}
@ -289,6 +280,8 @@ RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
if err != nil {
helper.Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
}
handleCompression(to, nil)
}
}

View File

@ -35,127 +35,6 @@ func NewContext() pongo2.Context {
return ctx
}
func (node *TreeNode) handleCollections() {
for _, colConfig := range node.Config.This.Collections {
if colConfig != nil {
if colConfig.Name == nil || *colConfig.Name == "" {
helper.Log.Panicf("missing Name in collection config in '%s'", node.InputPath)
}
if colConfig.URL == nil || *colConfig.URL == "" {
helper.Log.Panicf("missing EntriesJSON in collection config in '%s'", node.InputPath)
}
}
if node.ColMap == nil {
node.ColMap = make(helper.MapString)
}
ctx := NewContext()
ctx["This"] = node.Config.This
ctx["Data"] = node.Config.Data
url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
if err != nil {
helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", node.InputPath, err)
}
colData := helper.JSONWebRequest(url)
node.ColMap[*colConfig.Name] = colData
if navT := colConfig.NavTemplate; navT != nil {
var entries []interface{}
var ok bool
if navT.EntriesAttribute != "" {
var colDataMap map[string]interface{}
if colDataMap, ok = colData.(map[string]interface{}); ok {
entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
if !ok {
helper.Log.Debug(spew.Sdump(colDataMap))
helper.Log.Panicf("invalid json data in [%s] from url '%s' for entries", navT.EntriesAttribute, url)
}
}
} else {
entries, ok = colData.([]interface{})
}
if !ok {
helper.Log.Debug(spew.Sdump(colData))
helper.Log.Panicf("invalid json data from url '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", url)
}
// build navigation with detail sites
for idx, colEl := range entries {
ctxE := make(pongo2.Context)
err := helper.Merge(&ctxE, ctx)
if err != nil {
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
}
var jsonCtx map[string]interface{}
if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
helper.Log.Debug(spew.Sdump(colEl))
helper.Log.Panicf("no json object for entry index %d from url '%s'", idx, url)
}
err = helper.Merge(&ctxE, pongo2.Context(jsonCtx))
if err != nil {
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
}
tpl := ""
if navT.Template != "" {
tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", node.InputPath, err)
}
}
if tpl == "" {
tpl = *node.Config.Template
}
dataKey := ""
if navT.DataKey != "" {
dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", node.InputPath, err)
}
}
goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", node.InputPath, err)
}
goTo = strings.Trim(goTo, "/")
goTo = path.Clean(goTo)
if strings.Contains(goTo, "..") {
helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
}
if goTo == "." {
helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
}
if goTo == "" {
helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
}
navname := ""
if navT.Navname != "" {
navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", node.InputPath, err)
}
}
body := ""
if navT.Body != "" {
body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", node.InputPath, err)
}
}
node.addSubNode(tpl, goTo, navname, colEl, dataKey, body, navT.Hidden)
}
}
}
}
func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfig) {
inPath := inBase
if subDir != "" {
@ -229,28 +108,47 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx interface{}, dataMapKey string, body string, hideInNav bool) {
newNode := new(TreeNode)
newNode.root = node.root
newNode.fillConfig(
node.InputPath,
node.OutputPath,
subDir,
node.Config,
)
newPathConfig := new(PathConfig)
if navname != "" {
newNode.Config.This = ThisPathConfig{
newPathConfig.This = ThisPathConfig{
Navname: &navname,
}
}
if dataMapKey != "" {
if newNode.Config.Data == nil {
newNode.Config.Data = make(helper.MapString)
if newPathConfig.Data == nil {
newPathConfig.Data = make(helper.MapString)
}
// as submap in Data
newNode.Config.Data[dataMapKey] = ctx
newPathConfig.Data[dataMapKey] = ctx
} else if m, ok := ctx.(map[string]interface{}); ok {
// direct set data
newNode.Config.Data = m
newPathConfig.Data = m
}
mergedConfig := new(PathConfig)
err := helper.Merge(mergedConfig, node.Config)
if err != nil {
helper.Log.Panicf("merge of path config failed: %s", err)
}
// dont merge Data[DataKey]
if dataMapKey != "" {
mergedConfig.Data[dataMapKey] = nil
} else {
mergedConfig.Data = make(helper.MapString)
}
err = helper.Merge(mergedConfig, newPathConfig)
if err != nil {
helper.Log.Panicf("merge of path config failed: %s", err)
}
newNode.fillConfig(
node.InputPath,
node.OutputPath,
subDir,
mergedConfig,
)
// fake via normal file behavior
newNode.Config.Template = &tplFilename
newNode.InputFiles = []string{""} // empty file is special for use InputString

67
pkg/mark2web/htaccess.go Normal file
View File

@ -0,0 +1,67 @@
package mark2web
import (
"io/ioutil"
"regexp"
"gitbase.de/apairon/mark2web/pkg/helper"
)
func htaccessRedirect(outDir, goTo string) {
switch Config.Webserver.Type {
case "apache":
htaccessFile := outDir + "/.htaccess"
helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goTo)
err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
RewriteRule ^$ %{REQUEST_URI}`+goTo+`/ [R,L]
`), 0644)
if err != nil {
helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
}
}
}
// WriteWebserverConfig build the config for pre compression and more
func WriteWebserverConfig() {
switch Config.Webserver.Type {
case "apache":
configStr := `
AddCharset UTF-8 .html
AddCharset UTF-8 .json
AddCharset UTF-8 .js
AddCharset UTF-8 .css
RewriteEngine on
`
if Config.Compress.GZIP {
for ext, contentType := range Config.Compress.Extensions {
rExt := regexp.QuoteMeta(ext)
configStr += `
RewriteCond "%{HTTP:Accept-encoding}" "gzip"
RewriteCond "%{REQUEST_FILENAME}\.gz" -s
RewriteRule "^(.*)` + rExt + `" "$1` + rExt + `\.gz" [QSA]
RewriteRule "` + rExt + `\.gz$" "-" [E=no-gzip:1]
<FilesMatch "` + rExt + `\.gz$">
ForceType '` + contentType + `; charset=UTF-8'
Header append Content-Encoding gzip
Header append Vary Accept-Encoding
</FilesMatch>
`
}
}
if configStr != "" {
htaccessFile := Config.Directories.Output + "/.htaccess"
helper.Log.Noticef("writing webserver config to: %s", htaccessFile)
err := ioutil.WriteFile(htaccessFile, []byte(configStr), 0644)
if err != nil {
helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
}
}
}
}

View File

@ -10,5 +10,7 @@ func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
ProcessAssets()
WriteWebserverConfig()
Wait()
}

View File

@ -2,6 +2,7 @@ GET https://mark2web.basiscms.de/api/collections/get/mark2webBlog
?sort[date]=-1
&limit=101
&token=985cee34099f4d3b08f18fc22f6296
&filter[link][$exists]=0
###

View File

@ -2,6 +2,7 @@ Webserver:
Type: "apache" # generates .htaccess
Assets:
Compress: True
FromPath: "project-files"
ToPath: "project-files"
Action: "copy" # symlink, copy or move
@ -12,3 +13,9 @@ Assets:
OtherFiles:
Action: "copy"
Compress:
GZIP: True
Extensions:
.html: text/html
.css: text/css
.js: text/javascript

View File

@ -1,7 +1,7 @@
This:
#This:
Collections:
- Name: blog1st
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1'
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}'
NavTemplate:
EntriesAttribute: entries
GoTo: '{{ date }}-{{ title }}'
@ -12,7 +12,7 @@ This:
Hidden: true # hide from nav, but use this feature for rendering detail sites
- Name: blog1skip
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100'
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}'
NavTemplate:
EntriesAttribute: entries
GoTo: '{{ date }}-{{ title }}'

View File

@ -169,8 +169,8 @@ label {font-weight:600;}
#header {
position:absolute;
top:50px;
left:150px;
right:150px;
left:0px;
right:0px;
height:auto;
z-index:300;
background:none;

View File

@ -1,6 +1,6 @@
$('#preloader').fadeIn('fast');
$(window).on('load', function() {
$('.spinner').fadeOut();
$('#preloader').delay(350).fadeOut('slow');
$('body').delay(350).css({'overflow':'visible'});
$('#preloader').delay(100).fadeOut('slow');
$('body').delay(100).css({'overflow':'visible'});
});