diff --git a/build/RELEASE.md b/build/RELEASE.md index 37419a6..9df2759 100644 --- a/build/RELEASE.md +++ b/build/RELEASE.md @@ -1,3 +1,8 @@ NEUERUNGEN: -- `t=ZIEL_VERZEICHNIS` Parameter im `image_process` Filter +- Cached Collection Webrequests +- recursive Collections +- markdown-Filter `s=SYNTAX_HIGHLIGHT_SHEMA` Parameter +- image_process nutzt alle CPU-Kerne +- GZIP Vor-Komprimierung der Inhalte und Assets +- Code neu organisiert \ No newline at end of file diff --git a/build/VERSION b/build/VERSION index 8cfbc90..cff0b81 100644 --- a/build/VERSION +++ b/build/VERSION @@ -1 +1 @@ -1.1.1 \ No newline at end of file +1.2.0-pre \ No newline at end of file diff --git a/pkg/mark2web/assets.go b/pkg/mark2web/assets.go index bf965f5..51c5012 100644 --- a/pkg/mark2web/assets.go +++ b/pkg/mark2web/assets.go @@ -27,6 +27,10 @@ func ProcessAssets() { if err != nil { helper.Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err) } + + if Config.Assets.Compress { + compressFilesInDir(to) + } } } diff --git a/pkg/mark2web/collection.go b/pkg/mark2web/collection.go new file mode 100644 index 0000000..2d34b5b --- /dev/null +++ b/pkg/mark2web/collection.go @@ -0,0 +1,160 @@ +package mark2web + +import ( + "path" + "strings" + + "gitbase.de/apairon/mark2web/pkg/helper" + "github.com/davecgh/go-spew/spew" + "github.com/flosch/pongo2" +) + +type colCacheEntry struct { + data interface{} + hit int + navnames []string +} + +var colCache = make(map[string]*colCacheEntry) + +func (node *TreeNode) handleCollections() { + collections := append(node.Config.Collections, node.Config.This.Collections...) + for _, colConfig := range collections { + if colConfig != nil { + if colConfig.Name == nil || *colConfig.Name == "" { + helper.Log.Panicf("missing Name in collection config in '%s'", node.InputPath) + } + if colConfig.URL == nil || *colConfig.URL == "" { + helper.Log.Panicf("missing EntriesJSON in collection config in '%s'", node.InputPath) + } + } + + if node.ColMap == nil { + node.ColMap = make(helper.MapString) + } + ctx := NewContext() + ctx["This"] = node.Config.This + ctx["Data"] = node.Config.Data + + url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx) + if err != nil { + helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", node.InputPath, err) + } + + var colData interface{} + if cacheEntry, ok := colCache[url]; ok { + colData = cacheEntry.data + cacheEntry.hit++ + } else { + colData = helper.JSONWebRequest(url) + colCache[url] = &colCacheEntry{ + data: colData, + navnames: make([]string, 0), + } + } + + node.ColMap[*colConfig.Name] = colData + + if navT := colConfig.NavTemplate; navT != nil { + var entries []interface{} + var ok bool + if navT.EntriesAttribute != "" { + var colDataMap map[string]interface{} + if colDataMap, ok = colData.(map[string]interface{}); ok { + entries, ok = colDataMap[navT.EntriesAttribute].([]interface{}) + if !ok { + helper.Log.Debug(spew.Sdump(colDataMap)) + helper.Log.Panicf("invalid json data in [%s] from url '%s' for entries", navT.EntriesAttribute, url) + } + } + } else { + entries, ok = colData.([]interface{}) + } + if !ok { + helper.Log.Debug(spew.Sdump(colData)) + helper.Log.Panicf("invalid json data from url '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", url) + } + + // build navigation with detail sites + for idx, colEl := range entries { + ctxE := make(pongo2.Context) + err := helper.Merge(&ctxE, ctx) + if err != nil { + helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err) + } + var jsonCtx map[string]interface{} + if jsonCtx, ok = colEl.(map[string]interface{}); !ok { + helper.Log.Debug(spew.Sdump(colEl)) + helper.Log.Panicf("no json object for entry index %d from url '%s'", idx, url) + } + err = helper.Merge(&ctxE, pongo2.Context(jsonCtx)) + if err != nil { + helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err) + } + + tpl := "" + if navT.Template != "" { + tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE) + if err != nil { + helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", node.InputPath, err) + } + } + if tpl == "" { + tpl = *node.Config.Template + } + + dataKey := "" + if navT.DataKey != "" { + dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE) + if err != nil { + helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", node.InputPath, err) + } + } + + goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE) + if err != nil { + helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", node.InputPath, err) + } + goTo = strings.Trim(goTo, "/") + goTo = path.Clean(goTo) + + if strings.Contains(goTo, "..") { + helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo) + } + if goTo == "." { + helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath) + } + if goTo == "" { + helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath) + } + + navname := "" + if navT.Navname != "" { + navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE) + if err != nil { + helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", node.InputPath, err) + } + } + body := "" + if navT.Body != "" { + body, err = pongo2.RenderTemplateString(navT.Body, ctxE) + if err != nil { + helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", node.InputPath, err) + } + } + + if l := len(colCache[url].navnames); colCache[url].hit > 1 && + l > 0 && + navname == colCache[url].navnames[l-1] { + // navname before used same url, so recursion loop + helper.Log.Panicf("collection request loop detected for in '%s' for url: %s", node.InputPath, url) + } + + colCache[url].navnames = append(colCache[url].navnames, navname) + + node.addSubNode(tpl, goTo, navname, colEl, dataKey, body, navT.Hidden) + } + } + } + +} diff --git a/pkg/mark2web/compress.go b/pkg/mark2web/compress.go new file mode 100644 index 0000000..6b97007 --- /dev/null +++ b/pkg/mark2web/compress.go @@ -0,0 +1,89 @@ +package mark2web + +import ( + "bytes" + "compress/gzip" + "io" + "io/ioutil" + "os" + "path" + + "gitbase.de/apairon/mark2web/pkg/helper" +) + +func handleCompression(filename string, content []byte) { + ThreadStart(func() { + if _, ok := Config.Compress.Extensions[path.Ext(filename)]; ok { + + if Config.Compress.GZIP { + gzFilename := filename + ".gz" + + helper.Log.Infof("writing to compressed output file: %s", gzFilename) + var buf bytes.Buffer + + zw, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + if err != nil { + helper.Log.Panicf("could not initialize gzip writer for '%s': %s", filename, err) + } + + if content != nil { + // content given + _, err = zw.Write(content) + if err != nil { + helper.Log.Panicf("could not write gziped content for '%s': %s", filename, err) + } + } else { + // read file + f, err := os.Open(filename) + if err != nil { + helper.Log.Panicf("could not open file '%s': %s", filename, err) + } + defer f.Close() + _, err = io.Copy(zw, f) + if err != nil { + helper.Log.Panicf("could not gzip file '%s': %s", filename, err) + } + } + + err = zw.Close() + if err != nil { + helper.Log.Panicf("could not close gziped content for '%s': %s", filename, err) + } + + f, err := os.Create(gzFilename) + if err != nil { + helper.Log.Panicf("could not create file '%s': %s", gzFilename, err) + } + defer f.Close() + + _, err = buf.WriteTo(f) + if err != nil { + helper.Log.Panicf("could not write to file '%s': %s", gzFilename, err) + } + } + } + }) +} + +func compressFilesInDir(dir string) { + helper.Log.Noticef("compressing configured files in: %s", dir) + + var _processDir func(string) + _processDir = func(d string) { + entries, err := ioutil.ReadDir(d) + if err != nil { + helper.Log.Panicf("could not read dir '%s': %s", d, err) + } + + for _, entry := range entries { + if entry.IsDir() { + _processDir(d + "/" + entry.Name()) + } else { + handleCompression(d+"/"+entry.Name(), nil) + } + } + } + + _processDir(dir) + +} diff --git a/pkg/mark2web/config_global.go b/pkg/mark2web/config_global.go index a851598..9f53a15 100644 --- a/pkg/mark2web/config_global.go +++ b/pkg/mark2web/config_global.go @@ -13,6 +13,7 @@ type GlobalConfig struct { } `yaml:"Webserver"` Assets struct { + Compress bool `yaml:"Compress"` FromPath string `yaml:"FromPath"` ToPath string `yaml:"ToPath"` Action string `yaml:"Action"` @@ -26,6 +27,11 @@ type GlobalConfig struct { Action string `yaml:"Action"` } `yaml:"OtherFiles"` + Compress struct { + GZIP bool `yaml:"GZIP"` + Extensions map[string]string `yaml:"Extensions"` + } `yaml:"Compress"` + Directories struct { Input string Output string diff --git a/pkg/mark2web/config_path.go b/pkg/mark2web/config_path.go index e8af503..8da2020 100644 --- a/pkg/mark2web/config_path.go +++ b/pkg/mark2web/config_path.go @@ -84,4 +84,7 @@ type PathConfig struct { Imaging *ImagingConfig `yaml:"Imaging"` Data helper.MapString `yaml:"Data"` + + // Collections here are recursive if saved as nav, so request should be filtered + Collections []*CollectionConfig `yaml:"Collections"` } diff --git a/pkg/mark2web/content.go b/pkg/mark2web/content.go index 5b0ebde..e5bfc39 100644 --- a/pkg/mark2web/content.go +++ b/pkg/mark2web/content.go @@ -84,18 +84,7 @@ func (node *TreeNode) ProcessContent() { } goToFixed = path.Clean(goToFixed) - switch Config.Webserver.Type { - case "apache": - htaccessFile := node.OutputPath + "/.htaccess" - helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed) - err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on -RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L] -`), 0644) - if err != nil { - helper.Log.Panicf("could not write '%s': %s", htaccessFile, err) - } - break - } + htaccessRedirect(node.OutputPath, goToFixed) } for _, file := range node.InputFiles { @@ -274,6 +263,8 @@ RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L] helper.Log.Panicf("could not write to output file '%s': %s", outFile, err) } + handleCompression(outFile, []byte(result)) + //fmt.Println(string(html)) } } @@ -289,6 +280,8 @@ RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L] if err != nil { helper.Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err) } + + handleCompression(to, nil) } } diff --git a/pkg/mark2web/context.go b/pkg/mark2web/context.go index 0d916ca..981f48a 100644 --- a/pkg/mark2web/context.go +++ b/pkg/mark2web/context.go @@ -35,127 +35,6 @@ func NewContext() pongo2.Context { return ctx } -func (node *TreeNode) handleCollections() { - for _, colConfig := range node.Config.This.Collections { - if colConfig != nil { - if colConfig.Name == nil || *colConfig.Name == "" { - helper.Log.Panicf("missing Name in collection config in '%s'", node.InputPath) - } - if colConfig.URL == nil || *colConfig.URL == "" { - helper.Log.Panicf("missing EntriesJSON in collection config in '%s'", node.InputPath) - } - } - - if node.ColMap == nil { - node.ColMap = make(helper.MapString) - } - ctx := NewContext() - ctx["This"] = node.Config.This - ctx["Data"] = node.Config.Data - - url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx) - if err != nil { - helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", node.InputPath, err) - } - - colData := helper.JSONWebRequest(url) - node.ColMap[*colConfig.Name] = colData - - if navT := colConfig.NavTemplate; navT != nil { - var entries []interface{} - var ok bool - if navT.EntriesAttribute != "" { - var colDataMap map[string]interface{} - if colDataMap, ok = colData.(map[string]interface{}); ok { - entries, ok = colDataMap[navT.EntriesAttribute].([]interface{}) - if !ok { - helper.Log.Debug(spew.Sdump(colDataMap)) - helper.Log.Panicf("invalid json data in [%s] from url '%s' for entries", navT.EntriesAttribute, url) - } - } - } else { - entries, ok = colData.([]interface{}) - } - if !ok { - helper.Log.Debug(spew.Sdump(colData)) - helper.Log.Panicf("invalid json data from url '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", url) - } - - // build navigation with detail sites - for idx, colEl := range entries { - ctxE := make(pongo2.Context) - err := helper.Merge(&ctxE, ctx) - if err != nil { - helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err) - } - var jsonCtx map[string]interface{} - if jsonCtx, ok = colEl.(map[string]interface{}); !ok { - helper.Log.Debug(spew.Sdump(colEl)) - helper.Log.Panicf("no json object for entry index %d from url '%s'", idx, url) - } - err = helper.Merge(&ctxE, pongo2.Context(jsonCtx)) - if err != nil { - helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err) - } - - tpl := "" - if navT.Template != "" { - tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE) - if err != nil { - helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", node.InputPath, err) - } - } - if tpl == "" { - tpl = *node.Config.Template - } - - dataKey := "" - if navT.DataKey != "" { - dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE) - if err != nil { - helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", node.InputPath, err) - } - } - - goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE) - if err != nil { - helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", node.InputPath, err) - } - goTo = strings.Trim(goTo, "/") - goTo = path.Clean(goTo) - - if strings.Contains(goTo, "..") { - helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo) - } - if goTo == "." { - helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath) - } - if goTo == "" { - helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath) - } - - navname := "" - if navT.Navname != "" { - navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE) - if err != nil { - helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", node.InputPath, err) - } - } - body := "" - if navT.Body != "" { - body, err = pongo2.RenderTemplateString(navT.Body, ctxE) - if err != nil { - helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", node.InputPath, err) - } - } - - node.addSubNode(tpl, goTo, navname, colEl, dataKey, body, navT.Hidden) - } - } - } - -} - func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfig) { inPath := inBase if subDir != "" { @@ -229,28 +108,47 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx interface{}, dataMapKey string, body string, hideInNav bool) { newNode := new(TreeNode) newNode.root = node.root - newNode.fillConfig( - node.InputPath, - node.OutputPath, - subDir, - node.Config, - ) + + newPathConfig := new(PathConfig) if navname != "" { - newNode.Config.This = ThisPathConfig{ + newPathConfig.This = ThisPathConfig{ Navname: &navname, } } if dataMapKey != "" { - if newNode.Config.Data == nil { - newNode.Config.Data = make(helper.MapString) + if newPathConfig.Data == nil { + newPathConfig.Data = make(helper.MapString) } // as submap in Data - newNode.Config.Data[dataMapKey] = ctx + newPathConfig.Data[dataMapKey] = ctx } else if m, ok := ctx.(map[string]interface{}); ok { // direct set data - newNode.Config.Data = m + newPathConfig.Data = m } + mergedConfig := new(PathConfig) + err := helper.Merge(mergedConfig, node.Config) + if err != nil { + helper.Log.Panicf("merge of path config failed: %s", err) + } + // dont merge Data[DataKey] + if dataMapKey != "" { + mergedConfig.Data[dataMapKey] = nil + } else { + mergedConfig.Data = make(helper.MapString) + } + err = helper.Merge(mergedConfig, newPathConfig) + if err != nil { + helper.Log.Panicf("merge of path config failed: %s", err) + } + + newNode.fillConfig( + node.InputPath, + node.OutputPath, + subDir, + mergedConfig, + ) + // fake via normal file behavior newNode.Config.Template = &tplFilename newNode.InputFiles = []string{""} // empty file is special for use InputString diff --git a/pkg/mark2web/htaccess.go b/pkg/mark2web/htaccess.go new file mode 100644 index 0000000..95117ea --- /dev/null +++ b/pkg/mark2web/htaccess.go @@ -0,0 +1,67 @@ +package mark2web + +import ( + "io/ioutil" + "regexp" + + "gitbase.de/apairon/mark2web/pkg/helper" +) + +func htaccessRedirect(outDir, goTo string) { + switch Config.Webserver.Type { + case "apache": + htaccessFile := outDir + "/.htaccess" + helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goTo) + err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on +RewriteRule ^$ %{REQUEST_URI}`+goTo+`/ [R,L] +`), 0644) + if err != nil { + helper.Log.Panicf("could not write '%s': %s", htaccessFile, err) + } + } +} + +// WriteWebserverConfig build the config for pre compression and more +func WriteWebserverConfig() { + switch Config.Webserver.Type { + case "apache": + configStr := ` + AddCharset UTF-8 .html + AddCharset UTF-8 .json + AddCharset UTF-8 .js + AddCharset UTF-8 .css + + RewriteEngine on +` + + if Config.Compress.GZIP { + for ext, contentType := range Config.Compress.Extensions { + rExt := regexp.QuoteMeta(ext) + + configStr += ` + RewriteCond "%{HTTP:Accept-encoding}" "gzip" + RewriteCond "%{REQUEST_FILENAME}\.gz" -s + RewriteRule "^(.*)` + rExt + `" "$1` + rExt + `\.gz" [QSA] + + RewriteRule "` + rExt + `\.gz$" "-" [E=no-gzip:1] + + + ForceType '` + contentType + `; charset=UTF-8' + Header append Content-Encoding gzip + Header append Vary Accept-Encoding + + +` + } + } + + if configStr != "" { + htaccessFile := Config.Directories.Output + "/.htaccess" + helper.Log.Noticef("writing webserver config to: %s", htaccessFile) + err := ioutil.WriteFile(htaccessFile, []byte(configStr), 0644) + if err != nil { + helper.Log.Panicf("could not write '%s': %s", htaccessFile, err) + } + } + } +} diff --git a/pkg/mark2web/run.go b/pkg/mark2web/run.go index 203f545..3fda4d5 100644 --- a/pkg/mark2web/run.go +++ b/pkg/mark2web/run.go @@ -10,5 +10,7 @@ func Run(inDir, outDir string, defaultPathConfig *PathConfig) { ProcessAssets() + WriteWebserverConfig() + Wait() } diff --git a/test/test.rest b/test/test.rest index 1943a7d..4059dd2 100644 --- a/test/test.rest +++ b/test/test.rest @@ -2,6 +2,7 @@ GET https://mark2web.basiscms.de/api/collections/get/mark2webBlog ?sort[date]=-1 &limit=101 &token=985cee34099f4d3b08f18fc22f6296 + &filter[link][$exists]=0 ### diff --git a/website/config.yml b/website/config.yml index 7c7e4c2..4ccce57 100644 --- a/website/config.yml +++ b/website/config.yml @@ -2,6 +2,7 @@ Webserver: Type: "apache" # generates .htaccess Assets: + Compress: True FromPath: "project-files" ToPath: "project-files" Action: "copy" # symlink, copy or move @@ -12,3 +13,9 @@ Assets: OtherFiles: Action: "copy" +Compress: + GZIP: True + Extensions: + .html: text/html + .css: text/css + .js: text/javascript \ No newline at end of file diff --git a/website/content/de/01_Navigation/04_Blog/config.yml b/website/content/de/01_Navigation/04_Blog/config.yml index 060561e..b22e788 100644 --- a/website/content/de/01_Navigation/04_Blog/config.yml +++ b/website/content/de/01_Navigation/04_Blog/config.yml @@ -1,7 +1,7 @@ -This: +#This: Collections: - Name: blog1st - URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1' + URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}' NavTemplate: EntriesAttribute: entries GoTo: '{{ date }}-{{ title }}' @@ -12,7 +12,7 @@ This: Hidden: true # hide from nav, but use this feature for rendering detail sites - Name: blog1skip - URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100' + URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}' NavTemplate: EntriesAttribute: entries GoTo: '{{ date }}-{{ title }}' diff --git a/website/project-files/css/main.css b/website/project-files/css/main.css index d8a4c74..5b735cb 100755 --- a/website/project-files/css/main.css +++ b/website/project-files/css/main.css @@ -169,8 +169,8 @@ label {font-weight:600;} #header { position:absolute; top:50px; - left:150px; - right:150px; + left:0px; + right:0px; height:auto; z-index:300; background:none; diff --git a/website/project-files/js/preloader.js b/website/project-files/js/preloader.js index 8bd54b4..7fd76ae 100755 --- a/website/project-files/js/preloader.js +++ b/website/project-files/js/preloader.js @@ -1,6 +1,6 @@ $('#preloader').fadeIn('fast'); $(window).on('load', function() { $('.spinner').fadeOut(); - $('#preloader').delay(350).fadeOut('slow'); - $('body').delay(350).css({'overflow':'visible'}); + $('#preloader').delay(100).fadeOut('slow'); + $('body').delay(100).css({'overflow':'visible'}); });