collections via markdown files

.gitignore

@@ -1,3 +1,4 @@
-test.html
-html/
-mark2web
+/html/
+/build/dist
+/coverage.out
+/test/out

.gitmodules

@@ -25,9 +25,6 @@
 [submodule "vendor/github.com/davecgh/go-spew"]
 	path = vendor/github.com/davecgh/go-spew
 	url = https://github.com/davecgh/go-spew
-[submodule "vendor/github.com/aymerick/raymond"]
-	path = vendor/github.com/aymerick/raymond
-	url = https://github.com/aymerick/raymond
 [submodule "vendor/github.com/imdario/mergo"]
 	path = vendor/github.com/imdario/mergo
 	url = https://github.com/imdario/mergo
@@ -40,9 +37,45 @@
 [submodule "vendor/github.com/juju/errors"]
 	path = vendor/github.com/juju/errors
 	url = https://github.com/juju/errors
-[submodule "vendor/github.com/gosimple/slug"]
-	path = vendor/github.com/gosimple/slug
-	url = https://github.com/gosimple/slug
-[submodule "vendor/github.com/rainycape/unidecode"]
-	path = vendor/github.com/rainycape/unidecode
-	url = https://github.com/rainycape/unidecode
+[submodule "vendor/github.com/flosch/pongo2-addons"]
+	path = vendor/github.com/flosch/pongo2-addons
+	url = https://github.com/flosch/pongo2-addons
+[submodule "vendor/github.com/extemporalgenome/slug"]
+	path = vendor/github.com/extemporalgenome/slug
+	url = https://github.com/extemporalgenome/slug
+[submodule "vendor/golang.org/x/text"]
+	path = vendor/golang.org/x/text
+	url = https://go.googlesource.com/text
+[submodule "vendor/github.com/flosch/go-humanize"]
+	path = vendor/github.com/flosch/go-humanize
+	url = https://github.com/flosch/go-humanize
+[submodule "vendor/github.com/russross/blackfriday"]
+	path = vendor/github.com/russross/blackfriday
+	url = https://github.com/russross/blackfriday
+[submodule "vendor/github.com/robertkrimen/otto"]
+	path = vendor/github.com/robertkrimen/otto
+	url = https://github.com/robertkrimen/otto
+[submodule "vendor/gopkg.in/sourcemap.v1"]
+	path = vendor/gopkg.in/sourcemap.v1
+	url = https://gopkg.in/sourcemap.v1
+[submodule "vendor/github.com/ddliu/motto"]
+	path = vendor/github.com/ddliu/motto
+	url = https://github.com/ddliu/motto
+[submodule "vendor/github.com/disintegration/imaging"]
+	path = vendor/github.com/disintegration/imaging
+	url = https://github.com/disintegration/imaging
+[submodule "vendor/golang.org/x/image"]
+	path = vendor/golang.org/x/image
+	url = https://go.googlesource.com/image
+[submodule "vendor/github.com/smartystreets/goconvey"]
+	path = vendor/github.com/smartystreets/goconvey
+	url = https://github.com/smartystreets/goconvey
+[submodule "vendor/github.com/jtolds/gls"]
+	path = vendor/github.com/jtolds/gls
+	url = https://github.com/jtolds/gls
+[submodule "vendor/github.com/smartystreets/assertions"]
+	path = vendor/github.com/smartystreets/assertions
+	url = https://github.com/smartystreets/assertions
+[submodule "vendor/github.com/itchio/go-brotli"]
+	path = vendor/github.com/itchio/go-brotli
+	url = https://github.com/itchio/go-brotli

.vscode/settings.json

@@ -1,5 +1,14 @@
 {
     "files.associations": {
         "**/templates/*.html": "django-html"
+    },
+    "saveAndRun": {
+        "commands": [
+            {
+                "match": "website/.*",
+                "cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create",
+                "silent": false
+            }
+        ]
     }
 }

Dockerfile

@@ -1,4 +0,0 @@
-FROM alpine
-RUN apk update && apk add openssh-client sshpass rsync gzip && rm -r /var/cache/
-ADD mark2web /
-CMD ["/mark2web"]

LICENSE

@@ -0,0 +1,26 @@
+Copyright (c) 2019, Sebastian Frank
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
+ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+The views and conclusions contained in the software and documentation are those
+of the authors and should not be interpreted as representing official policies,
+either expressed or implied, of the mark2web project.

README.md

@@ -1,5 +1,7 @@
 # mark2web
 
+[![Build Status](https://ci.basehosts.de/api/badges/apairon/mark2web/status.svg)](https://ci.basehosts.de/apairon/mark2web)
+
 mark2web ist ein Website-Generator, der als Eingabe Markdown-Dateien, Templates und Konfigurations-Dateien nutzt.
 
 Die vorgesehene Arbeitsweise ist die Pflege der Inhalte über eine Versionsverwaltung (z.B. git) und anschließende CI/CD-Pipeline, welche den Generator aufruft und die fertige Website publiziert.
@@ -8,4 +10,8 @@ Die vorgesehene Arbeitsweise ist die Pflege der Inhalte über eine Versionsverwa
 
 Die Dokumentation ist auf der [mark2web-Website](https://www.mark2web.de/) zu finden. Außerdem ist die Dokumentation im Verzeichnis `website/content` dieses Repositories, da dies das Ausgangsmaterial der Projekt-Website ist.  
   
-Die öffentliche Website ist mit **mark2web** generiert.
+Die öffentliche Website ist mit **mark2web** generiert.
+
+## Lizenz
+
+Das Projekt **mark2web** unterliegt der Lizenz "Simplified BSD License" (siehe [LICENSE](LICENSE)).

RELEASE.md

@@ -1,8 +0,0 @@
-Dies ist das erste Release von *mark2web*.  
-In diesem Release werden folgende Features unterstützt:
-
-- Verabeitung von Markdown-Dateien für Website-Inhalte
-- Umwandlung der Ordnerstruktur des `content`-Verzeichnis in Navigationsbäume
-- Verarbeitung von Pongo2-Templates mit dem Inhalt zur finalen Website
-- Kopieren der Assets ins Zielverzeichnis
-- Anpassung des Asset-Pfads in den HTML-Dateien

VERSION

@@ -1 +0,0 @@
-1.0.0

build.sh

@@ -1,4 +0,0 @@
-#!/bin/sh
-
-mkdir -p dist
-go build -v -ldflags "-X main.Version=`git describe --tags --long` -X main.GitHash=`git rev-parse HEAD` -X main.BuildTime=`date -u '+%Y-%m-%d_%I:%M:%S%p'`" -o dist/mark2web-`cat VERSION`-${GOOS}-${GOARCH}${FILEEXT}

build/RELEASE.md

@@ -0,0 +1,9 @@
+NEUERUNGEN:
+
+- Cached Collection Webrequests
+- recursive Collections
+- Datei basierte Collections
+- markdown-Filter `s=SYNTAX_HIGHLIGHT_SHEMA` Parameter
+- image_process nutzt alle CPU-Kerne
+- GZIP/Brotli Vor-Komprimierung der Inhalte und Assets
+- Code neu organisiert

build/VERSION

@@ -0,0 +1 @@
+1.2.0-pre

build/ci/.drone.yml

@@ -6,22 +6,45 @@ workspace:
   path: src/gitbase.de/apairon/mark2web
 
 steps:
-  - name: build for linux
-    image: golang:latest
-    environment:
-      CGO_ENABLED: 0
-      GOOS: linux
-      GOARCH: amd64
+  - name: init submodules
+    image: docker:git
     commands:
       - git submodule update --init --recursive
       - git fetch --tags
-      - ./build.sh
+    when:
+      event: [ push, tag ]
+
+  - name: test
+    image: golang:latest
+    environment:
+      CGO_ENABLED: 1
+      GOOS: linux
+      GOARCH: amd64
+    commands:
+      # fake term for goconvey color output
+      - env TERM=xterm-color256 go test -v -coverprofile coverage.out ./pkg/*
+    when:
+      event: [ push, tag ]
+
+  - name: build for linux
+    image: golang:latest
+    environment:
+      CGO_ENABLED: 1
+      GOOS: linux
+      GOARCH: amd64
+    commands:
+      - scripts/build.sh
+    when:
+      event: [ push, tag ]
 
   - name: test with example content
     image: alpine
     commands:
-      - ./dist/mark2web-`cat VERSION`-linux-amd64 -version
-      - ./dist/mark2web-`cat VERSION`-linux-amd64 -in example -out example_out -create -logLevel debug
+      - apk add --no-cache libc6-compat
+      - dist/mark2web-`cat build/VERSION`-linux-amd64 -version
+      - dist/mark2web-`cat build/VERSION`-linux-amd64 -in example -out example_out -create -logLevel debug
+    when:
+      event: [ push, tag ]
 
   - name: build for freebsd
     image: golang:latest
@@ -30,7 +53,7 @@ steps:
       GOOS: freebsd
       GOARCH: amd64
     commands:
-      - ./build.sh
+      - scripts/build.sh
     when:
       event: [ tag ]
 
@@ -41,7 +64,7 @@ steps:
       GOOS: darwin
       GOARCH: amd64
     commands:
-      - ./build.sh
+      - scripts/build.sh
     when:
       event: [ tag ]
 
@@ -53,15 +76,15 @@ steps:
       GOARCH: amd64
       FILEEXT: .exe
     commands:
-      - ./build.sh
+      - scripts/build.sh
     when:
       event: [ tag ]
 
   - name: build docker image
     image: docker
     commands:
-      - cp dist/mark2web-`cat VERSION`-linux-amd64 mark2web
-      - docker build -t apairon/mark2web .
+      - cp dist/mark2web-`cat build/VERSION`-linux-amd64 build/package/mark2web
+      - docker build -t apairon/mark2web build/package
     volumes:
       - name: docker
         path: /var/run/docker.sock
@@ -69,15 +92,34 @@ steps:
       branch: [ master ]
       event: [ push ]
 
-  - name: deploy website
+  - name: build website
     image: apairon/mark2web:latest
     pull: never
     commands:
       - /mark2web -version
-      - /mark2web -in website -out html -create -logLevel debug
+      - /mark2web -in website -out html -create -logLevel info
     when:
       branch: [ master ]
-      event: [ push ]
+      event: [ promote, push ]
+      target: [ "", website ]
+
+  - name: deploy website
+    image: apairon/mark2web:latest
+    pull: never
+    environment:
+      RSYNC_PASS:
+        from_secret: rsync_pass    
+    commands:
+      - '
+        rsync -rlcgD -i -u -v --stats
+          --delete
+          -e "sshpass -p $${RSYNC_PASS} ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -p 22222"
+          html/
+          basiskonfiguration_mark2web_rsync@deploy.bc1.basehosts.de:./'
+    when:
+      branch: [ master ]
+      event: [ promote, push ]
+      target: [ "", website ]
 
   - name: prepare release
     image: apairon/mark2web:latest
@@ -96,8 +138,8 @@ steps:
       base_url: https://gitbase.de
       files:
         - dist/*
-      title: VERSION
-      note: RELEASE.md
+      title: build/VERSION
+      note: build/RELEASE.md
       checksum:
         - md5
         - sha256
@@ -107,4 +149,4 @@ steps:
 volumes:
   - name: docker
     host:
-      path: /var/run/docker.sock
+      path: /var/run/docker.sock

build/package/Dockerfile

@@ -0,0 +1,4 @@
+FROM alpine
+RUN apk update && apk add ca-certificates openssh-client sshpass rsync gzip libc6-compat && rm -r /var/cache/
+ADD mark2web /
+CMD ["/mark2web"]

cmd/mark2web/main.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path"
+
+	"gitbase.de/apairon/mark2web/pkg/filter"
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"gitbase.de/apairon/mark2web/pkg/mark2web"
+)
+
+var (
+	// Version is the app's version string
+	Version = "UNKNOWN"
+	// GitHash is the current git hash for this version
+	GitHash = "UNKNOWN"
+	// BuildTime is the time of build of this app
+	BuildTime = "UNKNOWN"
+)
+
+func main() {
+	inDir := flag.String("in", "./", "input directory")
+	outDir := flag.String("out", "html", "output directory")
+	createOutDir := flag.Bool("create", false, "create output directory if not existing")
+	//clearOutDir := flag.Bool("clear", false, "clear output directory before generating website")
+	logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error")
+	version := flag.Bool("version", false, "print version of this executable")
+
+	flag.Parse()
+	if version != nil && *version {
+		fmt.Printf(`%11s: %s
+%11s: %s
+%11s: %s
+`, "version", Version, "git hash", GitHash, "build time", BuildTime)
+		os.Exit(0)
+	}
+
+	level := "info"
+	if logLevel != nil {
+		level = *logLevel
+	}
+	helper.ConfigureLogger(level)
+
+	if inDir == nil || *inDir == "" {
+		helper.Log.Panic("input directory not specified")
+	}
+	iDir := path.Clean(*inDir)
+	inDir = &iDir
+	helper.Log.Infof("input directory: %s", *inDir)
+
+	if outDir == nil || *outDir == "" {
+		helper.Log.Panic("output directory not specified")
+	}
+	oDir := path.Clean(*outDir)
+	outDir = &oDir
+	helper.Log.Infof("output directory: %s", *outDir)
+
+	if createOutDir != nil && *createOutDir {
+		if _, err := os.Stat(*outDir); os.IsNotExist(err) {
+			helper.Log.Debugf("output directory '%s' does not exist", *outDir)
+			helper.Log.Debugf("trying to create output directory: %s", *outDir)
+			err := os.MkdirAll(*outDir, 0755)
+			if err != nil {
+				helper.Log.Panic(err)
+			}
+			helper.Log.Noticef("created output directory: %s", *outDir)
+		} else {
+			helper.Log.Noticef("output directory '%s' already exists", *outDir)
+		}
+	}
+
+	if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
+		helper.Log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
+	} else {
+		if fD == nil {
+			helper.Log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
+		} else if !fD.IsDir() {
+			helper.Log.Panicf("output directory '%s' is not a directory", *outDir)
+		}
+	}
+
+	helper.Log.Debug("reading global config...")
+	configFilename := *inDir + "/config.yml"
+	err := mark2web.Config.ReadFromFile(configFilename)
+	if err != nil {
+		helper.Log.Panicf("could not read file '%s': %s", configFilename, err)
+	}
+	mark2web.Config.Directories.Input = *inDir
+	mark2web.Config.Directories.Output = *outDir
+
+	helper.Log.Debugf("reading input directory %s", *inDir)
+
+	defaultTemplate := "base.html"
+	defaultInputFile := "README.md"
+	defaultOutputFile := "index.html"
+	defaultPathStrip := "^[0-9]*_(.*)"
+	defaultPathIgnoreForNav := "^_"
+	defaultFilenameStrip := "(.*).md$"
+	defaultFilenameIgnore := "^_"
+	defaultFilenameOutputExtension := "html"
+
+	defaultPathConfig := new(mark2web.PathConfig)
+	defaultPathConfig.Template = &defaultTemplate
+	defaultPathConfig.Index = &mark2web.IndexConfig{
+		InputFile:  &defaultInputFile,
+		OutputFile: &defaultOutputFile,
+	}
+	defaultPathConfig.Path = &mark2web.DirnameConfig{
+		Strip:        &defaultPathStrip,
+		IgnoreForNav: &defaultPathIgnoreForNav,
+	}
+	defaultPathConfig.Filename = &mark2web.FilenameConfig{
+		Strip:           &defaultFilenameStrip,
+		Ignore:          &defaultFilenameIgnore,
+		OutputExtension: &defaultFilenameOutputExtension,
+	}
+	defaultPathConfig.Imaging = &mark2web.ImagingConfig{
+		Width:   1920,
+		Height:  1920,
+		Process: "fit",
+		Quality: 75,
+	}
+
+	filtersDir := *inDir + "/templates/filters"
+	if _, err := os.Stat(filtersDir); !os.IsNotExist(err) {
+		filter.RegisterFilters(filtersDir)
+	}
+
+	mark2web.Run(*inDir, *outDir, defaultPathConfig)
+
+}

example/content/de/config.yml

@@ -1,2 +1,2 @@
 This:
-    GoTo: main/home
+    GoTo: main/home/

example/content/en/config.yml

@@ -1,2 +1,2 @@
 This:
-    GoTo: main/home
+    GoTo: main/home/

main.go

@@ -1,745 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"path"
-	"reflect"
-	"regexp"
-	"strings"
-
-	"github.com/imdario/mergo"
-
-	"github.com/Depado/bfchroma"
-	"github.com/davecgh/go-spew/spew"
-	"github.com/flosch/pongo2"
-	"github.com/gosimple/slug"
-	"github.com/op/go-logging"
-	cpy "github.com/otiai10/copy"
-	"gopkg.in/russross/blackfriday.v2"
-	"gopkg.in/yaml.v2"
-)
-
-var (
-	// Version is the app's version string
-	Version = "UNKNOWN"
-	// GitHash is the current git hash for this version
-	GitHash = "UNKNOWN"
-	// BuildTime is the time of build of this app
-	BuildTime = "UNKNOWN"
-)
-
-var log = logging.MustGetLogger("myLogger")
-
-var inDir *string
-var outDir *string
-
-var templateCache = make(map[string]*pongo2.Template)
-
-// GlobalConfig is config which is used only once in root dir
-type GlobalConfig struct {
-	Webserver struct {
-		Type string `yaml:"Type"`
-	} `yaml:"Webserver"`
-
-	Assets struct {
-		FromPath    string `yaml:"FromPath"`
-		ToPath      string `yaml:"ToPath"`
-		Action      string `yaml:"Action"`
-		FixTemplate struct {
-			Find    string `yaml:"Find"`
-			Replace string `yaml:"Replace"`
-		} `yaml:"FixTemplate"`
-	} `yaml:"Assets"`
-
-	OtherFiles struct {
-		Action string `yaml:"Action"`
-	} `yaml:"OtherFiles"`
-}
-
-var globalConfig = new(GlobalConfig)
-
-// ThisPathConfig is struct for This in paths yaml
-type ThisPathConfig struct {
-	Navname *string     `yaml:"Navname"`
-	GoTo    *string     `yaml:"GoTo"`
-	Data    interface{} `yaml:"Data"`
-}
-
-type indexStruct struct {
-	InputFile  *string `yaml:"InputFile"`
-	OutputFile *string `yaml:"OutputFile"`
-}
-
-type metaStruct struct {
-	Title       *string `yaml:"Title"`
-	Description *string `yaml:"Description"`
-	Keywords    *string `yaml:"Keywords"`
-}
-
-type pathStruct struct {
-	Strip        *string `yaml:"Strip"`
-	IgnoreForNav *string `yaml:"IgnoreForNav"`
-}
-
-type filenameStruct struct {
-	Strip           *string `yaml:"Strip"`
-	Ignore          *string `yaml:"Ignore"`
-	OutputExtension *string `yaml:"OutputExtension"`
-}
-
-type markdownStruct struct {
-	ChromaRenderer *bool   `yaml:"ChromaRenderer"`
-	ChromaStyle    *string `yaml:"ChromaStyle"`
-}
-
-// PathConfig of subdir
-type PathConfig struct {
-	This     ThisPathConfig  `yaml:"This"`
-	Template *string         `yaml:"Template"`
-	Index    *indexStruct    `yaml:"Index"`
-	Meta     *metaStruct     `yaml:"Meta"`
-	Path     *pathStruct     `yaml:"Path"`
-	Filename *filenameStruct `yaml:"Filename"`
-	Markdown *markdownStruct `yaml:"Markdown"`
-
-	Data interface{} `yaml:"Data"`
-}
-
-// PathConfigTree is complete config tree of content dir
-type PathConfigTree struct {
-	InputPath  string
-	OutputPath string
-
-	InputFiles []string
-	OtherFiles []string
-
-	Config *PathConfig
-	Sub    []*PathConfigTree
-}
-
-var contentConfig = new(PathConfigTree)
-
-type ptrTransformer struct{}
-
-func (t ptrTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
-	if typ.Kind() == reflect.Ptr {
-		return func(dst, src reflect.Value) error {
-			if dst.CanSet() {
-				if dst.IsNil() {
-					dst.Set(src)
-				}
-			}
-			return nil
-		}
-	}
-	return nil
-}
-
-func merge(dst, src interface{}) error {
-	return mergo.Merge(dst, src, mergo.WithTransformers(ptrTransformer{}))
-}
-
-func backToRoot(curNavPath string) string {
-	tmpPath := ""
-	if curNavPath != "" {
-		for i := strings.Count(curNavPath, "/") + 1; i > 0; i-- {
-			tmpPath += "../"
-		}
-	}
-	return tmpPath
-}
-
-func readContentDir(inBase string, outBase string, dir string, conf *PathConfig, tree *PathConfigTree) {
-	inPath := inBase
-	if dir != "" {
-		inPath += "/" + dir
-	}
-
-	log.Infof("reading input directory: %s", inPath)
-
-	files, err := ioutil.ReadDir(inPath)
-	if err != nil {
-		log.Panic(err)
-	}
-
-	tree.InputPath = inPath
-
-	// read config
-	newConfig := new(PathConfig)
-	log.Debug("looking for config.yml ...")
-	configFile := inPath + "/config.yml"
-	if _, err = os.Stat(configFile); os.IsNotExist(err) {
-		log.Debug("no config.yml found in this directory, using upper configs")
-		merge(newConfig, conf)
-		// remove this
-		newConfig.This = ThisPathConfig{}
-	} else {
-		log.Debug("reading config...")
-		data, err := ioutil.ReadFile(configFile)
-		if err != nil {
-			log.Panicf("could not read file '%s': %s", configFile, err)
-		}
-		err = yaml.Unmarshal(data, newConfig)
-		if err != nil {
-			log.Panicf("could not parse YAML file '%s': %s", configFile, err)
-		}
-
-		log.Debug("merging config with upper config")
-		oldThis := newConfig.This
-		merge(newConfig, conf)
-		newConfig.This = oldThis
-
-		log.Debug(spew.Sdump(newConfig))
-	}
-
-	tree.Config = newConfig
-
-	// calc outDir
-	stripedDir := dir
-	var regexStr *string
-	if newConfig.Path != nil {
-		regexStr = newConfig.Path.Strip
-	}
-	if regexStr != nil && *regexStr != "" {
-		if regex, err := regexp.Compile(*regexStr); err != nil {
-			log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err)
-		} else {
-			stripedDir = regex.ReplaceAllString(stripedDir, "$1")
-		}
-	}
-
-	if tree.Config.This.Navname == nil {
-		navname := strings.Replace(stripedDir, "_", " ", -1)
-		tree.Config.This.Navname = &navname
-	}
-
-	stripedDir = slug.Make(stripedDir)
-	outPath := outBase + "/" + stripedDir
-	outPath = path.Clean(outPath)
-
-	log.Infof("calculated output directory: %s", outPath)
-	tree.OutputPath = outPath
-
-	// first only files
-	for _, f := range files {
-		p := inPath + "/" + f.Name()
-		if !f.IsDir() && f.Name() != "config.yml" {
-			switch path.Ext(f.Name()) {
-			case ".md":
-				log.Debugf(".MD %s", p)
-				if tree.InputFiles == nil {
-					tree.InputFiles = make([]string, 0)
-				}
-				tree.InputFiles = append(tree.InputFiles, f.Name())
-				break
-			default:
-				log.Debugf("FIL %s", p)
-				if tree.OtherFiles == nil {
-					tree.OtherFiles = make([]string, 0)
-				}
-				tree.OtherFiles = append(tree.OtherFiles, f.Name())
-			}
-		}
-	}
-
-	// only directorys, needed config before
-	for _, f := range files {
-		p := inPath + "/" + f.Name()
-		if f.IsDir() {
-			log.Debugf("DIR %s", p)
-			newTree := new(PathConfigTree)
-			if tree.Sub == nil {
-				tree.Sub = make([]*PathConfigTree, 0)
-			}
-			tree.Sub = append(tree.Sub, newTree)
-			readContentDir(inPath, outPath, f.Name(), newConfig, newTree)
-		}
-	}
-}
-
-type navElement struct {
-	Navname string
-	GoTo    string
-	Active  bool
-
-	Data interface{}
-
-	This ThisPathConfig
-
-	SubMap   *map[string]*navElement
-	SubSlice *[]*navElement
-}
-
-func buildNavigation(conf *PathConfigTree, curNavMap *map[string]*navElement, curNavSlice *[]*navElement, navActive *[]*navElement, activeNav string) {
-	for _, el := range conf.Sub {
-		var ignNav *string
-		if p := el.Config.Path; p != nil {
-			ignNav = p.IgnoreForNav
-		}
-		if ignNav != nil && *ignNav != "" {
-			regex, err := regexp.Compile(*ignNav)
-			if err != nil {
-				log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err)
-			}
-			if regex.MatchString(path.Base(el.InputPath)) {
-				log.Debugf("ignoring input directory '%s' in navigation", el.InputPath)
-				continue
-			}
-		}
-
-		elPath := strings.TrimPrefix(el.OutputPath, *outDir+"/")
-
-		subMap := make(map[string]*navElement)
-		subSlice := make([]*navElement, 0)
-		navEl := navElement{
-			Active:   strings.HasPrefix(activeNav, elPath),
-			Data:     el.Config.Data,
-			SubMap:   &subMap,
-			SubSlice: &subSlice,
-		}
-
-		navEl.This = el.Config.This
-
-		if navEl.Active {
-			// add to navActive level navigation
-			currentLevel := strings.Count(activeNav, "/")
-			if len(*navActive) <= currentLevel {
-				// not registered
-				*navActive = append(*navActive, &navEl)
-			}
-		}
-
-		n := el.Config.This.Navname
-		if n != nil {
-			navEl.Navname = *n
-		}
-		g := el.Config.This.GoTo
-		if g != nil {
-			if strings.HasPrefix(*g, "/") {
-				// abslute
-				navEl.GoTo = *g
-			} else {
-				// relative
-				navEl.GoTo = elPath + "/" + *g
-			}
-		} else {
-			navEl.GoTo = elPath + "/"
-		}
-
-		if activeNav != "" && activeNav != "/" {
-			// calculate relative path
-			bToRoot := backToRoot(activeNav)
-			navEl.GoTo = bToRoot + navEl.GoTo
-			navEl.GoTo = path.Clean(navEl.GoTo)
-		}
-
-		(*curNavMap)[navEl.Navname] = &navEl
-		if curNavSlice != nil {
-			*curNavSlice = append(*curNavSlice, &navEl)
-		}
-
-		buildNavigation(el, &subMap, &subSlice, navActive, activeNav)
-	}
-}
-
-func processContent(conf *PathConfigTree) {
-	log.Debugf("trying to create output directory: %s", conf.OutputPath)
-
-	if dirH, err := os.Stat(conf.OutputPath); os.IsNotExist(err) {
-		err := os.MkdirAll(conf.OutputPath, 0755)
-		if err != nil {
-			log.Panicf("could not create output directory '%s': %s", conf.OutputPath, err)
-		}
-		log.Noticef("created output directory: %s", conf.OutputPath)
-	} else if dirH != nil {
-		if dirH.IsDir() {
-			log.Noticef("output directory '%s' already exists", conf.OutputPath)
-		} else {
-			log.Panicf("output directory '%s' is no directory", conf.OutputPath)
-		}
-	} else {
-		log.Panicf("unknown error for output directory '%s': %s", conf.OutputPath, err)
-	}
-
-	curNavPath := strings.TrimPrefix(conf.OutputPath, *outDir)
-	curNavPath = strings.TrimPrefix(curNavPath, "/")
-	curNavPath = path.Clean(curNavPath)
-	if curNavPath == "." {
-		curNavPath = ""
-	}
-
-	goTo := conf.Config.This.GoTo
-	if goTo != nil && *goTo != "" {
-		goToFixed := *goTo
-		if strings.HasPrefix(goToFixed, "/") {
-			goToFixed = backToRoot(curNavPath) + goToFixed
-		}
-		goToFixed = path.Clean(goToFixed)
-
-		switch globalConfig.Webserver.Type {
-		case "apache":
-			htaccessFile := conf.OutputPath + "/.htaccess"
-			log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed)
-			err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
-RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
-`), 0644)
-			if err != nil {
-				log.Panicf("could not write '%s': %s", htaccessFile, err)
-			}
-			break
-		}
-	}
-
-	for _, file := range conf.InputFiles {
-		inFile := conf.InputPath + "/" + file
-		log.Debugf("reading file: %s", inFile)
-
-		input, err := ioutil.ReadFile(inFile)
-		if err != nil {
-			log.Panicf("could not read '%s':%s", inFile, err)
-		}
-		log.Infof("processing input file '%s'", inFile)
-
-		newConfig := new(PathConfig)
-
-		regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?")
-		yamlData := regex.Find(input)
-		if string(yamlData) != "" {
-			log.Debugf("found yaml header in '%s', merging config", inFile)
-			err = yaml.Unmarshal(yamlData, newConfig)
-			if err != nil {
-				log.Panicf("could not parse YAML header from '%s': %s", inFile, err)
-			}
-
-			log.Debug("merging config with upper config")
-			oldThis := newConfig.This
-			merge(newConfig, conf.Config)
-			newConfig.This = oldThis
-
-			log.Debug(spew.Sdump(newConfig))
-
-			input = regex.ReplaceAll(input, []byte(""))
-		} else {
-			merge(newConfig, conf.Config)
-		}
-
-		// ignore ???
-		ignoreFile := false
-		var ignoreRegex *string
-		var stripRegex *string
-		var outputExt *string
-		if f := newConfig.Filename; f != nil {
-			ignoreRegex = f.Ignore
-			stripRegex = f.Strip
-			outputExt = f.OutputExtension
-		}
-		if ignoreRegex != nil && *ignoreRegex != "" {
-			regex, err := regexp.Compile(*ignoreRegex)
-			if err != nil {
-				log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
-			}
-			ignoreFile = regex.MatchString(file)
-		}
-
-		if ignoreFile {
-			log.Infof("ignoring file '%s', because of filename.ignore", inFile)
-		} else {
-
-			// build output filename
-			outputFilename := file
-
-			var indexInputFile *string
-			var indexOutputFile *string
-			if i := newConfig.Index; i != nil {
-				indexInputFile = i.InputFile
-				indexOutputFile = i.OutputFile
-			}
-
-			if indexInputFile != nil && *indexInputFile == file && indexOutputFile != nil && *indexOutputFile != "" {
-				outputFilename = *indexOutputFile
-			} else {
-				if stripRegex != nil && *stripRegex != "" {
-					regex, err := regexp.Compile(*stripRegex)
-					if err != nil {
-						log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
-					}
-					outputFilename = regex.ReplaceAllString(outputFilename, "$1")
-				}
-				if outputExt != nil && *outputExt != "" {
-					outputFilename += "." + *outputExt
-				}
-			}
-
-			outFile := conf.OutputPath + "/" + outputFilename
-			log.Debugf("using '%s' as output file", outFile)
-
-			var options []blackfriday.Option
-
-			var chromaRenderer *bool
-			var chromaStyle *string
-			if m := newConfig.Markdown; m != nil {
-				chromaRenderer = m.ChromaRenderer
-				chromaStyle = m.ChromaStyle
-			}
-			if chromaStyle == nil {
-				style := "monokai"
-				chromaStyle = &style
-			}
-			if chromaRenderer != nil && *chromaRenderer {
-				options = []blackfriday.Option{
-					blackfriday.WithRenderer(
-						bfchroma.NewRenderer(
-							bfchroma.Style(*chromaStyle),
-						),
-					),
-				}
-			}
-
-			html := blackfriday.Run(input, options...)
-
-			// use --- for splitting document in markdown parts
-			regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?")
-			inputParts := regex.Split(string(input), -1)
-			htmlParts := make([]*pongo2.Value, 0)
-			for _, iPart := range inputParts {
-				htmlParts = append(htmlParts, pongo2.AsSafeValue(string(blackfriday.Run([]byte(iPart), options...))))
-			}
-
-			log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
-			templateFile := *inDir + "/templates/" + *newConfig.Template
-			template := templateCache[templateFile]
-			if template == nil {
-				var err error
-				if template, err = pongo2.FromFile(templateFile); err != nil {
-					log.Panicf("could not parse template '%s': %s", templateFile, err)
-				} else {
-					templateCache[templateFile] = template
-				}
-			}
-
-			// build navigation
-			navMap := make(map[string]*navElement)
-			navSlice := make([]*navElement, 0)
-			navActive := make([]*navElement, 0)
-			buildNavigation(contentConfig, &navMap, &navSlice, &navActive, curNavPath)
-
-			// read yaml header as data for template
-			ctx := make(pongo2.Context)
-			ctx["This"] = newConfig.This
-			ctx["Meta"] = newConfig.Meta
-			ctx["Data"] = newConfig.Data
-			ctx["NavMap"] = navMap
-			ctx["NavSlice"] = navSlice
-			ctx["NavActive"] = navActive
-			ctx["Body"] = pongo2.AsSafeValue(string(html))
-			ctx["BodyParts"] = htmlParts
-
-			result, err := template.Execute(ctx)
-			if err != nil {
-				log.Panicf("could not execute template '%s' for input file '%s': %s", templateFile, inFile, err)
-			}
-
-			if find := globalConfig.Assets.FixTemplate.Find; find != "" {
-				log.Debugf("fixing assets paths in '%s' for '%s'", templateFile, inFile)
-				bToRoot := backToRoot(curNavPath)
-				regex, err := regexp.Compile(find)
-				if err != nil {
-					log.Panicf("could not compile regexp '%s' for assets path: %s", find, err)
-				}
-				repl := globalConfig.Assets.FixTemplate.Replace
-				repl = bToRoot + globalConfig.Assets.ToPath + "/" + repl
-				repl = path.Clean(repl) + "/"
-				log.Debugf("new assets paths: %s", repl)
-				result = regex.ReplaceAllString(result, repl)
-			}
-
-			log.Noticef("writing to output file: %s", outFile)
-			err = ioutil.WriteFile(outFile, []byte(result), 0644)
-			if err != nil {
-				log.Panicf("could not write to output file '%s': %s", outFile, err)
-			}
-
-			//fmt.Println(string(html))
-		}
-	}
-
-	// process other files, copy...
-	for _, file := range conf.OtherFiles {
-		switch globalConfig.OtherFiles.Action {
-		case "copy":
-			from := conf.InputPath + "/" + file
-			to := conf.OutputPath + "/" + file
-			log.Noticef("copying file from '%s' to '%s'", from, to)
-			err := cpy.Copy(from, to)
-			if err != nil {
-				log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
-			}
-		}
-	}
-
-	for _, el := range conf.Sub {
-		processContent(el)
-	}
-}
-
-func processAssets() {
-	switch globalConfig.Assets.Action {
-	case "copy":
-		from := globalConfig.Assets.FromPath
-		to := globalConfig.Assets.ToPath
-		if !strings.HasPrefix(from, "/") {
-			from = *inDir + "/" + from
-		}
-		if !strings.HasPrefix(to, "/") {
-			to = *outDir + "/" + to
-		}
-		log.Noticef("copying assets from '%s' to '%s'", from, to)
-		err := cpy.Copy(from, to)
-		if err != nil {
-			log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
-		}
-	}
-}
-
-func main() {
-	spew.Config.DisablePointerAddresses = true
-	spew.Config.DisableCapacities = true
-	spew.Config.DisableMethods = true
-	spew.Config.DisablePointerMethods = true
-
-	inDir = flag.String("in", "./", "input directory")
-	outDir = flag.String("out", "html", "output directory")
-	createOutDir := flag.Bool("create", false, "create output directory if not existing")
-	//clearOutDir := flag.Bool("clear", false, "clear output directory before generating website")
-	logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error")
-	version := flag.Bool("version", false, "print version of this executable")
-
-	flag.Parse()
-	if version != nil && *version {
-		fmt.Printf(`%11s: %s
-%11s: %s
-%11s: %s
-`, "version", Version, "git hash", GitHash, "build time", BuildTime)
-		os.Exit(0)
-	}
-
-	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
-	logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
-		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
-	))
-	logBackendLeveled := logging.AddModuleLevel(logBackendFormatter)
-	logBackendLevel := logging.INFO
-	if logLevel != nil {
-		switch *logLevel {
-		case "debug":
-			logBackendLevel = logging.DEBUG
-			break
-
-		case "info":
-			logBackendLevel = logging.INFO
-			break
-
-		case "notice":
-			logBackendLevel = logging.NOTICE
-			break
-
-		case "warning":
-			logBackendLevel = logging.WARNING
-			break
-
-		case "error":
-			logBackendLevel = logging.ERROR
-			break
-
-		}
-	}
-	logBackendLeveled.SetLevel(logBackendLevel, "")
-	logging.SetBackend(logBackendLeveled)
-
-	if inDir == nil || *inDir == "" {
-		log.Panic("input directory not specified")
-	}
-	log.Infof("input directory: %s", *inDir)
-
-	if outDir == nil || *outDir == "" {
-		log.Panic("output directory not specified")
-	}
-	log.Infof("output directory: %s", *outDir)
-
-	if createOutDir != nil && *createOutDir {
-		if _, err := os.Stat(*outDir); os.IsNotExist(err) {
-			log.Debugf("output directory '%s' does not exist", *outDir)
-			log.Debugf("trying to create output directory: %s", *outDir)
-			err := os.MkdirAll(*outDir, 0755)
-			if err != nil {
-				log.Panic(err)
-			}
-			log.Noticef("created output directory: %s", *outDir)
-		} else {
-			log.Noticef("output directory '%s' already exists", *outDir)
-		}
-	}
-
-	if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
-		log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
-	} else {
-		if fD == nil {
-			log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
-		} else if !fD.IsDir() {
-			log.Panicf("output directory '%s' is not a directory", *outDir)
-		}
-	}
-
-	log.Debug("reading global config...")
-	p := *inDir + "/config.yml"
-	data, err := ioutil.ReadFile(p)
-	if err != nil {
-		log.Panicf("could not read file '%s': %s", p, err)
-	}
-	err = yaml.Unmarshal(data, globalConfig)
-	if err != nil {
-		log.Panicf("could not parse YAML file '%s': %s", p, err)
-	}
-
-	log.Debug(spew.Sdump(globalConfig))
-
-	log.Debugf("reading input directory %s", *inDir)
-
-	defaultTemplate := "base.html"
-	defaultInputFile := "README.md"
-	defaultOutputFile := "index.html"
-	defaultPathStrip := "^[0-9]*_(.*)"
-	defaultPathIgnoreForNav := "^_"
-	defaultFilenameStrip := "(.*).md$"
-	defaultFilenameIgnore := "^_"
-	defaultFilenameOutputExtension := "html"
-
-	defaultPathConfig := new(PathConfig)
-	defaultPathConfig.Template = &defaultTemplate
-	defaultPathConfig.Index = &indexStruct{
-		InputFile:  &defaultInputFile,
-		OutputFile: &defaultOutputFile,
-	}
-	defaultPathConfig.Path = &pathStruct{
-		Strip:        &defaultPathStrip,
-		IgnoreForNav: &defaultPathIgnoreForNav,
-	}
-	defaultPathConfig.Filename = &filenameStruct{
-		Strip:           &defaultFilenameStrip,
-		Ignore:          &defaultFilenameIgnore,
-		OutputExtension: &defaultFilenameOutputExtension,
-	}
-
-	readContentDir(*inDir+"/content", *outDir, "", defaultPathConfig, contentConfig)
-	//spew.Dump(contentConfig)
-
-	//spew.Dump(navMap)
-
-	processContent(contentConfig)
-
-	processAssets()
-
-}

pkg/filter/custom.go

@@ -0,0 +1,73 @@
+package filter
+
+import (
+	"io/ioutil"
+	"path"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"gitbase.de/apairon/mark2web/pkg/mark2web"
+	"github.com/ddliu/motto"
+	"github.com/flosch/pongo2"
+	_ "github.com/robertkrimen/otto/underscore"
+)
+
+// RegisterFilters reads a directory and register filters from files within it
+func RegisterFilters(dir string) {
+	files, err := ioutil.ReadDir(dir)
+	if err != nil {
+		helper.Log.Panicf("could not read from template filters dir '%s': %s", dir, err)
+	}
+	for _, f := range files {
+		if !f.IsDir() {
+			switch path.Ext(f.Name()) {
+			case ".js":
+				fileBase := strings.TrimSuffix(f.Name(), ".js")
+				jsFile := dir + "/" + f.Name()
+				helper.Log.Debugf("trying to register filter from: %s", jsFile)
+				/*
+					jsStr, err := ioutil.ReadFile(jsFile)
+					if err != nil {
+						Log.Panicf("could not read '%s': %s", jsFile, err)
+					}
+				*/
+				vm := motto.New()
+				fn, err := vm.Run(jsFile)
+				if err != nil {
+					helper.Log.Panicf("error in javascript vm for '%s': %s", jsFile, err)
+				}
+				if !fn.IsFunction() {
+					helper.Log.Panicf("%s does not contain a function code", jsFile)
+				}
+
+				err = pongo2.RegisterFilter(
+					fileBase,
+					func(in, param *pongo2.Value) (out *pongo2.Value, erro *pongo2.Error) {
+						thisObj, _ := vm.Object("({})")
+						if mark2web.CurrentContext != nil {
+							thisObj.Set("context", *mark2web.CurrentContext)
+						}
+
+						if err != nil {
+							helper.Log.Panicf("could not set context as in '%s': %s", jsFile, err)
+						}
+						ret, err := fn.Call(thisObj.Value(), in.Interface(), param.Interface())
+						if err != nil {
+							helper.Log.Panicf("error in javascript file '%s' while calling returned function: %s", jsFile, err)
+						}
+						retGo, err := ret.Export()
+						if err != nil {
+							helper.Log.Panicf("export error for '%s': %s", jsFile, err)
+						}
+						return pongo2.AsValue(retGo), nil
+					},
+				)
+				if err != nil {
+					helper.Log.Panicf("could not register filter from '%s': %s", jsFile, err)
+				}
+
+			}
+		}
+	}
+
+}

pkg/filter/dump.go

@@ -0,0 +1,12 @@
+package filter
+
+import (
+	"github.com/davecgh/go-spew/spew"
+	"github.com/flosch/pongo2"
+)
+
+// DumpFilter is a pongo2 filter, which returns a spew.Dump of the input
+func DumpFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
+	dumpString := spew.Sdump(in.Interface())
+	return pongo2.AsValue(string(dumpString)), nil
+}

pkg/filter/dump_test.go

@@ -0,0 +1,24 @@
+package filter
+
+import (
+	"testing"
+
+	"github.com/davecgh/go-spew/spew"
+
+	"github.com/flosch/pongo2"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestDumpFilter(t *testing.T) {
+	Convey("set context", t, func() {
+		ctx := pongo2.Context{
+			"testvar": "test",
+		}
+		Convey("parse template", func() {
+			output, err := pongo2.RenderTemplateString("{{ testvar|safe|dump }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, spew.Sdump("test"))
+		})
+	})
+}

pkg/filter/image_process.go

@@ -0,0 +1,230 @@
+package filter
+
+import (
+	"crypto/md5"
+	"errors"
+	"fmt"
+	"image"
+	"net/http"
+	"net/url"
+	"os"
+	"path"
+	"strconv"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"gitbase.de/apairon/mark2web/pkg/mark2web"
+	"github.com/disintegration/imaging"
+	"github.com/flosch/pongo2"
+)
+
+func parseImageParams(str string) (*mark2web.ImagingConfig, error) {
+	p := mark2web.ImagingConfig{}
+	if str == "" {
+		helper.Merge(&p, mark2web.CurrentTreeNode.Config.Imaging)
+		// Filename and Format are only valid for current image
+		p.Filename = ""
+		p.Format = ""
+		return &p, nil
+	}
+	for _, s := range strings.Split(str, ",") {
+		e := strings.Split(s, "=")
+		if len(e) < 2 {
+			return nil, fmt.Errorf("invalid image parameter: %s", s)
+		}
+		var err error
+		switch e[0] {
+		case "w":
+			p.Width, err = strconv.Atoi(e[1])
+		case "h":
+			p.Height, err = strconv.Atoi(e[1])
+		case "f":
+			p.Filename = e[1]
+		case "t":
+			p.TargetDir = e[1]
+		case "p":
+			p.Process = e[1]
+		case "a":
+			p.Anchor = e[1]
+		case "q":
+			p.Quality, err = strconv.Atoi(e[1])
+			if p.Quality < 0 || p.Quality > 100 {
+				err = errors.New("q= must be between 1 and 100")
+			}
+		default:
+			return nil, fmt.Errorf("invalid image parameter: %s", s)
+		}
+		if err != nil {
+			return nil, fmt.Errorf("could not convert image parameter to correct value type for '%s': %s", s, err)
+		}
+	}
+	return &p, nil
+}
+
+func getImageFromURL(url string) (image.Image, string, error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
+	}
+
+	img, format, err := image.Decode(resp.Body)
+	if err != nil {
+		return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
+	}
+
+	return img, format, nil
+}
+
+// ImageProcessFilter read the image url and process parameters and saves the resized/processed image
+// param: w=WITDH,h=HEIGHT
+func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
+	imgSource := in.String()
+	p, err := parseImageParams(param.String())
+	if err != nil {
+		return nil, &pongo2.Error{
+			Sender:    "filter:image_resize",
+			OrigError: err,
+		}
+	}
+	if p == nil {
+		return nil, &pongo2.Error{
+			Sender:    "filter:image_resize",
+			OrigError: errors.New("no imaging config defined"),
+		}
+	}
+
+	var img image.Image
+	if p.Process == "" {
+		p.Process = "resize"
+	}
+	filePrefix := fmt.Sprintf(
+		"%s_%dx%d_q%03d",
+		p.Process,
+		p.Width,
+		p.Height,
+		p.Quality,
+	)
+	if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
+		// remote file
+		img, p.Format, err = getImageFromURL(imgSource)
+		if err != nil {
+			return nil, &pongo2.Error{
+				Sender:    "filter:image_resize",
+				OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
+			}
+		}
+		// build filename
+		if p.Filename == "" {
+			var fBase string
+			if u, _ := url.Parse(imgSource); u != nil {
+				fBase = strings.Split(path.Base(u.Path), ".")[0]
+			}
+
+			p.Filename = fmt.Sprintf(
+				"%s_%x_%s.%s",
+				filePrefix,
+				md5.Sum([]byte(imgSource)),
+				fBase,
+				p.Format,
+			)
+		}
+	} else {
+		// local file
+		imgSource = mark2web.CurrentTreeNode.ResolveInputPath(imgSource)
+		if p.Filename == "" {
+			p.Filename = fmt.Sprintf(
+				"%s_%s",
+				filePrefix,
+				path.Base(imgSource),
+			)
+		}
+	}
+
+	var imgTarget string
+	if p.TargetDir != "" {
+		imgTarget = mark2web.CurrentTreeNode.ResolveOutputPath(
+			path.Clean(p.TargetDir) + "/" +
+				p.Filename,
+		)
+
+		pt := path.Dir(imgTarget)
+		if _, err := os.Stat(pt); os.IsNotExist(err) {
+			helper.Log.Infof("create image target dir: %s", pt)
+			if err := os.MkdirAll(pt, 0755); err != nil {
+				return nil, &pongo2.Error{
+					Sender:    "filter:image_resize",
+					OrigError: fmt.Errorf("could not create image target dir '%s': %s", pt, err),
+				}
+			}
+		}
+
+		p.Filename = mark2web.CurrentTreeNode.ResolveNavPath(p.TargetDir + "/" + p.Filename)
+
+	} else {
+		imgTarget = mark2web.CurrentTreeNode.ResolveOutputPath(p.Filename)
+	}
+
+	if f, err := os.Stat(imgTarget); err == nil && !f.IsDir() {
+		helper.Log.Noticef("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
+	} else {
+		mark2web.ThreadStart(func() {
+			helper.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
+			if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
+				// webrequest before finding target filename, because of file format in filename
+			} else {
+				img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
+				if err != nil {
+					helper.Log.Panicf("filter:image_resize, could not open image '%s': %s", imgSource, err)
+				}
+			}
+
+			switch p.Process {
+			case "resize":
+				img = imaging.Resize(img, p.Width, p.Height, imaging.Lanczos)
+			case "fit":
+				img = imaging.Fit(img, p.Width, p.Height, imaging.Lanczos)
+			case "fill":
+				var anchor imaging.Anchor
+				switch strings.ToLower(p.Anchor) {
+				case "":
+					fallthrough
+				case "center":
+					anchor = imaging.Center
+				case "topleft":
+					anchor = imaging.TopLeft
+				case "top":
+					anchor = imaging.Top
+				case "topright":
+					anchor = imaging.TopRight
+				case "left":
+					anchor = imaging.Left
+				case "right":
+					anchor = imaging.Right
+				case "bottomleft":
+					anchor = imaging.BottomLeft
+				case "bottom":
+					anchor = imaging.Bottom
+				case "bottomright":
+					anchor = imaging.BottomRight
+				default:
+					helper.Log.Panicf("filter:image_resize, unknown anchor a=%s definition", p.Anchor)
+				}
+				img = imaging.Fill(img, p.Width, p.Height, anchor, imaging.Lanczos)
+			default:
+				helper.Log.Panicf("filter:image_resize, invalid p parameter '%s'", p.Process)
+			}
+
+			var encodeOptions = make([]imaging.EncodeOption, 0)
+			if p.Quality > 0 {
+				encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
+			}
+
+			err = imaging.Save(img, imgTarget, encodeOptions...)
+			if err != nil {
+				helper.Log.Panicf("filter:image_resize, could save image '%s': %s", imgTarget, err)
+			}
+			helper.Log.Noticef("finished image: %s", imgTarget)
+		})
+	}
+	return pongo2.AsValue(mark2web.CurrentTreeNode.ResolveNavPath(p.Filename)), nil
+}

pkg/filter/image_process_test.go

@@ -0,0 +1,54 @@
+package filter
+
+import (
+	"os"
+	"testing"
+
+	"gitbase.de/apairon/mark2web/pkg/mark2web"
+
+	"github.com/flosch/pongo2"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestImageProcessFilter(t *testing.T) {
+	Convey("set context", t, func() {
+		ctx := pongo2.Context{
+			"testlocal": "/img/test.jpg",
+			"testurl":   "http://url",
+		}
+
+		// we want to check files after function calls, so no multithreading
+		mark2web.SetNumCPU(1)
+
+		mark2web.Config.Directories.Input = "../../test/in"
+		mark2web.Config.Directories.Output = "../../test/out"
+
+		mark2web.CurrentTreeNode = &mark2web.TreeNode{
+			InputPath:  "../../test/in/content",
+			OutputPath: "../../test/out",
+			Config: &mark2web.PathConfig{
+				Imaging: &mark2web.ImagingConfig{
+					Quality: 60,
+					Height:  300,
+					Width:   300,
+					Process: "fit",
+				},
+			},
+		}
+
+		os.Remove("../../test/out/fit_300x300_q060_test.jpg")
+
+		Convey("local image with defaults", func() {
+			output, err := pongo2.RenderTemplateString("{{ testlocal|image_process }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, "fit_300x300_q060_test.jpg")
+
+			Convey("local image with fit", func() {
+				output, err := pongo2.RenderTemplateString(`{{ testlocal|image_process:"p=fit,w=300,h=300,q=60,t=/" }}`, ctx)
+				So(err, ShouldBeNil)
+				So(output, ShouldEqual, "/fit_300x300_q060_test.jpg")
+			})
+		})
+	})
+}

pkg/filter/init.go

@@ -0,0 +1,26 @@
+package filter
+
+import (
+	"github.com/flosch/pongo2"
+	_ "github.com/flosch/pongo2-addons"
+)
+
+func init() {
+	err := pongo2.ReplaceFilter("markdown", MarkdownFilter)
+	if err != nil {
+		panic(err)
+	}
+
+	newFilters := map[string]pongo2.FilterFunction{
+		"image_process": ImageProcessFilter,
+		"relative_path": RelativePathFilter,
+		"json":          JSONFilter,
+		"dump":          DumpFilter,
+	}
+	for name, fn := range newFilters {
+		err := pongo2.RegisterFilter(name, fn)
+		if err != nil {
+			panic(err)
+		}
+	}
+}

pkg/filter/json.go

@@ -0,0 +1,35 @@
+package filter
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/flosch/pongo2"
+)
+
+// JSONFilter is a pongo2 filter, which returns a json string of the input
+func JSONFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
+	pretty := false
+	for _, s := range strings.Split(param.String(), ",") {
+		switch s {
+		case "pretty":
+			pretty = true
+		}
+	}
+	var err error
+	var jsonBytes []byte
+	if pretty {
+		jsonBytes, err = json.MarshalIndent(in.Interface(), "", "    ")
+
+	} else {
+		jsonBytes, err = json.Marshal(in.Interface())
+	}
+	if err != nil {
+		return nil, &pongo2.Error{
+			Sender:    "filter:json",
+			OrigError: err,
+		}
+	}
+
+	return pongo2.AsSafeValue(string(jsonBytes)), nil
+}

pkg/filter/json_test.go

@@ -0,0 +1,50 @@
+package filter
+
+import (
+	"math"
+	"testing"
+
+	"github.com/flosch/pongo2"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestJSONFilter(t *testing.T) {
+	Convey("set context", t, func() {
+		ctx := pongo2.Context{
+			"teststr": "test",
+			"testmap": map[string]interface{}{
+				"float": 1.23,
+				"int":   5,
+				"str":   "test",
+			},
+			"testerr": math.Inf(1),
+		}
+		Convey("parse template", func() {
+			output, err := pongo2.RenderTemplateString("{{ teststr|safe|json }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, `"test"`)
+
+			output, err = pongo2.RenderTemplateString("{{ testmap|safe|json }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, `{"float":1.23,"int":5,"str":"test"}`)
+
+			output, err = pongo2.RenderTemplateString(`{{ testmap|safe|json:"pretty" }}`, ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, `{
+    "float": 1.23,
+    "int": 5,
+    "str": "test"
+}`)
+
+			output, err = pongo2.RenderTemplateString("{{ testnil|safe|json }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, `null`)
+
+			output, err = pongo2.RenderTemplateString("{{ testerr|safe|json }}", ctx)
+			So(err, ShouldNotBeNil)
+			So(output, ShouldEqual, ``)
+
+		})
+	})
+}

pkg/filter/markdown.go

@@ -0,0 +1,51 @@
+package filter
+
+import (
+	"fmt"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"github.com/flosch/pongo2"
+)
+
+// MarkdownFilter is a pongo2 filter, which converts markdown to html
+func MarkdownFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
+	chromaRenderer := false
+	chromaStyle := ""
+	if pStr := param.String(); pStr != "" {
+		for _, s := range strings.Split(pStr, ",") {
+			e := strings.Split(s, "=")
+			if len(e) < 2 {
+				return nil, &pongo2.Error{
+					Sender:    "filter:markdown",
+					OrigError: fmt.Errorf("invalid parameter: %s", s),
+				}
+			}
+			switch e[0] {
+			case "s":
+				if e[1] == "" {
+					return nil, &pongo2.Error{
+						Sender:    "filter:markdown",
+						OrigError: fmt.Errorf("need a syntax sheme name for parameter '%s='", e[0]),
+					}
+				}
+				chromaRenderer = true
+				chromaStyle = e[1]
+			default:
+				return nil, &pongo2.Error{
+					Sender:    "filter:markdown",
+					OrigError: fmt.Errorf("unknown parameter '%s='", e[0]),
+				}
+			}
+
+		}
+	}
+	return pongo2.AsSafeValue(
+			string(
+				helper.RenderMarkdown(
+					[]byte(in.String()),
+					chromaRenderer,
+					chromaStyle,
+				))),
+		nil
+}

pkg/filter/markdown_test.go

@@ -0,0 +1,49 @@
+package filter
+
+import (
+	"testing"
+
+	"github.com/flosch/pongo2"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestMarkdownFilter(t *testing.T) {
+	Convey("set context", t, func() {
+		ctx := pongo2.Context{
+			"testvar":  "# test",
+			"testcode": "```sh\ntest=test\n```",
+		}
+		Convey("parse template", func() {
+			output, err := pongo2.RenderTemplateString("{{ testvar|markdown }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, "<h1>test</h1>\n")
+
+			output, err = pongo2.RenderTemplateString("{{ testcode|markdown }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, `<pre><code class="language-sh">test=test
+</code></pre>
+`)
+
+			output, err = pongo2.RenderTemplateString(`{{ testcode|markdown:"s=monokai" }}`, ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, `<pre style="color:#f8f8f2;background-color:#272822">test<span style="color:#f92672">=</span>test
+</pre>`)
+
+			output, err = pongo2.RenderTemplateString(`{{ testcode|markdown:"s=" }}`, ctx)
+			So(output, ShouldBeEmpty)
+			So(err, ShouldNotBeNil)
+			So(err.Error(), ShouldContainSubstring, "need a syntax sheme name for parameter")
+
+			output, err = pongo2.RenderTemplateString(`{{ testcode|markdown:"test=test" }}`, ctx)
+			So(output, ShouldBeEmpty)
+			So(err, ShouldNotBeNil)
+			So(err.Error(), ShouldContainSubstring, "unknown parameter")
+
+			output, err = pongo2.RenderTemplateString(`{{ testcode|markdown:"s=monokai,test" }}`, ctx)
+			So(output, ShouldBeEmpty)
+			So(err, ShouldNotBeNil)
+			So(err.Error(), ShouldContainSubstring, "invalid parameter: test")
+		})
+	})
+}

pkg/filter/relative_path.go

@@ -0,0 +1,15 @@
+package filter
+
+import (
+	"gitbase.de/apairon/mark2web/pkg/mark2web"
+	"github.com/flosch/pongo2"
+)
+
+// RelativePathFilter returns the relative path to navpoint based on current nav
+func RelativePathFilter(in, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
+	return pongo2.AsValue(
+		mark2web.CurrentTreeNode.ResolveNavPath(
+			in.String(),
+		),
+	), nil
+}

pkg/filter/relative_path_test.go

@@ -0,0 +1,53 @@
+package filter
+
+import (
+	"testing"
+
+	"gitbase.de/apairon/mark2web/pkg/mark2web"
+
+	"github.com/flosch/pongo2"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func TestRelativePathFilter(t *testing.T) {
+	Convey("set context", t, func() {
+		ctx := pongo2.Context{
+			"testrel": "rel",
+			"testabs": "/abs",
+			"testsub": "../sub/rel",
+		}
+
+		mark2web.Config.Directories.Output = "../../test/out"
+
+		mark2web.CurrentTreeNode = &mark2web.TreeNode{
+			InputPath:  "../../test/in/content",
+			OutputPath: "../../test/out/sub",
+			Config: &mark2web.PathConfig{
+				Imaging: &mark2web.ImagingConfig{
+					Quality: 60,
+					Height:  300,
+					Width:   300,
+					Process: "fit",
+				},
+			},
+		}
+
+		Convey("parse template", func() {
+
+			output, err := pongo2.RenderTemplateString("{{ testrel|relative_path }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, "rel")
+
+			output, err = pongo2.RenderTemplateString("{{ testabs|relative_path }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, "../abs")
+
+			/* TODO
+			output, err = pongo2.RenderTemplateString("{{ testsub|relative_path }}", ctx)
+			So(err, ShouldBeNil)
+			So(output, ShouldEqual, "rel")
+			*/
+		})
+	})
+}

pkg/helper/dir.go

@@ -0,0 +1,26 @@
+package helper
+
+import (
+	"os"
+)
+
+// CreateDirectory creates direcory with all missing parents and panic if error
+func CreateDirectory(dir string) {
+	Log.Debugf("trying to create output directory: %s", dir)
+
+	if dirH, err := os.Stat(dir); os.IsNotExist(err) {
+		err := os.MkdirAll(dir, 0755)
+		if err != nil {
+			Log.Panicf("could not create output directory '%s': %s", dir, err)
+		}
+		Log.Noticef("created output directory: %s", dir)
+	} else if dirH != nil {
+		if dirH.IsDir() {
+			Log.Noticef("output directory '%s' already exists", dir)
+		} else {
+			Log.Panicf("output directory '%s' is no directory", dir)
+		}
+	} else {
+		Log.Panicf("unknown error for output directory '%s': %s", dir, err)
+	}
+}

pkg/helper/logger.go

@@ -0,0 +1,52 @@
+package helper
+
+import (
+	"os"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/op/go-logging"
+)
+
+// Log is global logger
+var Log = logging.MustGetLogger("myLogger")
+
+// ConfigureLogger sets logger backend and level
+func ConfigureLogger(level string) {
+	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
+	logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
+		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
+	))
+	logBackendLeveled := logging.AddModuleLevel(logBackendFormatter)
+	logBackendLevel := logging.INFO
+	switch level {
+	case "debug":
+		logBackendLevel = logging.DEBUG
+		break
+
+	case "info":
+		logBackendLevel = logging.INFO
+		break
+
+	case "notice":
+		logBackendLevel = logging.NOTICE
+		break
+
+	case "warning":
+		logBackendLevel = logging.WARNING
+		break
+
+	case "error":
+		logBackendLevel = logging.ERROR
+		break
+
+	}
+	logBackendLeveled.SetLevel(logBackendLevel, "")
+	logging.SetBackend(logBackendLeveled)
+}
+
+func init() {
+	spew.Config.DisablePointerAddresses = true
+	spew.Config.DisableCapacities = true
+	spew.Config.DisableMethods = true
+	spew.Config.DisablePointerMethods = true
+}

pkg/helper/map_string.go

@@ -0,0 +1,47 @@
+package helper
+
+import "fmt"
+
+// MapString is a map[string]interface{} which always unmarsahls yaml to map[string]interface{}
+type MapString map[string]interface{}
+
+// UnmarshalYAML handles all maps as map[string]interface{} for later JSON
+// see https://github.com/elastic/beats/blob/6435194af9f42cbf778ca0a1a92276caf41a0da8/libbeat/common/mapstr.go
+func (ms *MapString) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	var result map[interface{}]interface{}
+	err := unmarshal(&result)
+	if err != nil {
+		return err
+	}
+	*ms = cleanUpInterfaceMap(result)
+	return nil
+}
+
+func cleanUpInterfaceArray(in []interface{}) []interface{} {
+	result := make([]interface{}, len(in))
+	for i, v := range in {
+		result[i] = cleanUpMapValue(v)
+	}
+	return result
+}
+
+func cleanUpInterfaceMap(in map[interface{}]interface{}) MapString {
+	result := make(MapString)
+	for k, v := range in {
+		result[fmt.Sprintf("%v", k)] = cleanUpMapValue(v)
+	}
+	return result
+}
+
+func cleanUpMapValue(v interface{}) interface{} {
+	switch v := v.(type) {
+	case []interface{}:
+		return cleanUpInterfaceArray(v)
+	case map[interface{}]interface{}:
+		return cleanUpInterfaceMap(v)
+	case string, bool, int, int8, int16, int32, int64, float32, float64:
+		return v
+	default:
+		return fmt.Sprintf("%v", v)
+	}
+}

pkg/helper/markdown.go

@@ -0,0 +1,30 @@
+package helper
+
+import (
+	"bytes"
+
+	"github.com/Depado/bfchroma"
+	"gopkg.in/russross/blackfriday.v2"
+)
+
+// RenderMarkdown renders input to html with chroma syntax highlighting if wanted
+func RenderMarkdown(input []byte, chromaRenderer bool, chromaStyle string) []byte {
+	var options []blackfriday.Option
+
+	if chromaStyle == "" {
+		chromaStyle = "monokai"
+	}
+	if chromaRenderer {
+		options = []blackfriday.Option{
+			blackfriday.WithRenderer(
+				bfchroma.NewRenderer(
+					bfchroma.Style(chromaStyle),
+				),
+			),
+		}
+	}
+
+	// fix \r from markdown for blackfriday
+	input = bytes.Replace(input, []byte("\r"), []byte(""), -1)
+	return blackfriday.Run(input, options...)
+}

pkg/helper/merge.go

@@ -0,0 +1,28 @@
+package helper
+
+import (
+	"reflect"
+
+	"github.com/imdario/mergo"
+)
+
+type ptrTransformer struct{}
+
+func (t ptrTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
+	if typ.Kind() == reflect.Ptr {
+		return func(dst, src reflect.Value) error {
+			if dst.CanSet() {
+				if dst.IsNil() {
+					dst.Set(src)
+				}
+			}
+			return nil
+		}
+	}
+	return nil
+}
+
+// Merge merges 2 objects or maps
+func Merge(dst, src interface{}) error {
+	return mergo.Merge(dst, src, mergo.WithTransformers(ptrTransformer{}))
+}

pkg/helper/regexp.go

@@ -0,0 +1,18 @@
+package helper
+
+import "regexp"
+
+// GetRegexpParams gets a map of named regexp group matches
+// use pe. (?P<Year>\d{4})-(?P<Month>\d{2})-(?P<Day>\d{2}) as regexp
+func GetRegexpParams(regEx *regexp.Regexp, str string) (paramsMap map[string]string) {
+
+	match := regEx.FindStringSubmatch(str)
+
+	paramsMap = make(map[string]string)
+	for i, name := range regEx.SubexpNames() {
+		if i > 0 && i <= len(match) {
+			paramsMap[name] = match[i]
+		}
+	}
+	return
+}

pkg/helper/webrequest.go

@@ -0,0 +1,53 @@
+package helper
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"strings"
+)
+
+// JSONWebRequest will GET a json object/array from a given URL
+func JSONWebRequest(url string) interface{} {
+	Log.Noticef("requesting url via GET %s", url)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		Log.Panicf("could not get url '%s': %s", url, err)
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		Log.Panicf("could not read body from url '%s': %s", url, err)
+	}
+
+	Log.Debugf("output from url '%s':\n%s", url, string(body))
+
+	if resp.StatusCode >= 400 {
+		Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
+	}
+
+	contentType := resp.Header.Get("Content-Type")
+
+	if strings.Contains(contentType, "json") {
+
+	} else {
+		Log.Panicf("is not json '%s' from url '%s'", contentType, url)
+	}
+
+	jsonMap := make(map[string]interface{})
+	err = json.Unmarshal(body, &jsonMap)
+	if err == nil {
+		return jsonMap
+	}
+
+	jsonArrayMap := make([]map[string]interface{}, 0)
+	err = json.Unmarshal(body, &jsonArrayMap)
+	if err == nil {
+		return jsonArrayMap
+	}
+
+	Log.Panicf("could not read json from '%s': invalid type", url)
+	return nil
+}

pkg/mark2web/assets.go

@@ -0,0 +1,56 @@
+package mark2web
+
+import (
+	"log"
+	"path"
+	"regexp"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	cpy "github.com/otiai10/copy"
+)
+
+// ProcessAssets copies the assets from input to output dir
+func ProcessAssets() {
+	switch Config.Assets.Action {
+	case "copy":
+		from := Config.Assets.FromPath
+		to := Config.Assets.ToPath
+		if !strings.HasPrefix(from, "/") {
+			from = Config.Directories.Input + "/" + from
+		}
+		if !strings.HasPrefix(to, "/") {
+			to = Config.Directories.Output + "/" + to
+		}
+		helper.Log.Noticef("copying assets from '%s' to '%s'", from, to)
+		err := cpy.Copy(from, to)
+		if err != nil {
+			helper.Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
+		}
+
+		if Config.Assets.Compress {
+			compressFilesInDir(to)
+		}
+	}
+}
+
+// fixAssetsPath replaces assets path based on current path
+func (node *TreeNode) fixAssetsPath(str string) string {
+	if find := Config.Assets.FixTemplate.Find; find != "" {
+		helper.Log.Debugf("fixing assets paths for path '%s'", node.CurrentNavPath())
+		repl := Config.Assets.FixTemplate.Replace
+		toPath := Config.Assets.ToPath
+
+		bToRoot := node.BackToRootPath()
+		regex, err := regexp.Compile(find)
+		if err != nil {
+			log.Panicf("could not compile regexp '%s' for assets path: %s", find, err)
+		}
+		repl = bToRoot + toPath + "/" + repl
+		repl = path.Clean(repl) + "/"
+		helper.Log.Debugf("new assets paths: %s", repl)
+		return regex.ReplaceAllString(str, repl)
+	}
+
+	return str
+}

pkg/mark2web/brotli.go

@@ -0,0 +1,48 @@
+// +build cgo
+
+package mark2web
+
+import (
+	"io"
+	"os"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"github.com/itchio/go-brotli/enc"
+)
+
+var brotliSupported = true
+
+func handleBrotliCompression(filename string, content []byte) {
+	brFilename := filename + ".br"
+
+	helper.Log.Infof("writing to compressed output file: %s", brFilename)
+
+	f, err := os.Create(brFilename)
+	if err != nil {
+		helper.Log.Panicf("could not create file '%s': %s", brFilename, err)
+	}
+	defer f.Close()
+
+	bw := enc.NewBrotliWriter(f, nil)
+	defer bw.Close()
+
+	if content != nil {
+		// content given
+		_, err = bw.Write(content)
+		if err != nil {
+			helper.Log.Panicf("could not write brotli content for '%s': %s", filename, err)
+		}
+	} else {
+		// read file
+		r, err := os.Open(filename)
+		if err != nil {
+			helper.Log.Panicf("could not open file '%s': %s", filename, err)
+		}
+		defer r.Close()
+
+		_, err = io.Copy(bw, r)
+		if err != nil {
+			helper.Log.Panicf("could not write brotli file for '%s': %s", filename, err)
+		}
+	}
+}

pkg/mark2web/collection.go

@@ -0,0 +1,213 @@
+package mark2web
+
+import (
+	"io/ioutil"
+	"path"
+	"regexp"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/flosch/pongo2"
+)
+
+type colCacheEntry struct {
+	data     interface{}
+	hit      int
+	navnames []string
+}
+
+var colCache = make(map[string]*colCacheEntry)
+
+func (node *TreeNode) handleCollections() {
+	collections := append(node.Config.Collections, node.Config.This.Collections...)
+	for _, colConfig := range collections {
+		if colConfig.Name == nil || *colConfig.Name == "" {
+			helper.Log.Panicf("missing Name in collection config in '%s'", node.InputPath)
+		}
+		if (colConfig.URL == nil || *colConfig.URL == "") &&
+			(colConfig.Directory == nil) {
+			helper.Log.Panicf("missing URL and Directory in collection config in '%s'", node.InputPath)
+		}
+
+		if node.ColMap == nil {
+			node.ColMap = make(helper.MapString)
+		}
+		ctx := NewContext()
+		ctx["This"] = node.Config.This
+		ctx["Data"] = node.Config.Data
+
+		var colData interface{}
+
+		errSrcText := ""
+		cacheKey := ""
+
+		if colConfig.URL != nil {
+			url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
+			if err != nil {
+				helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", node.InputPath, err)
+			}
+
+			errSrcText = "URL " + url
+			cacheKey = url
+
+			if cacheEntry, ok := colCache[url]; ok {
+				colData = cacheEntry.data
+				cacheEntry.hit++
+			} else {
+				helper.Log.Noticef("reading collection from: %s", errSrcText)
+				colData = helper.JSONWebRequest(url)
+				colCache[url] = &colCacheEntry{
+					data:     colData,
+					navnames: make([]string, 0),
+				}
+			}
+
+		} else {
+			path := node.ResolveInputPath(colConfig.Directory.Path)
+			errSrcText = "DIR " + path
+
+			helper.Log.Noticef("reading collection from: %s", errSrcText)
+			d, err := ioutil.ReadDir(path)
+			if err != nil {
+				helper.Log.Panicf("could not read directory '%s': %s", path, err)
+			}
+
+			mStr := "."
+			if colConfig.Directory.MatchFilename != "" {
+				mStr = colConfig.Directory.MatchFilename
+			}
+			matcher, err := regexp.Compile(mStr)
+			if err != nil {
+				helper.Log.Panicf("could not compile regex for MatchFilename '%s' in '%s': %s", mStr, path, err)
+			}
+
+			if colConfig.Directory.ReverseOrder {
+				for i := len(d)/2 - 1; i >= 0; i-- {
+					opp := len(d) - 1 - i
+					d[i], d[opp] = d[opp], d[i]
+				}
+			}
+
+			fcolData := make([]pongo2.Context, 0)
+			for _, fh := range d {
+				if !fh.IsDir() && matcher.MatchString(fh.Name()) {
+					inFile := path + "/" + fh.Name()
+					md, err := ioutil.ReadFile(inFile)
+					if err != nil {
+						helper.Log.Panicf("could not read file '%s': %s", inFile, err)
+					}
+					_, ctx := node.processMarkdownWithHeader(md, inFile)
+					(*ctx)["FilenameMatch"] = helper.GetRegexpParams(matcher, fh.Name())
+					fcolData = append(fcolData, *ctx)
+				}
+			}
+
+			colData = fcolData
+		}
+
+		node.ColMap[*colConfig.Name] = colData
+
+		if navT := colConfig.NavTemplate; navT != nil {
+			var entries []interface{}
+			var ok bool
+			if navT.EntriesAttribute != "" {
+				var colDataMap map[string]interface{}
+				if colDataMap, ok = colData.(map[string]interface{}); ok {
+					entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
+					if !ok {
+						helper.Log.Debug(spew.Sdump(colDataMap))
+						helper.Log.Panicf("invalid json data in [%s] from '%s' for entries", navT.EntriesAttribute, errSrcText)
+					}
+				}
+			} else {
+				entries, ok = colData.([]interface{})
+			}
+			if !ok {
+				helper.Log.Debug(spew.Sdump(colData))
+				helper.Log.Panicf("invalid json data from '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", errSrcText)
+			}
+
+			// build navigation with detail sites
+			for idx, colEl := range entries {
+				ctxE := make(pongo2.Context)
+				err := helper.Merge(&ctxE, ctx)
+				if err != nil {
+					helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
+				}
+				var jsonCtx map[string]interface{}
+				if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
+					helper.Log.Debug(spew.Sdump(colEl))
+					helper.Log.Panicf("no json object for entry index %d from '%s'", idx, errSrcText)
+				}
+				err = helper.Merge(&ctxE, pongo2.Context(jsonCtx))
+				if err != nil {
+					helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
+				}
+
+				tpl := ""
+				if navT.Template != "" {
+					tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", node.InputPath, err)
+					}
+				}
+				if tpl == "" {
+					tpl = *node.Config.Template
+				}
+
+				dataKey := ""
+				if navT.DataKey != "" {
+					dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", node.InputPath, err)
+					}
+				}
+
+				goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
+				if err != nil {
+					helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", node.InputPath, err)
+				}
+				goTo = strings.Trim(goTo, "/")
+				goTo = path.Clean(goTo)
+
+				if strings.Contains(goTo, "..") {
+					helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
+				}
+				if goTo == "." {
+					helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
+				}
+				if goTo == "" {
+					helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
+				}
+
+				navname := ""
+				if navT.Navname != "" {
+					navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", node.InputPath, err)
+					}
+				}
+				body := ""
+				if navT.Body != "" {
+					body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", node.InputPath, err)
+					}
+				}
+
+				if l := len(colCache[cacheKey].navnames); colCache[cacheKey].hit > 1 &&
+					l > 0 &&
+					navname == colCache[cacheKey].navnames[l-1] {
+					// navname before used same url, so recursion loop
+					helper.Log.Panicf("collection request loop detected for in '%s' for : %s", node.InputPath, errSrcText)
+				}
+
+				colCache[cacheKey].navnames = append(colCache[cacheKey].navnames, navname)
+
+				node.addSubNode(tpl, goTo, navname, colEl, dataKey, body, navT.Hidden)
+			}
+		}
+	}
+
+}

pkg/mark2web/compress.go

@@ -0,0 +1,84 @@
+package mark2web
+
+import (
+	"compress/gzip"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+)
+
+func handleCompression(filename string, content []byte) {
+	ThreadStart(func() {
+		if _, ok := Config.Compress.Extensions[path.Ext(filename)]; ok {
+
+			if Config.Compress.Brotli {
+				handleBrotliCompression(filename, content)
+			}
+
+			if Config.Compress.GZIP {
+				gzFilename := filename + ".gz"
+
+				helper.Log.Infof("writing to compressed output file: %s", gzFilename)
+
+				f, err := os.Create(gzFilename)
+				if err != nil {
+					helper.Log.Panicf("could not create file '%s': %s", gzFilename, err)
+				}
+				defer f.Close()
+
+				zw, err := gzip.NewWriterLevel(f, gzip.BestCompression)
+				if err != nil {
+					helper.Log.Panicf("could not initialize gzip writer for '%s': %s", filename, err)
+				}
+				defer zw.Close()
+
+				if content != nil {
+					// content given
+					_, err = zw.Write(content)
+					if err != nil {
+						helper.Log.Panicf("could not write gziped content for '%s': %s", filename, err)
+					}
+				} else {
+					// read file
+					r, err := os.Open(filename)
+					if err != nil {
+						helper.Log.Panicf("could not open file '%s': %s", filename, err)
+					}
+					defer r.Close()
+
+					_, err = io.Copy(zw, r)
+					if err != nil {
+						helper.Log.Panicf("could not gzip file '%s': %s", filename, err)
+					}
+				}
+
+			}
+		}
+	})
+}
+
+func compressFilesInDir(dir string) {
+	helper.Log.Noticef("compressing configured files in: %s", dir)
+
+	var _processDir func(string)
+	_processDir = func(d string) {
+		entries, err := ioutil.ReadDir(d)
+		if err != nil {
+			helper.Log.Panicf("could not read dir '%s': %s", d, err)
+		}
+
+		for _, entry := range entries {
+			if entry.IsDir() {
+				_processDir(d + "/" + entry.Name())
+			} else {
+				handleCompression(d+"/"+entry.Name(), nil)
+			}
+		}
+	}
+
+	_processDir(dir)
+
+}

pkg/mark2web/config_global.go

@@ -0,0 +1,56 @@
+package mark2web
+
+import (
+	"io/ioutil"
+
+	"gopkg.in/yaml.v2"
+)
+
+// GlobalConfig is config which is used only once in root dir
+type GlobalConfig struct {
+	Webserver struct {
+		Type string `yaml:"Type"`
+	} `yaml:"Webserver"`
+
+	Assets struct {
+		Compress    bool   `yaml:"Compress"`
+		FromPath    string `yaml:"FromPath"`
+		ToPath      string `yaml:"ToPath"`
+		Action      string `yaml:"Action"`
+		FixTemplate struct {
+			Find    string `yaml:"Find"`
+			Replace string `yaml:"Replace"`
+		} `yaml:"FixTemplate"`
+	} `yaml:"Assets"`
+
+	OtherFiles struct {
+		Action string `yaml:"Action"`
+	} `yaml:"OtherFiles"`
+
+	Compress struct {
+		Brotli     bool              `yaml:"Brotli"`
+		GZIP       bool              `yaml:"GZIP"`
+		Extensions map[string]string `yaml:"Extensions"`
+	} `yaml:"Compress"`
+
+	Directories struct {
+		Input  string
+		Output string
+	}
+}
+
+// Config is global config
+var Config = new(GlobalConfig)
+
+// ReadFromFile reads yaml config from file
+func (c *GlobalConfig) ReadFromFile(filename string) error {
+	data, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return err
+	}
+	err = yaml.Unmarshal(data, c)
+	if err != nil {
+		return err
+	}
+	return nil
+}

pkg/mark2web/config_path.go

@@ -0,0 +1,98 @@
+package mark2web
+
+import "gitbase.de/apairon/mark2web/pkg/helper"
+
+// CollectionDirectoryConfig specifies how to handle a directory of markdown files as a collection
+type CollectionDirectoryConfig struct {
+	Path          string `yaml:"Path"`
+	MatchFilename string `yaml:"MatchFilename"`
+	ReverseOrder  bool   `yaml:"ReverseOrder"`
+}
+
+// CollectionConfig describes a collection
+type CollectionConfig struct {
+	Name        *string                    `yaml:"Name"`
+	URL         *string                    `yaml:"URL"`
+	Directory   *CollectionDirectoryConfig `yaml:"Directory"`
+	NavTemplate *struct {
+		EntriesAttribute string `yaml:"EntriesAttribute"`
+		GoTo             string `yaml:"GoTo"`
+		Navname          string `yaml:"Navname"`
+		Body             string `yaml:"Body"`
+		DataKey          string `yaml:"DataKey"`
+		Hidden           bool   `yaml:"Hidden"`
+		Template         string `yaml:"Template"`
+		//Recursive        bool   `yaml:"Recursive"`
+	} `yaml:"NavTemplate"`
+}
+
+// ThisPathConfig is struct for This in paths yaml
+type ThisPathConfig struct {
+	Navname     *string             `yaml:"Navname"`
+	GoTo        *string             `yaml:"GoTo"`
+	Collections []*CollectionConfig `yaml:"Collections"`
+	Data        helper.MapString    `yaml:"Data"`
+}
+
+// IndexConfig describes index input and output file
+type IndexConfig struct {
+	InputFile   *string `yaml:"InputFile"`
+	InputString *string `yaml:"InputString"`
+	OutputFile  *string `yaml:"OutputFile"`
+}
+
+// MetaData describes meta data for current site/tree node
+type MetaData struct {
+	Title       *string `yaml:"Title"`
+	Description *string `yaml:"Description"`
+	Keywords    *string `yaml:"Keywords"`
+}
+
+// DirnameConfig describes how to handle directory names
+type DirnameConfig struct {
+	Strip        *string `yaml:"Strip"`
+	IgnoreForNav *string `yaml:"IgnoreForNav"`
+}
+
+// FilenameConfig describes how to handle filenames
+type FilenameConfig struct {
+	Strip           *string `yaml:"Strip"`
+	Ignore          *string `yaml:"Ignore"`
+	OutputExtension *string `yaml:"OutputExtension"`
+}
+
+// MarkdownConfig describes markdown handling
+type MarkdownConfig struct {
+	ChromaRenderer *bool   `yaml:"ChromaRenderer"`
+	ChromaStyle    *string `yaml:"ChromaStyle"`
+}
+
+// ImagingConfig defines parameter for imaging processing
+type ImagingConfig struct {
+	Width   int    `yaml:"Width"`
+	Height  int    `yaml:"Height"`
+	Process string `yaml:"Process"`
+	Anchor  string `yaml:"Anchor"`
+	Quality int    `yaml:"Quality"`
+
+	TargetDir string `yaml:"-"`
+	Filename  string `yaml:"-"`
+	Format    string `yaml:"-"`
+}
+
+// PathConfig of subdir
+type PathConfig struct {
+	This     ThisPathConfig  `yaml:"This"`
+	Template *string         `yaml:"Template"`
+	Index    *IndexConfig    `yaml:"Index"`
+	Meta     *MetaData       `yaml:"Meta"`
+	Path     *DirnameConfig  `yaml:"Path"`
+	Filename *FilenameConfig `yaml:"Filename"`
+	Markdown *MarkdownConfig `yaml:"Markdown"`
+	Imaging  *ImagingConfig  `yaml:"Imaging"`
+
+	Data helper.MapString `yaml:"Data"`
+
+	// Collections here are recursive if saved as nav, so request should be filtered
+	Collections []*CollectionConfig `yaml:"Collections"`
+}

pkg/mark2web/content.go

@@ -0,0 +1,301 @@
+package mark2web
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path"
+	"regexp"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/flosch/pongo2"
+	cpy "github.com/otiai10/copy"
+	"gopkg.in/yaml.v2"
+)
+
+// ReadContentDir walks through content directory and builds the tree of configurations
+func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string, conf *PathConfig) {
+	if node.root == nil {
+		// first node is root
+		node.root = node
+	}
+	node.fillConfig(inBase, outBase, dir, conf)
+
+	files, err := ioutil.ReadDir(node.InputPath)
+	if err != nil {
+		helper.Log.Panic(err)
+	}
+
+	// first only files
+	for _, f := range files {
+		p := node.InputPath + "/" + f.Name()
+		if !f.IsDir() && f.Name() != "config.yml" {
+			switch path.Ext(f.Name()) {
+			case ".md":
+				helper.Log.Debugf(".MD %s", p)
+				if node.InputFiles == nil {
+					node.InputFiles = make([]string, 0)
+				}
+				node.InputFiles = append(node.InputFiles, f.Name())
+				break
+			default:
+				helper.Log.Debugf("FIL %s", p)
+				if node.OtherFiles == nil {
+					node.OtherFiles = make([]string, 0)
+				}
+				node.OtherFiles = append(node.OtherFiles, f.Name())
+			}
+		}
+	}
+
+	// only directorys, needed config before
+	for _, f := range files {
+		p := node.InputPath + "/" + f.Name()
+		if f.IsDir() {
+			helper.Log.Debugf("DIR %s", p)
+			newTree := new(TreeNode)
+			newTree.root = node.root
+			if node.Sub == nil {
+				node.Sub = make([]*TreeNode, 0)
+			}
+			node.Sub = append(node.Sub, newTree)
+			newTree.ReadContentDir(node.InputPath, node.OutputPath, f.Name(), node.Config)
+		}
+	}
+}
+
+func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*PathConfig, *pongo2.Context) {
+
+	newConfig := new(PathConfig)
+
+	headerRegex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?")
+	yamlData := headerRegex.Find(md)
+	if string(yamlData) != "" {
+		// replace tabs
+		yamlData = bytes.Replace(yamlData, []byte("\t"), []byte("    "), -1)
+
+		helper.Log.Debugf("found yaml header in '%s', merging config", errorRef)
+		err := yaml.Unmarshal(yamlData, newConfig)
+		if err != nil {
+			helper.Log.Panicf("could not parse YAML header from '%s': %s", errorRef, err)
+		}
+
+		helper.Log.Debug("merging config with upper config")
+		oldThis := newConfig.This
+		helper.Merge(newConfig, node.Config)
+		newConfig.This = oldThis
+
+		helper.Log.Debug(spew.Sdump(newConfig))
+
+		md = headerRegex.ReplaceAll(md, []byte(""))
+	} else {
+		helper.Merge(newConfig, node.Config)
+	}
+
+	// use --- for splitting document in markdown parts
+	regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?")
+	inputParts := regex.Split(string(md), -1)
+	htmlParts := make([]*pongo2.Value, 0)
+
+	chromaRenderer := false
+	chromaStyle := "monokai"
+	if m := newConfig.Markdown; m != nil {
+		if m.ChromaRenderer != nil && *m.ChromaRenderer {
+			chromaRenderer = true
+		}
+		if m.ChromaStyle != nil && *m.ChromaStyle != "" {
+			chromaStyle = *m.ChromaStyle
+		}
+	}
+	for _, iPart := range inputParts {
+		htmlParts = append(htmlParts,
+			pongo2.AsSafeValue(
+				string(helper.RenderMarkdown([]byte(iPart), chromaRenderer, chromaStyle))))
+	}
+
+	// build navigation
+	navMap := make(map[string]*NavElement)
+	navSlice := make([]*NavElement, 0)
+	navActive := make([]*NavElement, 0)
+	node.buildNavigation(&navMap, &navSlice, &navActive)
+
+	// read yaml header as data for template
+	ctx := NewContext()
+	ctx["This"] = newConfig.This
+	ctx["Meta"] = newConfig.Meta
+	ctx["Markdown"] = newConfig.Markdown
+	ctx["Data"] = newConfig.Data
+	ctx["ColMap"] = node.root.ColMap // root as NavMap and NavSlice, for sub go to NavElement.ColMap
+	ctx["NavMap"] = navMap
+	ctx["NavSlice"] = navSlice
+	ctx["NavActive"] = navActive
+	ctx["Body"] = pongo2.AsSafeValue(string(helper.RenderMarkdown(md, chromaRenderer, chromaStyle)))
+	ctx["BodyParts"] = htmlParts
+	ctx["CurrentPath"] = node.CurrentNavPath()
+	// set active nav element
+	if len(navActive) > 0 {
+		ctx["NavElement"] = navActive[len(navActive)-1]
+	} else {
+		// if no active path to content, we are in root dir
+		ctx["NavElement"] = &NavElement{
+			GoTo:     node.BackToRootPath(),
+			Active:   true,
+			ColMap:   node.ColMap,
+			Data:     node.Config.Data,
+			This:     node.Config.This,
+			SubMap:   &navMap,
+			SubSlice: &navSlice,
+		}
+	}
+
+	return newConfig, &ctx
+}
+
+// ProcessContent walks recursivly through the input paths and processes all files for output
+func (node *TreeNode) ProcessContent() {
+	helper.CreateDirectory(node.OutputPath)
+
+	if node.root != node {
+		// write htaccess for rewrites, root will be written in WriteWebserverConfig()
+		goTo := node.Config.This.GoTo
+		if goTo != nil && *goTo != "" {
+			goToFixed := *goTo
+			if strings.HasPrefix(goToFixed, "/") {
+				goToFixed = node.BackToRootPath() + goToFixed
+			}
+			goToFixed = path.Clean(goToFixed)
+
+			htaccessRedirect(node.OutputPath, goToFixed)
+		}
+	}
+
+	for _, file := range node.InputFiles {
+		inFile := "InputString"
+
+		// ignore ???
+		ignoreFile := false
+		if file != "" {
+			inFile = node.InputPath + "/" + file
+			var ignoreRegex *string
+			if f := node.Config.Filename; f != nil {
+				ignoreRegex = f.Ignore
+			}
+			if ignoreRegex != nil && *ignoreRegex != "" {
+				regex, err := regexp.Compile(*ignoreRegex)
+				if err != nil {
+					helper.Log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
+				}
+				ignoreFile = regex.MatchString(file)
+			}
+		}
+
+		if ignoreFile {
+			helper.Log.Infof("ignoring file '%s', because of filename.ignore", inFile)
+		} else {
+			var input []byte
+
+			if file != "" {
+				helper.Log.Debugf("reading file: %s", inFile)
+
+				var err error
+				input, err = ioutil.ReadFile(inFile)
+				if err != nil {
+					helper.Log.Panicf("could not read '%s':%s", inFile, err)
+				}
+				helper.Log.Infof("processing input file '%s'", inFile)
+			} else {
+				// use input string if available and input filename == ""
+				var inputString *string
+				if i := node.Config.Index; i != nil {
+					inputString = i.InputString
+				}
+				if inputString != nil {
+					helper.Log.Debugf("using input string instead of file")
+					input = []byte(*inputString)
+				}
+			}
+
+			newConfig, ctx := node.processMarkdownWithHeader(input, inFile)
+
+			// build output filename
+			outputFilename := file
+
+			var stripRegex *string
+			var outputExt *string
+			if f := newConfig.Filename; f != nil {
+				stripRegex = f.Strip
+				outputExt = f.OutputExtension
+			}
+
+			var indexInputFile *string
+			var indexOutputFile *string
+			if i := newConfig.Index; i != nil {
+				indexInputFile = i.InputFile
+				indexOutputFile = i.OutputFile
+			}
+
+			if indexInputFile != nil &&
+				*indexInputFile == file &&
+				indexOutputFile != nil &&
+				*indexOutputFile != "" {
+				outputFilename = *indexOutputFile
+			} else {
+				if stripRegex != nil && *stripRegex != "" {
+					regex, err := regexp.Compile(*stripRegex)
+					if err != nil {
+						helper.Log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
+					}
+					outputFilename = regex.ReplaceAllString(outputFilename, "$1")
+				}
+				if outputExt != nil && *outputExt != "" {
+					outputFilename += "." + *outputExt
+				}
+			}
+
+			outFile := node.OutputPath + "/" + outputFilename
+			helper.Log.Debugf("using '%s' as output file", outFile)
+			helper.Log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
+			templateFilename := *newConfig.Template
+			result, err := renderTemplate(*newConfig.Template, node, newConfig, ctx)
+			if err != nil {
+				helper.Log.Panicf("could not execute template '%s' for input file '%s': %s", templateFilename, inFile, err)
+			}
+
+			result = node.fixAssetsPath(result)
+
+			helper.Log.Noticef("writing to output file: %s", outFile)
+			err = ioutil.WriteFile(outFile, []byte(result), 0644)
+			if err != nil {
+				helper.Log.Panicf("could not write to output file '%s': %s", outFile, err)
+			}
+
+			handleCompression(outFile, []byte(result))
+
+			//fmt.Println(string(html))
+		}
+	}
+
+	// process other files, copy...
+	for _, file := range node.OtherFiles {
+		switch Config.OtherFiles.Action {
+		case "copy":
+			from := node.InputPath + "/" + file
+			to := node.OutputPath + "/" + file
+			helper.Log.Noticef("copying file from '%s' to '%s'", from, to)
+			err := cpy.Copy(from, to)
+			if err != nil {
+				helper.Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
+			}
+
+			handleCompression(to, nil)
+		}
+	}
+
+	i := 0
+	// sub can dynamically increase, so no for range
+	for i < len(node.Sub) {
+		node.Sub[i].ProcessContent()
+		i++
+	}
+}

pkg/mark2web/context.go

@@ -0,0 +1,170 @@
+package mark2web
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/extemporalgenome/slug"
+	"github.com/flosch/pongo2"
+	"gopkg.in/yaml.v2"
+)
+
+// CurrentContext is current pongo2 template context
+var CurrentContext *pongo2.Context
+
+// CurrentTreeNode is current node we are on while processing template
+var CurrentTreeNode *TreeNode
+
+// NewContext returns prefilled context with some functions and variables
+func NewContext() pongo2.Context {
+	ctx := pongo2.Context{
+		"fnRequest": RequestFn,
+		"fnRender":  RenderFn,
+
+		"AssetsPath": Config.Assets.ToPath,
+		"Timestamp":  time.Now().Unix,
+	}
+	CurrentContext = &ctx
+
+	return ctx
+}
+
+func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfig) {
+	inPath := inBase
+	if subDir != "" {
+		inPath += "/" + subDir
+	}
+
+	helper.Log.Infof("reading input directory: %s", inPath)
+
+	node.InputPath = inPath
+
+	// read config
+	newConfig := new(PathConfig)
+	helper.Log.Debug("looking for config.yml ...")
+	configFile := inPath + "/config.yml"
+	if _, err := os.Stat(configFile); os.IsNotExist(err) {
+		helper.Log.Debug("no config.yml found in this directory, using upper configs")
+		helper.Merge(newConfig, conf)
+		// remove this
+		newConfig.This = ThisPathConfig{}
+	} else {
+		helper.Log.Debug("reading config...")
+		data, err := ioutil.ReadFile(configFile)
+		if err != nil {
+			helper.Log.Panicf("could not read file '%s': %s", configFile, err)
+		}
+		err = yaml.Unmarshal(data, newConfig)
+		if err != nil {
+			helper.Log.Panicf("could not parse YAML file '%s': %s", configFile, err)
+		}
+
+		helper.Log.Debug("merging config with upper config")
+		oldThis := newConfig.This
+		helper.Merge(newConfig, conf)
+		newConfig.This = oldThis
+
+		helper.Log.Debug(spew.Sdump(newConfig))
+	}
+
+	node.Config = newConfig
+
+	// calc outDir
+	stripedDir := subDir
+	var regexStr *string
+	if newConfig.Path != nil {
+		regexStr = newConfig.Path.Strip
+	}
+	if regexStr != nil && *regexStr != "" {
+		if regex, err := regexp.Compile(*regexStr); err != nil {
+			helper.Log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+subDir, err)
+		} else {
+			stripedDir = regex.ReplaceAllString(stripedDir, "$1")
+		}
+	}
+
+	if node.Config.This.Navname == nil {
+		navname := strings.Replace(stripedDir, "_", " ", -1)
+		node.Config.This.Navname = &navname
+	}
+
+	stripedDir = slug.Slug(stripedDir)
+	outPath := outBase + "/" + stripedDir
+	outPath = path.Clean(outPath)
+
+	helper.Log.Infof("calculated output directory: %s", outPath)
+	node.OutputPath = outPath
+
+	// handle collections
+	node.handleCollections()
+}
+
+func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx interface{}, dataMapKey string, body string, hideInNav bool) {
+	newNode := new(TreeNode)
+	newNode.root = node.root
+
+	newPathConfig := new(PathConfig)
+	if navname != "" {
+		newPathConfig.This = ThisPathConfig{
+			Navname: &navname,
+		}
+	}
+	if dataMapKey != "" {
+		if newPathConfig.Data == nil {
+			newPathConfig.Data = make(helper.MapString)
+		}
+		// as submap in Data
+		newPathConfig.Data[dataMapKey] = ctx
+	} else if m, ok := ctx.(map[string]interface{}); ok {
+		// direct set data
+		newPathConfig.Data = m
+	}
+
+	mergedConfig := new(PathConfig)
+	err := helper.Merge(mergedConfig, node.Config)
+	if err != nil {
+		helper.Log.Panicf("merge of path config failed: %s", err)
+	}
+	// dont merge Data[DataKey]
+	if dataMapKey != "" {
+		mergedConfig.Data[dataMapKey] = nil
+	} else {
+		mergedConfig.Data = make(helper.MapString)
+	}
+	err = helper.Merge(mergedConfig, newPathConfig)
+	if err != nil {
+		helper.Log.Panicf("merge of path config failed: %s", err)
+	}
+
+	newNode.fillConfig(
+		node.InputPath,
+		node.OutputPath,
+		subDir,
+		mergedConfig,
+	)
+
+	// fake via normal file behavior
+	newNode.Config.Template = &tplFilename
+	newNode.InputFiles = []string{""} // empty file is special for use InputString
+	indexInFile := ""
+	indexOutFile := "index.html"
+	if idx := newNode.Config.Index; idx != nil {
+		if idx.OutputFile != nil && *idx.OutputFile != "" {
+			indexOutFile = *idx.OutputFile
+		}
+	}
+	newNode.Config.Index = &IndexConfig{
+		InputFile:   &indexInFile,
+		OutputFile:  &indexOutFile,
+		InputString: &body,
+	}
+	newNode.Hidden = hideInNav
+
+	node.Sub = append(node.Sub, newNode)
+}

pkg/mark2web/context_fn.go

@@ -0,0 +1,31 @@
+package mark2web
+
+import (
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"github.com/flosch/pongo2"
+)
+
+// RequestFn will make a web request and returns map[string]interface form pongo2
+func RequestFn(url *pongo2.Value, args ...*pongo2.Value) *pongo2.Value {
+	u := url.String()
+	return pongo2.AsValue(helper.JSONWebRequest(u))
+}
+
+// RenderFn renders a pongo2 template with additional context
+func RenderFn(templateFilename, subDir, ctx *pongo2.Value, param ...*pongo2.Value) *pongo2.Value {
+	dataMapKey := ""
+	body := ""
+
+	for i, p := range param {
+		switch i {
+		case 0:
+			dataMapKey = p.String()
+		case 1:
+			body = p.String()
+		}
+	}
+
+	CurrentTreeNode.addSubNode(templateFilename.String(), subDir.String(), "", ctx.Interface(), dataMapKey, body, true)
+
+	return pongo2.AsValue(nil)
+}

pkg/mark2web/htaccess.go

@@ -0,0 +1,100 @@
+package mark2web
+
+import (
+	"io/ioutil"
+	"regexp"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+)
+
+func htaccessRedirect(outDir, goTo string) {
+	switch Config.Webserver.Type {
+	case "apache":
+		htaccessFile := outDir + "/.htaccess"
+		helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goTo)
+		err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
+RewriteRule ^$ %{REQUEST_URI}`+goTo+`/ [R,L]
+`), 0644)
+		if err != nil {
+			helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
+		}
+	}
+}
+
+// WriteWebserverConfig build the config for pre compression and more
+func (tree *TreeNode) WriteWebserverConfig() {
+	goTo := ""
+	if g := tree.Config.This.GoTo; g != nil && *g != "" {
+		goTo = strings.TrimPrefix(*g, "/")
+	}
+
+	switch Config.Webserver.Type {
+	case "apache":
+		configStr := `
+RewriteEngine on
+
+`
+
+		if goTo != "" {
+			configStr += `
+RewriteRule ^$ %{REQUEST_URI}` + goTo + `/ [R,L]
+
+`
+		}
+
+		configStr += `
+AddCharset UTF-8 .html
+AddCharset UTF-8 .json
+AddCharset UTF-8 .js
+AddCharset UTF-8 .css
+
+RemoveLanguage .br
+
+<IfModule mod_headers.c>
+`
+
+		rewriteMacro := func(e, c, x, xx string) string {
+			return `
+
+    ######` + e + `.` + x + `
+
+    RewriteCond "%{HTTP:Accept-encoding}" "` + xx + `"
+    RewriteCond "%{REQUEST_FILENAME}\.` + x + `" -s
+    RewriteRule "^(.*)` + e + `" "$1` + e + `\.` + x + `" [QSA]
+
+    RewriteRule "` + e + `\.` + x + `$" "-" [E=no-gzip:1,E=no-brotli]
+
+    <FilesMatch "` + e + `\.` + x + `$">
+      ForceType '` + c + `; charset=UTF-8'
+      Header append Content-Encoding ` + xx + `
+      Header append Vary Accept-Encoding
+    </FilesMatch>
+
+`
+		}
+
+		for ext, contentType := range Config.Compress.Extensions {
+			rExt := regexp.QuoteMeta(ext)
+			if brotliSupported && Config.Compress.Brotli {
+				configStr += rewriteMacro(rExt, contentType, "br", "br")
+			}
+			if Config.Compress.GZIP {
+				configStr += rewriteMacro(rExt, contentType, "gz", "gzip")
+			}
+		}
+
+		configStr += `
+</IfModule>
+`
+
+		if configStr != "" {
+			htaccessFile := Config.Directories.Output + "/.htaccess"
+			helper.Log.Noticef("writing webserver config to: %s", htaccessFile)
+			err := ioutil.WriteFile(htaccessFile, []byte(configStr), 0644)
+			if err != nil {
+				helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
+			}
+		}
+	}
+}

pkg/mark2web/navigation.go

@@ -0,0 +1,106 @@
+package mark2web
+
+import (
+	"path"
+	"regexp"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+)
+
+// NavElement is one element with ist attributes and subs
+type NavElement struct {
+	Navname string
+	GoTo    string
+	Active  bool
+
+	ColMap helper.MapString
+
+	Data interface{}
+
+	This ThisPathConfig
+
+	SubMap   *map[string]*NavElement
+	SubSlice *[]*NavElement
+}
+
+// buildNavigation builds the navigation trees for use in templates
+func (node *TreeNode) buildNavigation(curNavMap *map[string]*NavElement, curNavSlice *[]*NavElement, navActive *[]*NavElement) {
+	buildNavigationRecursive(node.root, curNavMap, curNavSlice, navActive, node.CurrentNavPath(), node.BackToRootPath())
+}
+
+func buildNavigationRecursive(tree *TreeNode, curNavMap *map[string]*NavElement, curNavSlice *[]*NavElement, navActive *[]*NavElement, activeNav string, backToRoot string) {
+	for _, el := range tree.Sub {
+		if el.Hidden {
+			continue // ignore hidden nav points from collections
+		}
+
+		var ignNav *string
+		if p := el.Config.Path; p != nil {
+			ignNav = p.IgnoreForNav
+		}
+		if ignNav != nil && *ignNav != "" {
+			regex, err := regexp.Compile(*ignNav)
+			if err != nil {
+				helper.Log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err)
+			}
+			if regex.MatchString(path.Base(el.InputPath)) {
+				helper.Log.Debugf("ignoring input directory '%s' in navigation", el.InputPath)
+				continue
+			}
+		}
+
+		elPath := strings.TrimPrefix(el.OutputPath, Config.Directories.Output+"/")
+
+		subMap := make(map[string]*NavElement)
+		subSlice := make([]*NavElement, 0)
+		navEl := NavElement{
+			Active:   strings.HasPrefix(activeNav, elPath),
+			Data:     el.Config.Data,
+			ColMap:   el.ColMap,
+			SubMap:   &subMap,
+			SubSlice: &subSlice,
+		}
+
+		navEl.This = el.Config.This
+
+		if navEl.Active {
+			// add to navActive level navigation
+			currentLevel := strings.Count(activeNav, "/")
+			if len(*navActive) <= currentLevel {
+				// not registered
+				*navActive = append(*navActive, &navEl)
+			}
+		}
+
+		n := el.Config.This.Navname
+		if n != nil {
+			navEl.Navname = *n
+		}
+		g := el.Config.This.GoTo
+		if g != nil {
+			if strings.HasPrefix(*g, "/") {
+				// abslute
+				navEl.GoTo = *g
+			} else {
+				// relative
+				navEl.GoTo = elPath + "/" + *g
+			}
+		} else {
+			navEl.GoTo = elPath + "/"
+		}
+
+		if activeNav != "" && activeNav != "/" {
+			// calculate relative path
+			navEl.GoTo = backToRoot + navEl.GoTo
+			navEl.GoTo = path.Clean(navEl.GoTo)
+		}
+
+		(*curNavMap)[path.Base(el.OutputPath)] = &navEl
+		if curNavSlice != nil {
+			*curNavSlice = append(*curNavSlice, &navEl)
+		}
+
+		buildNavigationRecursive(el, &subMap, &subSlice, navActive, activeNav, backToRoot)
+	}
+}

pkg/mark2web/no_cgo.go

@@ -0,0 +1,14 @@
+// +build !cgo
+
+package mark2web
+
+import "gitbase.de/apairon/mark2web/pkg/helper"
+
+var brotliSupported = false
+
+func init() {
+	helper.Log.Warning("cgo is disabled, so brotli compression is not supported")
+}
+
+func handleBrotliCompression(filename string, content []byte) {
+}

pkg/mark2web/path.go

@@ -0,0 +1,60 @@
+package mark2web
+
+import (
+	"path"
+	"strings"
+)
+
+// ResolveNavPath fixes nav target relative to current navigation path
+func (node *TreeNode) ResolveNavPath(target string) string {
+	if strings.HasPrefix(target, "/") {
+		target = node.BackToRootPath() + target
+	}
+	target = path.Clean(target)
+	return target
+}
+
+// ResolveOutputPath fixes output directory relative to current navigation path
+func (node *TreeNode) ResolveOutputPath(target string) string {
+	if strings.HasPrefix(target, "/") {
+		target = Config.Directories.Output + "/" + target
+	} else {
+		target = node.OutputPath + "/" + target
+	}
+	return path.Clean(target)
+}
+
+// ResolveInputPath fixes input directory relative to current navigation path
+func (node *TreeNode) ResolveInputPath(target string) string {
+	if strings.HasPrefix(target, "/") {
+		target = Config.Directories.Input + "/" + target
+	} else {
+		target = node.InputPath + "/" + target
+	}
+	return path.Clean(target)
+}
+
+// CurrentNavPath is current navigation path for this node
+func (node *TreeNode) CurrentNavPath() string {
+	curNavPath := strings.TrimPrefix(node.OutputPath, Config.Directories.Output)
+	curNavPath = strings.TrimPrefix(curNavPath, "/")
+	curNavPath = path.Clean(curNavPath)
+	if curNavPath == "." {
+		curNavPath = ""
+	}
+
+	return curNavPath
+}
+
+// BackToRootPath builds ../../ string
+func (node *TreeNode) BackToRootPath() string {
+	curNavPath := node.CurrentNavPath()
+
+	tmpPath := ""
+	if curNavPath != "" {
+		for i := strings.Count(curNavPath, "/") + 1; i > 0; i-- {
+			tmpPath += "../"
+		}
+	}
+	return tmpPath
+}

pkg/mark2web/render.go

@@ -0,0 +1,33 @@
+package mark2web
+
+import (
+	"log"
+
+	"github.com/flosch/pongo2"
+)
+
+var templateCache = make(map[string]*pongo2.Template)
+var templateDir string
+
+// SetTemplateDir sets base directory for searching template files
+func SetTemplateDir(dir string) {
+	templateDir = dir
+}
+
+// renderTemplate renders a pongo2 template with context
+func renderTemplate(filename string, node *TreeNode, pathConfig *PathConfig, ctx *pongo2.Context) (string, error) {
+	CurrentContext = ctx
+	CurrentTreeNode = node
+	templateFile := templateDir + "/" + filename
+	template := templateCache[templateFile]
+	if template == nil {
+		var err error
+		if template, err = pongo2.FromFile(templateFile); err != nil {
+			log.Panicf("could not parse template '%s': %s", templateFile, err)
+		} else {
+			templateCache[templateFile] = template
+		}
+	}
+
+	return template.Execute(*ctx)
+}

pkg/mark2web/run.go

@@ -0,0 +1,16 @@
+package mark2web
+
+// Run will do a complete run of mark2web
+func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
+	SetTemplateDir(inDir + "/templates")
+
+	tree := new(TreeNode)
+	tree.ReadContentDir(inDir+"/content", outDir, "", defaultPathConfig)
+	tree.ProcessContent()
+
+	ProcessAssets()
+
+	tree.WriteWebserverConfig()
+
+	Wait()
+}

pkg/mark2web/tree.go

@@ -0,0 +1,21 @@
+package mark2web
+
+import "gitbase.de/apairon/mark2web/pkg/helper"
+
+// TreeNode is complete config tree of content dir
+type TreeNode struct {
+	InputPath  string
+	OutputPath string
+	Hidden     bool // for collections which are not part of the navigation
+
+	ColMap helper.MapString
+
+	InputFiles []string
+	OtherFiles []string
+
+	Config *PathConfig
+	Sub    []*TreeNode
+
+	root   *TreeNode // shows always to root of tree
+	parent *TreeNode
+}

pkg/mark2web/wait.go

@@ -0,0 +1,60 @@
+package mark2web
+
+import (
+	"runtime"
+	"sync"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+)
+
+var wg sync.WaitGroup
+var numCPU = runtime.NumCPU()
+var curNumThreads = 1 // main thread is 1
+
+func init() {
+	helper.Log.Infof("number of CPU core: %d", numCPU)
+}
+
+// Wait will wait for all our internal go threads
+func Wait() {
+	wg.Wait()
+}
+
+// ThreadSetup adds 1 to wait group
+func ThreadSetup() {
+	curNumThreads++
+	wg.Add(1)
+}
+
+// ThreadDone removes 1 from wait group
+func ThreadDone() {
+	curNumThreads--
+	wg.Done()
+}
+
+// ThreadStart will start a thread an manages the wait group
+func ThreadStart(f func(), forceNewThread ...bool) {
+	force := false
+	if len(forceNewThread) > 0 && forceNewThread[0] {
+		force = true
+	}
+
+	if numCPU > curNumThreads || force {
+		// only new thread if empty CPU core available or forced
+		threadF := func() {
+			f()
+			ThreadDone()
+		}
+
+		ThreadSetup()
+		go threadF()
+	} else {
+		helper.Log.Debugf("no more CPU core (%d used), staying in main thread", curNumThreads)
+		f()
+	}
+}
+
+// SetNumCPU is for testing package without threading
+func SetNumCPU(i int) {
+	numCPU = i
+}

scripts/build.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+curdir=$(dirname $0)
+distdir="$curdir/../dist"
+
+mkdir -p "$distdir"
+go build -v -ldflags "-X main.Version=`git describe --tags --long` -X main.GitHash=`git rev-parse HEAD` -X main.BuildTime=`date -u '+%Y-%m-%d_%I:%M:%S%p'`" -o "$distdir/mark2web-`cat $curdir/../build/VERSION`-${GOOS}-${GOARCH}${FILEEXT}" "$curdir/../cmd/mark2web/main.go"

test/test.rest

@@ -0,0 +1,33 @@
+GET https://mark2web.basiscms.de/api/collections/get/mark2webBlog
+  ?sort[date]=-1
+  &limit=101
+  &token=985cee34099f4d3b08f18fc22f6296
+  &filter[link][$exists]=0
+
+###
+
+  &filter[link._id]=5c76a0f4643334fe0400039c
+  &filter[published]=true
+
+###
+
+GET https://mark2web.basiscms.de/api/imagestyles/style/klein
+  ?src=/vhosts/mark2web.basiscms.de/uploads/2019/02/27/5c767a7f3dec9computer-3368242_1920.jpg
+  &output=1
+  &token=89ff216524093123bf7a0a10f7b273
+
+###
+
+GET https://mark2web.basiscms.de/api/imagestyles/style
+
+###
+
+GET https://ci.basehosts.de/api/repos/apairon/mark2web/builds?page=1
+Authorization: Bearer { ci_token }
+
+###
+
+POST https://ci.basehosts.de/api/repos/apairon/mark2web/builds/63/promote?target=website
+Authorization: Bearer { ci_token }
+
+###

website/config.yml

@@ -2,6 +2,7 @@ Webserver:
   Type: "apache" # generates .htaccess
 
 Assets:
+  Compress: True
   FromPath: "project-files"
   ToPath: "project-files"
   Action: "copy" # symlink, copy or move
@@ -12,3 +13,10 @@ Assets:
 OtherFiles:
   Action: "copy"
 
+Compress:
+  Brotli: True
+  GZIP: True
+  Extensions:
+    .html: text/html
+    .css: text/css
+    .js: text/javascript

website/content/config.yml

@@ -9,3 +9,8 @@ Meta:
 Markdown:
   ChromaRenderer: True
   ChromaStyle: monokai
+
+Data:
+  debug: False
+  matomoSiteId: 89
+  token: 985cee34099f4d3b08f18fc22f6296 # cockpit api token

website/content/de/01_Navigation/01_mark2web/README.md

@@ -2,11 +2,19 @@
 Markdown:
   ChromaRenderer: False
 
+Data:
+  background: /img/coffee.jpg
+
+  slider:
+    - img: /img/coffee.jpg
+      alt: 
+      opacity: 1
+
 ---
 
 # mark2web
 
-mark2web ist ein Generator, der aus Markdown- und Konfig-Dateien in einer Ordnerstruktur eine statische Website unter Zuhilfenahme von Templates generiert.  
+mark2web ist ein Generator, der aus Markdown- und Konfig-Dateien in einer Ordnerstruktur eine statische Website unter Zuhilfenahme von Templates generiert.
 
 ```mermaid
 graph TD
@@ -28,25 +36,35 @@ graph TD
     classDef out stroke-width:5px,stroke:#b5c50f,fill:#ccc
     class M,C,D,A in
     class W out
+
+    click C "../benutzung/konfiguration" "Doku: Benutzung/Konfiguration"
+    click M "../benutzung/inhalte" "Doku: Benutzung/Inhalte"
+    click A "../benutzung/inhalte" "Doku: Benutzung/Inhalte"
+    click D "../benutzung/templates" "Doku: Benutzung/Templates"
 ```
 
 <script defer>
 	window.onload = function() {
 		mermaid.init(undefined,$("code.language-mermaid"));
+    $("code.language-mermaid").css("visibility", "visible");
 	};
 </script>
 
 ---
-  
-Der Generator selbst wurde in [Go](https://golang.org/) geschrieben. Es wurden dabei eine Vielzahl existierender Packages verwendet. Unter Anderem:
+
+Der Generator selbst wurde in [Go](https://golang.org/) geschrieben. Es wurden dabei eine Vielzahl existierender Packages verwendet.  
+
+Unter Anderem:
 
 - der Markdown-Parser [blackfriday](https://github.com/russross/blackfriday)
-- die Template-Sprache "Django Template Language" über das Paket [pongo2](github.com/flosch/pongo2)
-- das Logging-Paket [go-logging](github.com/op/go-logging)
+- die Template-Sprache "Django Template Language" über das Paket [pongo2](https://github.com/flosch/pongo2)
+- das Logging-Paket [go-logging](https://github.com/op/go-logging)
 - der YAML-Parser [go-yaml](https://github.com/go-yaml/yaml)
+- die Imaging Bibliothek [disintegration/imaging](github.com/disintegration/imaging)
+- der Javascript-Interpreter [otto](github.com/robertkrimen/otto) mit der Erweiterung [motto](github.com/ddliu/motto)
 
 Weitere Pakete, die verwendet wurden finden Sie in den Quellen.
 
 Diese Website wurde selbst mit mark2web generiert. Der entsprechende Quellcode, sowie die Quellen zu mark2web finden Sie unter:  
 
-**https://gitbase.de/apairon/mark2web**
+**[https://gitbase.de/apairon/mark2web](5c76a0f4643334fe0400039c)**

website/content/de/01_Navigation/02_Installation/README.md

@@ -1,20 +1,25 @@
+---
+Data:
+  background: /img/laptop.jpg
+
+  slider:
+    - img: /img/laptop.jpg
+      alt: 
+      opacity: 1
+
+---
 # Installation
 
-Damit die korrekten Versionsinformationen dynamisch in das finale mark2web-Binary eingefügt wurde, ist eine manuelle Installation aus dem Git-Repository sinnvoll.  
-Da die benötigten Pakete über die Go "vendor"-Funktionalität eingebunden sind ist ein `git submodule --init --recursive` nötig, wie im folgenden Abschnitt zu sehen ist:
-
 ```sh
-mkdir -p $GOPATH/src/gitbase.de/apairon
-git clone https://gitbase.de/apairon/mark2web.git $GOPATH/src/gitbase.de/apairon/mark2web
+go get -v gitbase.de/apairon/mark2web/cmd/mark2web
 
-cd $GOPATH/src/gitbase.de/apairon/mark2web
-git submodule update --init --recursive
-
-./build.sh
+# setze Versioninformationen ins Binary
+pkg=$GOPATH/src/gitbase.de/apairon/mark2web
+go install -v -ldflags "-X main.Version=`cat $pkg/build/VERSION` -X main.GitHash=`git --git-dir $pkg/.git rev-parse HEAD` -X main.BuildTime=`date -u '+%Y-%m-%d_%I:%M:%S%p'`" gitbase.de/apairon/mark2web/cmd/mark2web
 ```
 
 ---
 
-Eine Installation über `go install gitbase.de/apairon/mark2web` wird derzeit noch nicht unterstützt, da dabei die Informationen für `mark2web -version` nicht generiert werden.  
-  
-Später folgen vorkomplierte Releases über die Repository-Website.
+## Releases
+
+Vorkompilierte Binaries finden Sie auf der [Releases-Seite auf gitbase.de](https://gitbase.de/apairon/mark2web/releases).

website/content/de/01_Navigation/03_Benutzung/02_Konfiguration/README.md

@@ -1,139 +0,0 @@
-# Konfiguration
-
-Die Konfigurationsdatein sind im YAML-Format gehalten (siehe: [Wikipedia](https://de.wikipedia.org/wiki/YAML)).
-
-## globale Einstellungen
-
-Die obersten Verzeichnis sich befindende Datei `config.yml` kann z.B. folgenden Inhalt haben:
-
-```yaml
-Webserver:
-  Type: "apache"
-
-Assets:
-  FromPath: "assets"
-  ToPath: "assets"
-  Action: "copy"
-  FixTemplate:
-    Find: "\\.\\./assets/"
-    Replace: ""
-
-OtherFiles:
-  Action: "copy"
-```
-
-### Sektion `Webserver:`
-
-#### `Type:`
-
-Derzeit wird hier nur der Wert `apache` unterstützt. Ist dieser Wert gesetzt werden automatische .htaccess-Dateien mit mod_rewrite-Anweisungen generiert, die eine saubere Weiterleitung bei entsprechenden Konfig-Anweisungen im `content`-Verzeichnis ermöglichen.
-
-### Sektion `Assets:`
-
-`Assets:` steuert, wie mit Bild/JS/CSS Dateien umgegangen werden soll.
-
-#### `FromPath:`
-
-Lage des Asset-Verzeichnis unterhalb des `content`-Verzeichnis
-
-#### `ToPath:`
-
-Zielverzeichnis im Ausgabe-Verzeichnis der fertig generierten Website
-
-#### `Action:`
-
-Derzeit nur `copy`, also das Kopieren der Dateien und Unterordner ins Zielverzeichnis
-
-#### `FixTemplate:`
-
-Wenn hier `Find:` (regulärer Ausdruck) und `Replace:` (Ersetzung) angeben sind, werden die gefundenden Pfadangaben in der generierten HTML-Dateien durch den korrekten relativen Pfad zum Asset-Verzeichnis ersetzt.
-
-### Sektion `OtherFiles:`
-
-`OtherFiles:` definiert, wie mit anderen Dateien innerhalb des `content`-Verzeichnis umgegangen werden soll.
-
-#### `Action:`
-
-Derzeit nur `copy`, also das Kopieren der Dateien in das entsprechende Unterverzeichnis im Ausgabe-Verzeichnis
-
----
-
-## Konfiguration im `content`-Verzeichnis
-
-Im `content`-Verzeichnis, sowie in jedem Unterverzeichnis unterhalb von `content` kann sich eine `config.yml`-Datei befinden, wie aus folgendem Beispiel:
-
-```yaml
-This:
-  GoTo: "/de/service/impressum/"
-  Navname: "Impressumsverweis"
-
-```
-
-oder
-
-```yaml
-This:
-  Navname: "FAQ's"
-  Data:
-    slogan: "Wer nicht fragt, bekommt keine Antwort."
-
-Template: "base.html"
-
-Index:
-    InputFile: "README.md"
-    OutputFile: "index.html"
-
-Meta:
-  Title: "Fragen und Antworten"
-  Description: "Dies ist die Fragen und Antworten Unterseite."
-  Keywords: "FAQ, Fragen, Antworten"
-
-Data:
-  background: "bg.jpg"
-  slider:
-    - img: "assets/img/slider1.jpg"
-      alt: "Alternativtext 1"
-    - img: "assets/img/slider2.jpg"
-      alt: "Alternativtext 2"
-    - img: "assets/img/slider3.jpg"
-      alt: "Alternativtext 3"
-
-```
-
-### `This:` Sektion
-
-Sämtlich Werte unterhalb dieser Sektion gelten nur für den Inhalt, bzw. Navigationspunkt in dessen Ordner sich die `config.yml` befindet. Die Werte werden nicht an Unterordner wertervererbt.
-
-#### `GoTo:`
-
-Falls der Navigationspunkt selbst keinen Inhalt darstellen soll, sondern nur weiterleiten soll, so wird hier das Weiterleitungsziel eingegeben.  
-Das Ziel ist der absolute (startend mit `/`) oder relative Pfad zum Zielnavigationspunkt.  
-Die Schreibweise des Pfades ist so zu verwenden, wie der Pfad nach Umschreibung und Säuberung des Pfades im Zielverzeichnis dargestellt wird.  
-Aus `de/mainnav/03_Fragen und Antworten` wird also z.B. `de/mainnav/fragen-und-antworten`.
-
-#### `Navname:`
-
-Dieser Wert überschreibt den aus dem Ordnernamen automatisch abgeleiteten Navigationspunkt-Namen. Dies ist zum Beispiel dann nützlich, wenn Sonderzeichen im Verzeichnisnamen nicht vorkommen sollen, aber im Namen des Navigationspunkts gebraucht werden.
-
-#### `Data:`
-
-Unterhalb von `Data:` können beliebige Datenstrukturen erfasst werden. Da diese Struktur unterhalb von `This:` angeordnet ist, werden auch die Daten nicht weiter an Unterordner vererbt.  
-Hier können z.B. Informationen zum Navigationspunkt abgelegt werden, die im Template Zusatzinformationen darstellen (z.B. ein Slogan zu einem Navigationspunkt).
-
-### `Meta:` Sektion
-
-Unter `Title:`, `Description:` und `Keywords:` werden die typischen Metaangaben abgelegt, die im
-
-```html
-<head>
-    ...
-</head>
-```
-
-übllicherweise Verwendung finden. Die entsprechenden Platzhalter stehen im Template zur Verfügung.  
-  
-`Meta:` vererbt seine individuellen Informationen an die Unterordner weiter, sofern diese dort nicht selbst in einer `config.yml` oder im Kopf der Markdown-Datei definiert sind.
-
-### `Data:` Sektion
-
-`Data:` an dieser Stelle kann, wie auch `Data:` unterhalb von `This:`, beliebige Daten aufnehmen. Die Daten hier allerdings werden an Unterordner weitervererbt, sofern diese nicht dort oder in der Markdown-Datei selbst festegelegt überschrieben wurden.

website/content/de/01_Navigation/03_Benutzung/02_Konfiguration/config.yml

@@ -1,3 +0,0 @@
-This:
-  Data:
-    teaser: Globale Konfiguration und individuelle Content-Einstellungen

website/content/de/01_Navigation/03_Benutzung/config.yml

@@ -1,2 +0,0 @@
-This:
-  GoTo: ordnerstruktur

website/content/de/01_Navigation/03_Dokumentation/01_Ordnerstruktur/README.md

@@ -1,3 +1,14 @@
+---
+Data:
+  background: /img/folder.jpg
+
+  slider:
+    - img: /img/folder.jpg
+      alt: 
+      opacity: 1
+
+---
+
 # Benutzung
 
 ## Ordnerstruktur
@@ -38,6 +49,8 @@ DIR assets (kann auch abweichend benannt werden)
     DIR css
 
 DIR templates
+    DIR filters
+        FILE myfilter.js
     FIL base.html
     FIL base_sub.html
 

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/README.md

@@ -0,0 +1,16 @@
+---
+Template: base_doc.html
+
+Data:
+  background: /img/wire.jpg
+
+  slider:
+    - img: /img/wire.jpg
+      alt: 
+      opacity: 1
+
+---
+
+# Konfiguration
+
+Die Konfigurationsdatein sind im YAML-Format gehalten (siehe: [Wikipedia](https://de.wikipedia.org/wiki/YAML)).

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01000_globale Einstellungen.md

@@ -0,0 +1,17 @@
+Die obersten Verzeichnis sich befindende Datei `config.yml` kann z.B. folgenden Inhalt haben:
+
+```yaml
+Webserver:
+  Type: "apache"
+
+Assets:
+  FromPath: "assets"
+  ToPath: "assets"
+  Action: "copy"
+  FixTemplate:
+    Find: "\\.\\./assets/"
+    Replace: ""
+
+OtherFiles:
+  Action: "copy"
+```

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01100__Sektion Webserver.md

@@ -0,0 +1,5 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01110___Type.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Derzeit wird hier nur der Wert `apache` unterstützt. Ist dieser Wert gesetzt werden automatische .htaccess-Dateien mit mod_rewrite-Anweisungen generiert, die eine saubere Weiterleitung bei entsprechenden Konfig-Anweisungen im `content`-Verzeichnis ermöglichen.

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01200__Sektion Assets.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+`Assets:` steuert, wie mit Bild/JS/CSS Dateien umgegangen werden soll.

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01210___FromPath.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Lage des Asset-Verzeichnis unterhalb des `content`-Verzeichnis

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01220___ToPath.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Zielverzeichnis im Ausgabe-Verzeichnis der fertig generierten Website

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01230___Action.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Derzeit nur `copy`, also das Kopieren der Dateien und Unterordner ins Zielverzeichnis

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01240___FixTemplates.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Wenn hier `Find:` (regulärer Ausdruck) und `Replace:` (Ersetzung) angeben sind, werden die gefundenden Pfadangaben in der generierten HTML-Dateien durch den korrekten relativen Pfad zum Asset-Verzeichnis ersetzt.

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01300__Sektion OtherFiles.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+`OtherFiles:` definiert, wie mit anderen Dateien innerhalb des `content`-Verzeichnis umgegangen werden soll.

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_01310___Action.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Derzeit nur `copy`, also das Kopieren der Dateien in das entsprechende Unterverzeichnis im Ausgabe-Verzeichnis

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_02000_Konfiguration im Content-Verzeichnis.md

@@ -0,0 +1,45 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Im `content`-Verzeichnis, sowie in jedem Unterverzeichnis unterhalb von `content` kann sich eine `config.yml`-Datei befinden, wie aus folgendem Beispiel:
+
+```yaml
+This:
+  GoTo: "/de/service/impressum/"
+  Navname: "Impressumsverweis"
+
+```
+
+oder
+
+```yaml
+This:
+  Navname: "FAQ's"
+  Data:
+    slogan: "Wer nicht fragt, bekommt keine Antwort."
+
+Template: "base.html"
+
+Index:
+    InputFile: "README.md"
+    OutputFile: "index.html"
+
+Meta:
+  Title: "Fragen und Antworten"
+  Description: "Dies ist die Fragen und Antworten Unterseite."
+  Keywords: "FAQ, Fragen, Antworten"
+
+Data:
+  background: "bg.jpg"
+  slider:
+    - img: "assets/img/slider1.jpg"
+      alt: "Alternativtext 1"
+    - img: "assets/img/slider2.jpg"
+      alt: "Alternativtext 2"
+    - img: "assets/img/slider3.jpg"
+      alt: "Alternativtext 3"
+
+```

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_02100__Sektion This.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Sämtlich Werte unterhalb dieser Sektion gelten nur für den Inhalt, bzw. Navigationspunkt in dessen Ordner sich die `config.yml` befindet. Die Werte werden nicht an Unterordner wertervererbt.

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_02110___GoTo.md

@@ -0,0 +1,10 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Falls der Navigationspunkt selbst keinen Inhalt darstellen soll, sondern nur weiterleiten soll, so wird hier das Weiterleitungsziel eingegeben.  
+Das Ziel ist der absolute (startend mit `/`) oder relative Pfad zum Zielnavigationspunkt.  
+Die Schreibweise des Pfades ist so zu verwenden, wie der Pfad nach Umschreibung und Säuberung des Pfades im Zielverzeichnis dargestellt wird.  
+Aus `de/mainnav/03_Fragen und Antworten` wird also z.B. `de/mainnav/fragen-und-antworten`.

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_02120___Navname.md

@@ -0,0 +1,7 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Dieser Wert überschreibt den aus dem Ordnernamen automatisch abgeleiteten Navigationspunkt-Namen. Dies ist zum Beispiel dann nützlich, wenn Sonderzeichen im Verzeichnisnamen nicht vorkommen sollen, aber im Namen des Navigationspunkts gebraucht werden.

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_02130___Data.md

@@ -0,0 +1,8 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Unterhalb von `Data:` können beliebige Datenstrukturen erfasst werden. Da diese Struktur unterhalb von `This:` angeordnet ist, werden auch die Daten nicht weiter an Unterordner vererbt.  
+Hier können z.B. Informationen zum Navigationspunkt abgelegt werden, die im Template Zusatzinformationen darstellen (z.B. ein Slogan zu einem Navigationspunkt).

website/content/de/01_Navigation/03_Dokumentation/02_Konfiguration/_02200__Sektion Meta.md

@@ -0,0 +1,17 @@
+---
+Data:
+    Version: "ab v1.0"
+
+---
+
+Unter `Title:`, `Description:` und `Keywords:` werden die typischen Metaangaben abgelegt, die im
+
+```html
+<head>
+    ...
+</head>
+```
+
+übllicherweise Verwendung finden. Die entsprechenden Platzhalter stehen im Template zur Verfügung.  
+  
+`Meta:` vererbt seine individuellen Informationen an die Unterordner weiter, sofern diese dort nicht selbst in einer `config.yml` oder im Kopf der Markdown-Datei definiert sind.