reorganized code

assets.go

@@ -1,9 +1,10 @@
-package helper
+package mark2web
 
 import (
 	"strings"
 
 	"gitbase.de/apairon/mark2web/config"
+	"gitbase.de/apairon/mark2web/helper"
 	cpy "github.com/otiai10/copy"
 )
 
@@ -19,10 +20,10 @@ func ProcessAssets() {
 		if !strings.HasPrefix(to, "/") {
 			to = config.Config.Directories.Output + "/" + to
 		}
-		Log.Noticef("copying assets from '%s' to '%s'", from, to)
+		helper.Log.Noticef("copying assets from '%s' to '%s'", from, to)
 		err := cpy.Copy(from, to)
 		if err != nil {
-			Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
+			helper.Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
 		}
 	}
 }

build.sh

@@ -1,4 +1,5 @@
 #!/bin/sh
 
 mkdir -p dist
-go build -v -ldflags "-X main.Version=`git describe --tags --long` -X main.GitHash=`git rev-parse HEAD` -X main.BuildTime=`date -u '+%Y-%m-%d_%I:%M:%S%p'`" -o dist/mark2web-`cat VERSION`-${GOOS}-${GOARCH}${FILEEXT}
+go build -v -ldflags "-X main.Version=`git describe --tags --long` -X main.GitHash=`git rev-parse HEAD` -X main.BuildTime=`date -u '+%Y-%m-%d_%I:%M:%S%p'`" -o dist/mark2web-`cat VERSION`-${GOOS}-${GOARCH}${FILEEXT} mark2web/main.go
+

config/tree.go

@@ -2,9 +2,6 @@ package config
 
 import (
 	"fmt"
-	"reflect"
-
-	"github.com/imdario/mergo"
 )
 
 // MapString is a map[string]interface{} which always unmarsahls yaml to map[string]interface{}
@@ -149,24 +146,3 @@ type PathConfigTree struct {
 	Config *PathConfig
 	Sub    []*PathConfigTree
 }
-
-type ptrTransformer struct{}
-
-func (t ptrTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
-	if typ.Kind() == reflect.Ptr {
-		return func(dst, src reflect.Value) error {
-			if dst.CanSet() {
-				if dst.IsNil() {
-					dst.Set(src)
-				}
-			}
-			return nil
-		}
-	}
-	return nil
-}
-
-// Merge merges 2 objects or maps
-func Merge(dst, src interface{}) error {
-	return mergo.Merge(dst, src, mergo.WithTransformers(ptrTransformer{}))
-}

content.go

@@ -0,0 +1,296 @@
+package mark2web
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path"
+	"regexp"
+	"strings"
+
+	"gitbase.de/apairon/mark2web/config"
+	"gitbase.de/apairon/mark2web/context"
+	"gitbase.de/apairon/mark2web/helper"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/flosch/pongo2"
+	cpy "github.com/otiai10/copy"
+	"gopkg.in/yaml.v2"
+)
+
+// ReadContentDir walks through content directory and builds the tree of configurations
+func ReadContentDir(inBase string, outBase string, dir string, conf *config.PathConfig, tree *config.PathConfigTree) {
+	context.FillNodeConfig(tree, inBase, outBase, dir, conf)
+
+	files, err := ioutil.ReadDir(tree.InputPath)
+	if err != nil {
+		helper.Log.Panic(err)
+	}
+
+	// first only files
+	for _, f := range files {
+		p := tree.InputPath + "/" + f.Name()
+		if !f.IsDir() && f.Name() != "config.yml" {
+			switch path.Ext(f.Name()) {
+			case ".md":
+				helper.Log.Debugf(".MD %s", p)
+				if tree.InputFiles == nil {
+					tree.InputFiles = make([]string, 0)
+				}
+				tree.InputFiles = append(tree.InputFiles, f.Name())
+				break
+			default:
+				helper.Log.Debugf("FIL %s", p)
+				if tree.OtherFiles == nil {
+					tree.OtherFiles = make([]string, 0)
+				}
+				tree.OtherFiles = append(tree.OtherFiles, f.Name())
+			}
+		}
+	}
+
+	// only directorys, needed config before
+	for _, f := range files {
+		p := tree.InputPath + "/" + f.Name()
+		if f.IsDir() {
+			helper.Log.Debugf("DIR %s", p)
+			newTree := new(config.PathConfigTree)
+			if tree.Sub == nil {
+				tree.Sub = make([]*config.PathConfigTree, 0)
+			}
+			tree.Sub = append(tree.Sub, newTree)
+			ReadContentDir(tree.InputPath, tree.OutputPath, f.Name(), tree.Config, newTree)
+		}
+	}
+}
+
+// ProcessContent walks recursivly through the input paths and processes all files for output
+func ProcessContent(rootConf, conf *config.PathConfigTree) {
+	helper.CreateDirectory(conf.OutputPath)
+
+	curNavPath := strings.TrimPrefix(conf.OutputPath, config.Config.Directories.Output)
+	curNavPath = strings.TrimPrefix(curNavPath, "/")
+	curNavPath = path.Clean(curNavPath)
+	if curNavPath == "." {
+		curNavPath = ""
+	}
+
+	goTo := conf.Config.This.GoTo
+	if goTo != nil && *goTo != "" {
+		goToFixed := *goTo
+		if strings.HasPrefix(goToFixed, "/") {
+			goToFixed = helper.BackToRoot(curNavPath) + goToFixed
+		}
+		goToFixed = path.Clean(goToFixed)
+
+		switch config.Config.Webserver.Type {
+		case "apache":
+			htaccessFile := conf.OutputPath + "/.htaccess"
+			helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed)
+			err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
+RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
+`), 0644)
+			if err != nil {
+				helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
+			}
+			break
+		}
+	}
+
+	for _, file := range conf.InputFiles {
+		var input []byte
+		inFile := "InputString"
+
+		if file != "" {
+			inFile = conf.InputPath + "/" + file
+			helper.Log.Debugf("reading file: %s", inFile)
+
+			var err error
+			input, err = ioutil.ReadFile(inFile)
+			if err != nil {
+				helper.Log.Panicf("could not read '%s':%s", inFile, err)
+			}
+			helper.Log.Infof("processing input file '%s'", inFile)
+		} else {
+			// use input string if available and input filename == ""
+			var inputString *string
+			if i := conf.Config.Index; i != nil {
+				inputString = i.InputString
+			}
+			if inputString != nil {
+				helper.Log.Debugf("using input string instead of file")
+				input = []byte(*inputString)
+			}
+		}
+
+		newConfig := new(config.PathConfig)
+
+		regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?")
+		yamlData := regex.Find(input)
+		if string(yamlData) != "" {
+			// replace tabs
+			yamlData = bytes.Replace(yamlData, []byte("\t"), []byte("    "), -1)
+
+			helper.Log.Debugf("found yaml header in '%s', merging config", inFile)
+			err := yaml.Unmarshal(yamlData, newConfig)
+			if err != nil {
+				helper.Log.Panicf("could not parse YAML header from '%s': %s", inFile, err)
+			}
+
+			helper.Log.Debug("merging config with upper config")
+			oldThis := newConfig.This
+			helper.Merge(newConfig, conf.Config)
+			newConfig.This = oldThis
+
+			helper.Log.Debug(spew.Sdump(newConfig))
+
+			input = regex.ReplaceAll(input, []byte(""))
+		} else {
+			helper.Merge(newConfig, conf.Config)
+		}
+
+		// ignore ???
+		ignoreFile := false
+		var ignoreRegex *string
+		var stripRegex *string
+		var outputExt *string
+		if f := newConfig.Filename; f != nil {
+			ignoreRegex = f.Ignore
+			stripRegex = f.Strip
+			outputExt = f.OutputExtension
+		}
+		if ignoreRegex != nil && *ignoreRegex != "" {
+			regex, err := regexp.Compile(*ignoreRegex)
+			if err != nil {
+				helper.Log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
+			}
+			ignoreFile = regex.MatchString(file)
+		}
+
+		if ignoreFile {
+			helper.Log.Infof("ignoring file '%s', because of filename.ignore", inFile)
+		} else {
+
+			// build output filename
+			outputFilename := file
+
+			var indexInputFile *string
+			var indexOutputFile *string
+			if i := newConfig.Index; i != nil {
+				indexInputFile = i.InputFile
+				indexOutputFile = i.OutputFile
+			}
+
+			if indexInputFile != nil &&
+				*indexInputFile == file &&
+				indexOutputFile != nil &&
+				*indexOutputFile != "" {
+				outputFilename = *indexOutputFile
+			} else {
+				if stripRegex != nil && *stripRegex != "" {
+					regex, err := regexp.Compile(*stripRegex)
+					if err != nil {
+						helper.Log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
+					}
+					outputFilename = regex.ReplaceAllString(outputFilename, "$1")
+				}
+				if outputExt != nil && *outputExt != "" {
+					outputFilename += "." + *outputExt
+				}
+			}
+
+			outFile := conf.OutputPath + "/" + outputFilename
+			helper.Log.Debugf("using '%s' as output file", outFile)
+
+			// use --- for splitting document in markdown parts
+			regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?")
+			inputParts := regex.Split(string(input), -1)
+			htmlParts := make([]*pongo2.Value, 0)
+
+			chromaRenderer := false
+			chromaStyle := "monokai"
+			if m := newConfig.Markdown; m != nil {
+				if m.ChromaRenderer != nil && *m.ChromaRenderer {
+					chromaRenderer = true
+				}
+				if m.ChromaStyle != nil && *m.ChromaStyle != "" {
+					chromaStyle = *m.ChromaStyle
+				}
+			}
+			for _, iPart := range inputParts {
+				htmlParts = append(htmlParts,
+					pongo2.AsSafeValue(
+						string(helper.RenderMarkdown([]byte(iPart), chromaRenderer, chromaStyle))))
+			}
+
+			// build navigation
+			navMap := make(map[string]*NavElement)
+			navSlice := make([]*NavElement, 0)
+			navActive := make([]*NavElement, 0)
+			BuildNavigation(rootConf, &navMap, &navSlice, &navActive, curNavPath)
+
+			// read yaml header as data for template
+			ctx := context.NewContext()
+			ctx["This"] = newConfig.This
+			ctx["Meta"] = newConfig.Meta
+			ctx["Data"] = newConfig.Data
+			ctx["ColMap"] = rootConf.ColMap // root as NavMap and NavSlice, for sub go to NavElement.ColMap
+			ctx["NavMap"] = navMap
+			ctx["NavSlice"] = navSlice
+			ctx["NavActive"] = navActive
+			ctx["Body"] = pongo2.AsSafeValue(string(helper.RenderMarkdown(input, chromaRenderer, chromaStyle)))
+			ctx["BodyParts"] = htmlParts
+			ctx["CurrentPath"] = curNavPath
+			// set active nav element
+			if len(navActive) > 0 {
+				ctx["NavElement"] = navActive[len(navActive)-1]
+			} else {
+				// if no active path to content, we are in root dir
+				ctx["NavElement"] = &NavElement{
+					GoTo:     helper.BackToRoot(curNavPath),
+					Active:   true,
+					ColMap:   rootConf.ColMap,
+					Data:     rootConf.Config.Data,
+					This:     rootConf.Config.This,
+					SubMap:   &navMap,
+					SubSlice: &navSlice,
+				}
+			}
+
+			helper.Log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
+			templateFilename := *newConfig.Template
+			result, err := RenderTemplate(*newConfig.Template, conf, newConfig, &ctx)
+			if err != nil {
+				helper.Log.Panicf("could not execute template '%s' for input file '%s': %s", templateFilename, inFile, err)
+			}
+
+			result = FixAssetsPath(result, curNavPath)
+
+			helper.Log.Noticef("writing to output file: %s", outFile)
+			err = ioutil.WriteFile(outFile, []byte(result), 0644)
+			if err != nil {
+				helper.Log.Panicf("could not write to output file '%s': %s", outFile, err)
+			}
+
+			//fmt.Println(string(html))
+		}
+	}
+
+	// process other files, copy...
+	for _, file := range conf.OtherFiles {
+		switch config.Config.OtherFiles.Action {
+		case "copy":
+			from := conf.InputPath + "/" + file
+			to := conf.OutputPath + "/" + file
+			helper.Log.Noticef("copying file from '%s' to '%s'", from, to)
+			err := cpy.Copy(from, to)
+			if err != nil {
+				helper.Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
+			}
+		}
+	}
+
+	i := 0
+	for i < len(conf.Sub) {
+		ProcessContent(rootConf, conf.Sub[i])
+		i++
+	}
+}

context/context.go

@@ -0,0 +1,266 @@
+package context
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"regexp"
+	"strings"
+	"time"
+
+	"gitbase.de/apairon/mark2web/config"
+	"gitbase.de/apairon/mark2web/helper"
+	"github.com/davecgh/go-spew/spew"
+	"github.com/extemporalgenome/slug"
+	"github.com/flosch/pongo2"
+	"gopkg.in/yaml.v2"
+)
+
+var CurrentContext *pongo2.Context
+var CurrentTreeNodeConfig *config.PathConfigTree
+var CurrentPathConfig *config.PathConfig
+
+func NewContext() pongo2.Context {
+	ctx := pongo2.Context{
+		"fnRequest": RequestFn,
+		"fnRender":  RenderFn,
+
+		"AssetsPath": config.Config.Assets.ToPath,
+		"Timestamp":  time.Now().Unix,
+	}
+	CurrentContext = &ctx
+
+	return ctx
+}
+
+func FillNodeConfig(node *config.PathConfigTree, inBase, outBase, dir string, conf *config.PathConfig) {
+	inPath := inBase
+	if dir != "" {
+		inPath += "/" + dir
+	}
+
+	helper.Log.Infof("reading input directory: %s", inPath)
+
+	node.InputPath = inPath
+
+	// read config
+	newConfig := new(config.PathConfig)
+	helper.Log.Debug("looking for config.yml ...")
+	configFile := inPath + "/config.yml"
+	if _, err := os.Stat(configFile); os.IsNotExist(err) {
+		helper.Log.Debug("no config.yml found in this directory, using upper configs")
+		helper.Merge(newConfig, conf)
+		// remove this
+		newConfig.This = config.ThisPathConfig{}
+	} else {
+		helper.Log.Debug("reading config...")
+		data, err := ioutil.ReadFile(configFile)
+		if err != nil {
+			helper.Log.Panicf("could not read file '%s': %s", configFile, err)
+		}
+		err = yaml.Unmarshal(data, newConfig)
+		if err != nil {
+			helper.Log.Panicf("could not parse YAML file '%s': %s", configFile, err)
+		}
+
+		helper.Log.Debug("merging config with upper config")
+		oldThis := newConfig.This
+		helper.Merge(newConfig, conf)
+		newConfig.This = oldThis
+
+		helper.Log.Debug(spew.Sdump(newConfig))
+	}
+
+	node.Config = newConfig
+
+	// calc outDir
+	stripedDir := dir
+	var regexStr *string
+	if newConfig.Path != nil {
+		regexStr = newConfig.Path.Strip
+	}
+	if regexStr != nil && *regexStr != "" {
+		if regex, err := regexp.Compile(*regexStr); err != nil {
+			helper.Log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err)
+		} else {
+			stripedDir = regex.ReplaceAllString(stripedDir, "$1")
+		}
+	}
+
+	if node.Config.This.Navname == nil {
+		navname := strings.Replace(stripedDir, "_", " ", -1)
+		node.Config.This.Navname = &navname
+	}
+
+	stripedDir = slug.Slug(stripedDir)
+	outPath := outBase + "/" + stripedDir
+	outPath = path.Clean(outPath)
+
+	helper.Log.Infof("calculated output directory: %s", outPath)
+	node.OutputPath = outPath
+
+	// handle collections
+	for _, colConfig := range newConfig.This.Collections {
+		if colConfig != nil {
+			if colConfig.Name == nil || *colConfig.Name == "" {
+				helper.Log.Panicf("missing Name in collection config in '%s'", inPath)
+			}
+			if colConfig.URL == nil || *colConfig.URL == "" {
+				helper.Log.Panicf("missing EntriesJSON in collection config in '%s'", inPath)
+			}
+		}
+
+		if node.ColMap == nil {
+			node.ColMap = make(config.MapString)
+		}
+		ctx := NewContext()
+		ctx["This"] = node.Config.This
+		ctx["Data"] = node.Config.Data
+
+		url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
+		if err != nil {
+			helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", inPath, err)
+		}
+
+		colData := helper.JSONWebRequest(url)
+		node.ColMap[*colConfig.Name] = colData
+
+		if navT := colConfig.NavTemplate; navT != nil {
+			var entries []interface{}
+			var ok bool
+			if navT.EntriesAttribute != "" {
+				var colDataMap map[string]interface{}
+				if colDataMap, ok = colData.(map[string]interface{}); ok {
+					entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
+					if !ok {
+						helper.Log.Debug(spew.Sdump(colDataMap))
+						helper.Log.Panicf("invalid json data in [%s] from url '%s' for entries", navT.EntriesAttribute, url)
+					}
+				}
+			} else {
+				entries, ok = colData.([]interface{})
+			}
+			if !ok {
+				helper.Log.Debug(spew.Sdump(colData))
+				helper.Log.Panicf("invalid json data from url '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", url)
+			}
+
+			// build navigation with detail sites
+			for idx, colEl := range entries {
+				ctxE := make(pongo2.Context)
+				err := helper.Merge(&ctxE, ctx)
+				if err != nil {
+					helper.Log.Panicf("could not merge context in '%s': %s", inPath, err)
+				}
+				var jsonCtx map[string]interface{}
+				if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
+					helper.Log.Debug(spew.Sdump(colEl))
+					helper.Log.Panicf("no json object for entry index %d from url '%s'", idx, url)
+				}
+				err = helper.Merge(&ctxE, pongo2.Context(jsonCtx))
+				if err != nil {
+					helper.Log.Panicf("could not merge context in '%s': %s", inPath, err)
+				}
+
+				tpl := ""
+				if navT.Template != "" {
+					tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", inPath, err)
+					}
+				}
+				if tpl == "" {
+					tpl = *newConfig.Template
+				}
+
+				dataKey := ""
+				if navT.DataKey != "" {
+					dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", inPath, err)
+					}
+				}
+
+				goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
+				if err != nil {
+					helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", inPath, err)
+				}
+				goTo = strings.Trim(goTo, "/")
+				goTo = path.Clean(goTo)
+
+				if strings.Contains(goTo, "..") {
+					helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", inPath, goTo)
+				}
+				if goTo == "." {
+					helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", inPath)
+				}
+				if goTo == "" {
+					helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", inPath)
+				}
+
+				navname := ""
+				if navT.Navname != "" {
+					navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", inPath, err)
+					}
+				}
+				body := ""
+				if navT.Body != "" {
+					body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
+					if err != nil {
+						helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", inPath, err)
+					}
+				}
+
+				Add2Nav(node, node.Config, tpl, goTo, navname, colEl, dataKey, body, navT.Hidden)
+			}
+		}
+
+	}
+}
+
+func Add2Nav(currentNode *config.PathConfigTree, pathConfig *config.PathConfig, tplFilename, outDir string, navname string, ctx interface{}, dataMapKey string, body string, hidden bool) {
+	newNodeConfig := new(config.PathConfigTree)
+	FillNodeConfig(
+		newNodeConfig,
+		currentNode.InputPath,
+		currentNode.OutputPath,
+		outDir,
+		pathConfig,
+	)
+	if navname != "" {
+		newNodeConfig.Config.This = config.ThisPathConfig{
+			Navname: &navname,
+		}
+	}
+	if dataMapKey != "" {
+		if newNodeConfig.Config.Data == nil {
+			newNodeConfig.Config.Data = make(config.MapString)
+		}
+		// as submap in Data
+		newNodeConfig.Config.Data[dataMapKey] = ctx
+	} else if m, ok := ctx.(map[string]interface{}); ok {
+		// direct set data
+		newNodeConfig.Config.Data = m
+	}
+
+	// fake via normal file behavior
+	newNodeConfig.Config.Template = &tplFilename
+	newNodeConfig.InputFiles = []string{""} // empty file is special for use InputString
+	indexInFile := ""
+	indexOutFile := "index.html"
+	if idx := newNodeConfig.Config.Index; idx != nil {
+		if idx.OutputFile != nil && *idx.OutputFile != "" {
+			indexOutFile = *idx.OutputFile
+		}
+	}
+	newNodeConfig.Config.Index = &config.IndexConfig{
+		InputFile:   &indexInFile,
+		OutputFile:  &indexOutFile,
+		InputString: &body,
+	}
+	newNodeConfig.Hidden = hidden
+
+	currentNode.Sub = append(currentNode.Sub, newNodeConfig)
+}