mark2web/helper/content.go

package helper

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"gitbase.de/apairon/mark2web/config"
	"github.com/Depado/bfchroma"
	"github.com/davecgh/go-spew/spew"
	"github.com/extemporalgenome/slug"
	"github.com/flosch/pongo2"
	cpy "github.com/otiai10/copy"
	"gopkg.in/russross/blackfriday.v2"
	"gopkg.in/yaml.v2"
)

func newContext() map[string]interface{} {
	return map[string]interface{}{
		"fnRequest": RequestFn,
		"fnRender":  RenderFn,

		"AssetsPath": config.Config.Assets.ToPath,
	}
}

func fillNodeConfig(node *config.PathConfigTree, inBase, outBase, dir string, conf *config.PathConfig) {
	inPath := inBase
	if dir != "" {
		inPath += "/" + dir
	}

	Log.Infof("reading input directory: %s", inPath)

	node.InputPath = inPath

	// read config
	newConfig := new(config.PathConfig)
	Log.Debug("looking for config.yml ...")
	configFile := inPath + "/config.yml"
	if _, err := os.Stat(configFile); os.IsNotExist(err) {
		Log.Debug("no config.yml found in this directory, using upper configs")
		config.Merge(newConfig, conf)
		// remove this
		newConfig.This = config.ThisPathConfig{}
	} else {
		Log.Debug("reading config...")
		data, err := ioutil.ReadFile(configFile)
		if err != nil {
			Log.Panicf("could not read file '%s': %s", configFile, err)
		}
		err = yaml.Unmarshal(data, newConfig)
		if err != nil {
			Log.Panicf("could not parse YAML file '%s': %s", configFile, err)
		}

		Log.Debug("merging config with upper config")
		oldThis := newConfig.This
		config.Merge(newConfig, conf)
		newConfig.This = oldThis

		Log.Debug(spew.Sdump(newConfig))
	}

	node.Config = newConfig

	// calc outDir
	stripedDir := dir
	var regexStr *string
	if newConfig.Path != nil {
		regexStr = newConfig.Path.Strip
	}
	if regexStr != nil && *regexStr != "" {
		if regex, err := regexp.Compile(*regexStr); err != nil {
			Log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err)
		} else {
			stripedDir = regex.ReplaceAllString(stripedDir, "$1")
		}
	}

	if node.Config.This.Navname == nil {
		navname := strings.Replace(stripedDir, "_", " ", -1)
		node.Config.This.Navname = &navname
	}

	stripedDir = slug.Slug(stripedDir)
	outPath := outBase + "/" + stripedDir
	outPath = path.Clean(outPath)

	Log.Infof("calculated output directory: %s", outPath)
	node.OutputPath = outPath

	// handle collections
	for _, colConfig := range newConfig.This.Collections {
		if colConfig != nil {
			if colConfig.Name == nil || *colConfig.Name == "" {
				Log.Panicf("missing Name in collection config in '%s'", inPath)
			}
			if colConfig.EntriesJSON == nil || *colConfig.EntriesJSON == "" {
				Log.Panicf("missing EntriesJSON in collection config in '%s'", inPath)
			}
		}

		if node.ColMap == nil {
			node.ColMap = make(map[string]interface{})
		}
		ctx := newContext()
		jsonStr, err := pongo2.RenderTemplateString(*colConfig.EntriesJSON, ctx)
		if err != nil {
			Log.Panicf("invalid template string in '%s': %s", inPath, err)
		}

		var colSlice []map[string]interface{}
		err = json.Unmarshal([]byte(jsonStr), &colSlice)
		if err != nil {
			Log.Panicf("invalid JSON in EntriesJSON in '%s': %s", inPath, err)
		}
		node.ColMap[*colConfig.Name] = colSlice
		//node.ColMap[*colConfig.Name] = jsonWebRequest(*colConfig.URL)

		if navT := colConfig.NavTemplate; navT != nil {
			tpl := navT.Template
			if tpl == "" {
				tpl = *newConfig.Template
			}

			// build navigation with detail sites
			for _, colEl := range colSlice {
				ctxE := make(map[string]interface{})
				err := config.Merge(&ctxE, ctx)
				if err != nil {
					Log.Panicf("could not merge context in '%s': %s", inPath, err)
				}
				err = config.Merge(&ctxE, colEl)
				if err != nil {
					Log.Panicf("could not merge context in '%s': %s", inPath, err)
				}

				goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
				if err != nil {
					Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", inPath, err)
				}
				goTo = strings.Trim(goTo, "/")
				goTo = path.Clean(goTo)

				if strings.Contains(goTo, "..") {
					Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", inPath, goTo)
				}
				if goTo == "." {
					Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", inPath)
				}
				if goTo == "" {
					Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", inPath)
				}

				navname := ""
				if navT.Navname != "" {
					navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
					if err != nil {
						Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", inPath, err)
					}
				}
				body := ""
				if navT.Body != "" {
					body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
					if err != nil {
						Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", inPath, err)
					}
				}

				add2Nav(node, node.Config, tpl, goTo, navname, colEl, navT.DataKey, body, navT.Hidden)
			}
		}

	}
}

// ReadContentDir walks through content directory and builds the tree of configurations
func ReadContentDir(inBase string, outBase string, dir string, conf *config.PathConfig, tree *config.PathConfigTree) {
	fillNodeConfig(tree, inBase, outBase, dir, conf)

	files, err := ioutil.ReadDir(tree.InputPath)
	if err != nil {
		Log.Panic(err)
	}

	// first only files
	for _, f := range files {
		p := tree.InputPath + "/" + f.Name()
		if !f.IsDir() && f.Name() != "config.yml" {
			switch path.Ext(f.Name()) {
			case ".md":
				Log.Debugf(".MD %s", p)
				if tree.InputFiles == nil {
					tree.InputFiles = make([]string, 0)
				}
				tree.InputFiles = append(tree.InputFiles, f.Name())
				break
			default:
				Log.Debugf("FIL %s", p)
				if tree.OtherFiles == nil {
					tree.OtherFiles = make([]string, 0)
				}
				tree.OtherFiles = append(tree.OtherFiles, f.Name())
			}
		}
	}

	// only directorys, needed config before
	for _, f := range files {
		p := tree.InputPath + "/" + f.Name()
		if f.IsDir() {
			Log.Debugf("DIR %s", p)
			newTree := new(config.PathConfigTree)
			if tree.Sub == nil {
				tree.Sub = make([]*config.PathConfigTree, 0)
			}
			tree.Sub = append(tree.Sub, newTree)
			ReadContentDir(tree.InputPath, tree.OutputPath, f.Name(), tree.Config, newTree)
		}
	}
}

// ProcessContent walks recursivly through the input paths and processes all files for output
func ProcessContent(rootConf, conf *config.PathConfigTree) {
	CreateDirectory(conf.OutputPath)

	curNavPath := strings.TrimPrefix(conf.OutputPath, config.Config.Directories.Output)
	curNavPath = strings.TrimPrefix(curNavPath, "/")
	curNavPath = path.Clean(curNavPath)
	if curNavPath == "." {
		curNavPath = ""
	}

	goTo := conf.Config.This.GoTo
	if goTo != nil && *goTo != "" {
		goToFixed := *goTo
		if strings.HasPrefix(goToFixed, "/") {
			goToFixed = BackToRoot(curNavPath) + goToFixed
		}
		goToFixed = path.Clean(goToFixed)

		switch config.Config.Webserver.Type {
		case "apache":
			htaccessFile := conf.OutputPath + "/.htaccess"
			Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed)
			err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
`), 0644)
			if err != nil {
				Log.Panicf("could not write '%s': %s", htaccessFile, err)
			}
			break
		}
	}

	for _, file := range conf.InputFiles {
		var input []byte
		inFile := "InputString"

		if file != "" {
			inFile = conf.InputPath + "/" + file
			Log.Debugf("reading file: %s", inFile)

			var err error
			input, err = ioutil.ReadFile(inFile)
			if err != nil {
				Log.Panicf("could not read '%s':%s", inFile, err)
			}
			Log.Infof("processing input file '%s'", inFile)
		} else {
			// use input string if available and input filename == ""
			var inputString *string
			if i := conf.Config.Index; i != nil {
				inputString = i.InputString
			}
			if inputString != nil {
				Log.Debugf("using input string instead of file")
				input = []byte(*inputString)
			}
		}

		newConfig := new(config.PathConfig)

		regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?")
		yamlData := regex.Find(input)
		if string(yamlData) != "" {
			// replace tabs
			yamlData = bytes.Replace(yamlData, []byte("\t"), []byte("    "), -1)

			Log.Debugf("found yaml header in '%s', merging config", inFile)
			err := yaml.Unmarshal(yamlData, newConfig)
			if err != nil {
				Log.Panicf("could not parse YAML header from '%s': %s", inFile, err)
			}

			Log.Debug("merging config with upper config")
			oldThis := newConfig.This
			config.Merge(newConfig, conf.Config)
			newConfig.This = oldThis

			Log.Debug(spew.Sdump(newConfig))

			input = regex.ReplaceAll(input, []byte(""))
		} else {
			config.Merge(newConfig, conf.Config)
		}

		// ignore ???
		ignoreFile := false
		var ignoreRegex *string
		var stripRegex *string
		var outputExt *string
		if f := newConfig.Filename; f != nil {
			ignoreRegex = f.Ignore
			stripRegex = f.Strip
			outputExt = f.OutputExtension
		}
		if ignoreRegex != nil && *ignoreRegex != "" {
			regex, err := regexp.Compile(*ignoreRegex)
			if err != nil {
				Log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
			}
			ignoreFile = regex.MatchString(file)
		}

		if ignoreFile {
			Log.Infof("ignoring file '%s', because of filename.ignore", inFile)
		} else {

			// build output filename
			outputFilename := file

			var indexInputFile *string
			var indexOutputFile *string
			if i := newConfig.Index; i != nil {
				indexInputFile = i.InputFile
				indexOutputFile = i.OutputFile
			}

			if indexInputFile != nil &&
				*indexInputFile == file &&
				indexOutputFile != nil &&
				*indexOutputFile != "" {
				outputFilename = *indexOutputFile
			} else {
				if stripRegex != nil && *stripRegex != "" {
					regex, err := regexp.Compile(*stripRegex)
					if err != nil {
						Log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
					}
					outputFilename = regex.ReplaceAllString(outputFilename, "$1")
				}
				if outputExt != nil && *outputExt != "" {
					outputFilename += "." + *outputExt
				}
			}

			outFile := conf.OutputPath + "/" + outputFilename
			Log.Debugf("using '%s' as output file", outFile)

			// use --- for splitting document in markdown parts
			regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?")
			inputParts := regex.Split(string(input), -1)
			htmlParts := make([]*pongo2.Value, 0)
			for _, iPart := range inputParts {
				htmlParts = append(htmlParts,
					pongo2.AsSafeValue(
						string(renderMarkdown([]byte(iPart), newConfig.Markdown))))
			}

			// build navigation
			navMap := make(map[string]*NavElement)
			navSlice := make([]*NavElement, 0)
			navActive := make([]*NavElement, 0)
			BuildNavigation(rootConf, &navMap, &navSlice, &navActive, curNavPath)

			// read yaml header as data for template
			ctx := newContext()
			ctx["This"] = newConfig.This
			ctx["Meta"] = newConfig.Meta
			ctx["Data"] = newConfig.Data
			ctx["ColMap"] = rootConf.ColMap // root as NavMap and NavSlice, for sub go to NavElement.ColMap
			ctx["NavMap"] = navMap
			ctx["NavSlice"] = navSlice
			ctx["NavActive"] = navActive
			ctx["Body"] = pongo2.AsSafeValue(string(renderMarkdown(input, newConfig.Markdown)))
			ctx["BodyParts"] = htmlParts
			ctx["CurrentPath"] = curNavPath
			// set active nav element
			if len(navActive) > 0 {
				ctx["NavElement"] = navActive[len(navActive)-1]
			} else {
				// if no active path to content, we are in root dir
				ctx["NavElement"] = &NavElement{
					GoTo:     BackToRoot(curNavPath),
					Active:   true,
					ColMap:   rootConf.ColMap,
					Data:     rootConf.Config.Data,
					This:     rootConf.Config.This,
					SubMap:   &navMap,
					SubSlice: &navSlice,
				}
			}

			Log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
			templateFilename := *newConfig.Template
			result, err := RenderTemplate(*newConfig.Template, conf, newConfig, &ctx)
			if err != nil {
				Log.Panicf("could not execute template '%s' for input file '%s': %s", templateFilename, inFile, err)
			}

			result = FixAssetsPath(result, curNavPath)

			Log.Noticef("writing to output file: %s", outFile)
			err = ioutil.WriteFile(outFile, []byte(result), 0644)
			if err != nil {
				Log.Panicf("could not write to output file '%s': %s", outFile, err)
			}

			//fmt.Println(string(html))
		}
	}

	// process other files, copy...
	for _, file := range conf.OtherFiles {
		switch config.Config.OtherFiles.Action {
		case "copy":
			from := conf.InputPath + "/" + file
			to := conf.OutputPath + "/" + file
			Log.Noticef("copying file from '%s' to '%s'", from, to)
			err := cpy.Copy(from, to)
			if err != nil {
				Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
			}
		}
	}

	i := 0
	for i < len(conf.Sub) {
		ProcessContent(rootConf, conf.Sub[i])
		i++
	}
}

func renderMarkdown(input []byte, markdownConf *config.MarkdownConfig) []byte {
	var options []blackfriday.Option

	var chromaRenderer *bool
	var chromaStyle *string
	if m := markdownConf; m != nil {
		chromaRenderer = m.ChromaRenderer
		chromaStyle = m.ChromaStyle
	}
	if chromaStyle == nil {
		style := "monokai"
		chromaStyle = &style
	}
	if chromaRenderer != nil && *chromaRenderer {
		options = []blackfriday.Option{
			blackfriday.WithRenderer(
				bfchroma.NewRenderer(
					bfchroma.Style(*chromaStyle),
				),
			),
		}
	}

	// fix \r from markdown for blackfriday
	input = bytes.Replace(input, []byte("\r"), []byte(""), -1)
	return blackfriday.Run(input, options...)
}