mark2web/main.go

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"reflect"
	"regexp"
	"strings"

	"github.com/imdario/mergo"

	"gitbase.de/apairon/mark2web/helper"
	"github.com/Depado/bfchroma"
	"github.com/davecgh/go-spew/spew"
	"github.com/flosch/pongo2"
	"github.com/gosimple/slug"
	"github.com/op/go-logging"
	cpy "github.com/otiai10/copy"
	"gopkg.in/russross/blackfriday.v2"
	"gopkg.in/yaml.v2"
)

var (
	// Version is the app's version string
	Version = "UNKNOWN"
	// GitHash is the current git hash for this version
	GitHash = "UNKNOWN"
	// BuildTime is the time of build of this app
	BuildTime = "UNKNOWN"
)

var log = logging.MustGetLogger("myLogger")

var inDir *string
var outDir *string

var templateCache = make(map[string]*pongo2.Template)

// GlobalConfig is config which is used only once in root dir
type GlobalConfig struct {
	Webserver struct {
		Type string `yaml:"Type"`
	} `yaml:"Webserver"`

	Assets struct {
		FromPath    string `yaml:"FromPath"`
		ToPath      string `yaml:"ToPath"`
		Action      string `yaml:"Action"`
		FixTemplate struct {
			Find    string `yaml:"Find"`
			Replace string `yaml:"Replace"`
		} `yaml:"FixTemplate"`
	} `yaml:"Assets"`

	OtherFiles struct {
		Action string `yaml:"Action"`
	} `yaml:"OtherFiles"`
}

var globalConfig = new(GlobalConfig)

// ThisPathConfig is struct for This in paths yaml
type ThisPathConfig struct {
	Navname *string     `yaml:"Navname"`
	GoTo    *string     `yaml:"GoTo"`
	Data    interface{} `yaml:"Data"`
}

type indexStruct struct {
	InputFile  *string `yaml:"InputFile"`
	OutputFile *string `yaml:"OutputFile"`
}

type metaStruct struct {
	Title       *string `yaml:"Title"`
	Description *string `yaml:"Description"`
	Keywords    *string `yaml:"Keywords"`
}

type pathStruct struct {
	Strip        *string `yaml:"Strip"`
	IgnoreForNav *string `yaml:"IgnoreForNav"`
}

type filenameStruct struct {
	Strip           *string `yaml:"Strip"`
	Ignore          *string `yaml:"Ignore"`
	OutputExtension *string `yaml:"OutputExtension"`
}

type markdownStruct struct {
	ChromaRenderer *bool   `yaml:"ChromaRenderer"`
	ChromaStyle    *string `yaml:"ChromaStyle"`
}

// PathConfig of subdir
type PathConfig struct {
	This     ThisPathConfig  `yaml:"This"`
	Template *string         `yaml:"Template"`
	Index    *indexStruct    `yaml:"Index"`
	Meta     *metaStruct     `yaml:"Meta"`
	Path     *pathStruct     `yaml:"Path"`
	Filename *filenameStruct `yaml:"Filename"`
	Markdown *markdownStruct `yaml:"Markdown"`

	Data interface{} `yaml:"Data"`
}

// PathConfigTree is complete config tree of content dir
type PathConfigTree struct {
	InputPath  string
	OutputPath string

	InputFiles []string
	OtherFiles []string

	Config *PathConfig
	Sub    []*PathConfigTree
}

var contentConfig = new(PathConfigTree)

type ptrTransformer struct{}

func (t ptrTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
	if typ.Kind() == reflect.Ptr {
		return func(dst, src reflect.Value) error {
			if dst.CanSet() {
				if dst.IsNil() {
					dst.Set(src)
				}
			}
			return nil
		}
	}
	return nil
}

func merge(dst, src interface{}) error {
	return mergo.Merge(dst, src, mergo.WithTransformers(ptrTransformer{}))
}

func backToRoot(curNavPath string) string {
	tmpPath := ""
	if curNavPath != "" {
		for i := strings.Count(curNavPath, "/") + 1; i > 0; i-- {
			tmpPath += "../"
		}
	}
	return tmpPath
}

func readContentDir(inBase string, outBase string, dir string, conf *PathConfig, tree *PathConfigTree) {
	inPath := inBase
	if dir != "" {
		inPath += "/" + dir
	}

	log.Infof("reading input directory: %s", inPath)

	files, err := ioutil.ReadDir(inPath)
	if err != nil {
		log.Panic(err)
	}

	tree.InputPath = inPath

	// read config
	newConfig := new(PathConfig)
	log.Debug("looking for config.yml ...")
	configFile := inPath + "/config.yml"
	if _, err = os.Stat(configFile); os.IsNotExist(err) {
		log.Debug("no config.yml found in this directory, using upper configs")
		merge(newConfig, conf)
		// remove this
		newConfig.This = ThisPathConfig{}
	} else {
		log.Debug("reading config...")
		data, err := ioutil.ReadFile(configFile)
		if err != nil {
			log.Panicf("could not read file '%s': %s", configFile, err)
		}
		err = yaml.Unmarshal(data, newConfig)
		if err != nil {
			log.Panicf("could not parse YAML file '%s': %s", configFile, err)
		}

		log.Debug("merging config with upper config")
		oldThis := newConfig.This
		merge(newConfig, conf)
		newConfig.This = oldThis

		log.Debug(spew.Sdump(newConfig))
	}

	tree.Config = newConfig

	// calc outDir
	stripedDir := dir
	var regexStr *string
	if newConfig.Path != nil {
		regexStr = newConfig.Path.Strip
	}
	if regexStr != nil && *regexStr != "" {
		if regex, err := regexp.Compile(*regexStr); err != nil {
			log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err)
		} else {
			stripedDir = regex.ReplaceAllString(stripedDir, "$1")
		}
	}

	if tree.Config.This.Navname == nil {
		navname := strings.Replace(stripedDir, "_", " ", -1)
		tree.Config.This.Navname = &navname
	}

	stripedDir = slug.Make(stripedDir)
	outPath := outBase + "/" + stripedDir
	outPath = path.Clean(outPath)

	log.Infof("calculated output directory: %s", outPath)
	tree.OutputPath = outPath

	// first only files
	for _, f := range files {
		p := inPath + "/" + f.Name()
		if !f.IsDir() && f.Name() != "config.yml" {
			switch path.Ext(f.Name()) {
			case ".md":
				log.Debugf(".MD %s", p)
				if tree.InputFiles == nil {
					tree.InputFiles = make([]string, 0)
				}
				tree.InputFiles = append(tree.InputFiles, f.Name())
				break
			default:
				log.Debugf("FIL %s", p)
				if tree.OtherFiles == nil {
					tree.OtherFiles = make([]string, 0)
				}
				tree.OtherFiles = append(tree.OtherFiles, f.Name())
			}
		}
	}

	// only directorys, needed config before
	for _, f := range files {
		p := inPath + "/" + f.Name()
		if f.IsDir() {
			log.Debugf("DIR %s", p)
			newTree := new(PathConfigTree)
			if tree.Sub == nil {
				tree.Sub = make([]*PathConfigTree, 0)
			}
			tree.Sub = append(tree.Sub, newTree)
			readContentDir(inPath, outPath, f.Name(), newConfig, newTree)
		}
	}
}

type navElement struct {
	Navname string
	GoTo    string
	Active  bool

	Data interface{}

	This ThisPathConfig

	SubMap   *map[string]*navElement
	SubSlice *[]*navElement
}

func buildNavigation(conf *PathConfigTree, curNavMap *map[string]*navElement, curNavSlice *[]*navElement, navActive *[]*navElement, activeNav string) {
	for _, el := range conf.Sub {
		var ignNav *string
		if p := el.Config.Path; p != nil {
			ignNav = p.IgnoreForNav
		}
		if ignNav != nil && *ignNav != "" {
			regex, err := regexp.Compile(*ignNav)
			if err != nil {
				log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err)
			}
			if regex.MatchString(path.Base(el.InputPath)) {
				log.Debugf("ignoring input directory '%s' in navigation", el.InputPath)
				continue
			}
		}

		elPath := strings.TrimPrefix(el.OutputPath, *outDir+"/")

		subMap := make(map[string]*navElement)
		subSlice := make([]*navElement, 0)
		navEl := navElement{
			Active:   strings.HasPrefix(activeNav, elPath),
			Data:     el.Config.Data,
			SubMap:   &subMap,
			SubSlice: &subSlice,
		}

		navEl.This = el.Config.This

		if navEl.Active {
			// add to navActive level navigation
			currentLevel := strings.Count(activeNav, "/")
			if len(*navActive) <= currentLevel {
				// not registered
				*navActive = append(*navActive, &navEl)
			}
		}

		n := el.Config.This.Navname
		if n != nil {
			navEl.Navname = *n
		}
		g := el.Config.This.GoTo
		if g != nil {
			if strings.HasPrefix(*g, "/") {
				// abslute
				navEl.GoTo = *g
			} else {
				// relative
				navEl.GoTo = elPath + "/" + *g
			}
		} else {
			navEl.GoTo = elPath + "/"
		}

		if activeNav != "" && activeNav != "/" {
			// calculate relative path
			bToRoot := backToRoot(activeNav)
			navEl.GoTo = bToRoot + navEl.GoTo
			navEl.GoTo = path.Clean(navEl.GoTo)
		}

		(*curNavMap)[navEl.Navname] = &navEl
		if curNavSlice != nil {
			*curNavSlice = append(*curNavSlice, &navEl)
		}

		buildNavigation(el, &subMap, &subSlice, navActive, activeNav)
	}
}

func processContent(conf *PathConfigTree) {
	log.Debugf("trying to create output directory: %s", conf.OutputPath)

	if dirH, err := os.Stat(conf.OutputPath); os.IsNotExist(err) {
		err := os.MkdirAll(conf.OutputPath, 0755)
		if err != nil {
			log.Panicf("could not create output directory '%s': %s", conf.OutputPath, err)
		}
		log.Noticef("created output directory: %s", conf.OutputPath)
	} else if dirH != nil {
		if dirH.IsDir() {
			log.Noticef("output directory '%s' already exists", conf.OutputPath)
		} else {
			log.Panicf("output directory '%s' is no directory", conf.OutputPath)
		}
	} else {
		log.Panicf("unknown error for output directory '%s': %s", conf.OutputPath, err)
	}

	curNavPath := strings.TrimPrefix(conf.OutputPath, *outDir)
	curNavPath = strings.TrimPrefix(curNavPath, "/")
	curNavPath = path.Clean(curNavPath)
	if curNavPath == "." {
		curNavPath = ""
	}

	goTo := conf.Config.This.GoTo
	if goTo != nil && *goTo != "" {
		goToFixed := *goTo
		if strings.HasPrefix(goToFixed, "/") {
			goToFixed = backToRoot(curNavPath) + goToFixed
		}
		goToFixed = path.Clean(goToFixed)

		switch globalConfig.Webserver.Type {
		case "apache":
			htaccessFile := conf.OutputPath + "/.htaccess"
			log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed)
			err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
`), 0644)
			if err != nil {
				log.Panicf("could not write '%s': %s", htaccessFile, err)
			}
			break
		}
	}

	for _, file := range conf.InputFiles {
		inFile := conf.InputPath + "/" + file
		log.Debugf("reading file: %s", inFile)

		input, err := ioutil.ReadFile(inFile)
		if err != nil {
			log.Panicf("could not read '%s':%s", inFile, err)
		}
		log.Infof("processing input file '%s'", inFile)

		newConfig := new(PathConfig)

		regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?")
		yamlData := regex.Find(input)
		if string(yamlData) != "" {
			log.Debugf("found yaml header in '%s', merging config", inFile)
			err = yaml.Unmarshal(yamlData, newConfig)
			if err != nil {
				log.Panicf("could not parse YAML header from '%s': %s", inFile, err)
			}

			log.Debug("merging config with upper config")
			oldThis := newConfig.This
			merge(newConfig, conf.Config)
			newConfig.This = oldThis

			log.Debug(spew.Sdump(newConfig))

			input = regex.ReplaceAll(input, []byte(""))
		} else {
			merge(newConfig, conf.Config)
		}

		// ignore ???
		ignoreFile := false
		var ignoreRegex *string
		var stripRegex *string
		var outputExt *string
		if f := newConfig.Filename; f != nil {
			ignoreRegex = f.Ignore
			stripRegex = f.Strip
			outputExt = f.OutputExtension
		}
		if ignoreRegex != nil && *ignoreRegex != "" {
			regex, err := regexp.Compile(*ignoreRegex)
			if err != nil {
				log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
			}
			ignoreFile = regex.MatchString(file)
		}

		if ignoreFile {
			log.Infof("ignoring file '%s', because of filename.ignore", inFile)
		} else {

			// build output filename
			outputFilename := file

			var indexInputFile *string
			var indexOutputFile *string
			if i := newConfig.Index; i != nil {
				indexInputFile = i.InputFile
				indexOutputFile = i.OutputFile
			}

			if indexInputFile != nil && *indexInputFile == file && indexOutputFile != nil && *indexOutputFile != "" {
				outputFilename = *indexOutputFile
			} else {
				if stripRegex != nil && *stripRegex != "" {
					regex, err := regexp.Compile(*stripRegex)
					if err != nil {
						log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
					}
					outputFilename = regex.ReplaceAllString(outputFilename, "$1")
				}
				if outputExt != nil && *outputExt != "" {
					outputFilename += "." + *outputExt
				}
			}

			outFile := conf.OutputPath + "/" + outputFilename
			log.Debugf("using '%s' as output file", outFile)

			var options []blackfriday.Option

			var chromaRenderer *bool
			var chromaStyle *string
			if m := newConfig.Markdown; m != nil {
				chromaRenderer = m.ChromaRenderer
				chromaStyle = m.ChromaStyle
			}
			if chromaStyle == nil {
				style := "monokai"
				chromaStyle = &style
			}
			if chromaRenderer != nil && *chromaRenderer {
				options = []blackfriday.Option{
					blackfriday.WithRenderer(
						bfchroma.NewRenderer(
							bfchroma.Style(*chromaStyle),
						),
					),
				}
			}

			// fix \r from markdown for blackfriday
			input = bytes.Replace(input, []byte("\r"), []byte(""), -1)
			html := blackfriday.Run(input, options...)

			// use --- for splitting document in markdown parts
			regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?")
			inputParts := regex.Split(string(input), -1)
			htmlParts := make([]*pongo2.Value, 0)
			for _, iPart := range inputParts {
				htmlParts = append(htmlParts, pongo2.AsSafeValue(string(blackfriday.Run([]byte(iPart), options...))))
			}

			log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
			templateFile := *inDir + "/templates/" + *newConfig.Template
			template := templateCache[templateFile]
			if template == nil {
				var err error
				if template, err = pongo2.FromFile(templateFile); err != nil {
					log.Panicf("could not parse template '%s': %s", templateFile, err)
				} else {
					templateCache[templateFile] = template
				}
			}

			// build navigation
			navMap := make(map[string]*navElement)
			navSlice := make([]*navElement, 0)
			navActive := make([]*navElement, 0)
			buildNavigation(contentConfig, &navMap, &navSlice, &navActive, curNavPath)

			// read yaml header as data for template
			ctx := make(pongo2.Context)
			ctx["This"] = newConfig.This
			ctx["Meta"] = newConfig.Meta
			ctx["Data"] = newConfig.Data
			ctx["NavMap"] = navMap
			ctx["NavSlice"] = navSlice
			ctx["NavActive"] = navActive
			ctx["Body"] = pongo2.AsSafeValue(string(html))
			ctx["BodyParts"] = htmlParts

			// register functions
			ctx["fnRequest"] = helper.Request

			result, err := template.Execute(ctx)
			if err != nil {
				log.Panicf("could not execute template '%s' for input file '%s': %s", templateFile, inFile, err)
			}

			if find := globalConfig.Assets.FixTemplate.Find; find != "" {
				log.Debugf("fixing assets paths in '%s' for '%s'", templateFile, inFile)
				bToRoot := backToRoot(curNavPath)
				regex, err := regexp.Compile(find)
				if err != nil {
					log.Panicf("could not compile regexp '%s' for assets path: %s", find, err)
				}
				repl := globalConfig.Assets.FixTemplate.Replace
				repl = bToRoot + globalConfig.Assets.ToPath + "/" + repl
				repl = path.Clean(repl) + "/"
				log.Debugf("new assets paths: %s", repl)
				result = regex.ReplaceAllString(result, repl)
			}

			log.Noticef("writing to output file: %s", outFile)
			err = ioutil.WriteFile(outFile, []byte(result), 0644)
			if err != nil {
				log.Panicf("could not write to output file '%s': %s", outFile, err)
			}

			//fmt.Println(string(html))
		}
	}

	// process other files, copy...
	for _, file := range conf.OtherFiles {
		switch globalConfig.OtherFiles.Action {
		case "copy":
			from := conf.InputPath + "/" + file
			to := conf.OutputPath + "/" + file
			log.Noticef("copying file from '%s' to '%s'", from, to)
			err := cpy.Copy(from, to)
			if err != nil {
				log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
			}
		}
	}

	for _, el := range conf.Sub {
		processContent(el)
	}
}

func processAssets() {
	switch globalConfig.Assets.Action {
	case "copy":
		from := globalConfig.Assets.FromPath
		to := globalConfig.Assets.ToPath
		if !strings.HasPrefix(from, "/") {
			from = *inDir + "/" + from
		}
		if !strings.HasPrefix(to, "/") {
			to = *outDir + "/" + to
		}
		log.Noticef("copying assets from '%s' to '%s'", from, to)
		err := cpy.Copy(from, to)
		if err != nil {
			log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
		}
	}
}

func main() {
	spew.Config.DisablePointerAddresses = true
	spew.Config.DisableCapacities = true
	spew.Config.DisableMethods = true
	spew.Config.DisablePointerMethods = true

	inDir = flag.String("in", "./", "input directory")
	outDir = flag.String("out", "html", "output directory")
	createOutDir := flag.Bool("create", false, "create output directory if not existing")
	//clearOutDir := flag.Bool("clear", false, "clear output directory before generating website")
	logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error")
	version := flag.Bool("version", false, "print version of this executable")

	flag.Parse()
	if version != nil && *version {
		fmt.Printf(`%11s: %s
%11s: %s
%11s: %s
`, "version", Version, "git hash", GitHash, "build time", BuildTime)
		os.Exit(0)
	}

	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
	))
	logBackendLeveled := logging.AddModuleLevel(logBackendFormatter)
	logBackendLevel := logging.INFO
	if logLevel != nil {
		switch *logLevel {
		case "debug":
			logBackendLevel = logging.DEBUG
			break

		case "info":
			logBackendLevel = logging.INFO
			break

		case "notice":
			logBackendLevel = logging.NOTICE
			break

		case "warning":
			logBackendLevel = logging.WARNING
			break

		case "error":
			logBackendLevel = logging.ERROR
			break

		}
	}
	logBackendLeveled.SetLevel(logBackendLevel, "")
	logging.SetBackend(logBackendLeveled)

	if inDir == nil || *inDir == "" {
		log.Panic("input directory not specified")
	}
	log.Infof("input directory: %s", *inDir)

	if outDir == nil || *outDir == "" {
		log.Panic("output directory not specified")
	}
	log.Infof("output directory: %s", *outDir)

	if createOutDir != nil && *createOutDir {
		if _, err := os.Stat(*outDir); os.IsNotExist(err) {
			log.Debugf("output directory '%s' does not exist", *outDir)
			log.Debugf("trying to create output directory: %s", *outDir)
			err := os.MkdirAll(*outDir, 0755)
			if err != nil {
				log.Panic(err)
			}
			log.Noticef("created output directory: %s", *outDir)
		} else {
			log.Noticef("output directory '%s' already exists", *outDir)
		}
	}

	if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
		log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
	} else {
		if fD == nil {
			log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
		} else if !fD.IsDir() {
			log.Panicf("output directory '%s' is not a directory", *outDir)
		}
	}

	log.Debug("reading global config...")
	p := *inDir + "/config.yml"
	data, err := ioutil.ReadFile(p)
	if err != nil {
		log.Panicf("could not read file '%s': %s", p, err)
	}
	err = yaml.Unmarshal(data, globalConfig)
	if err != nil {
		log.Panicf("could not parse YAML file '%s': %s", p, err)
	}

	log.Debug(spew.Sdump(globalConfig))

	log.Debugf("reading input directory %s", *inDir)

	defaultTemplate := "base.html"
	defaultInputFile := "README.md"
	defaultOutputFile := "index.html"
	defaultPathStrip := "^[0-9]*_(.*)"
	defaultPathIgnoreForNav := "^_"
	defaultFilenameStrip := "(.*).md$"
	defaultFilenameIgnore := "^_"
	defaultFilenameOutputExtension := "html"

	defaultPathConfig := new(PathConfig)
	defaultPathConfig.Template = &defaultTemplate
	defaultPathConfig.Index = &indexStruct{
		InputFile:  &defaultInputFile,
		OutputFile: &defaultOutputFile,
	}
	defaultPathConfig.Path = &pathStruct{
		Strip:        &defaultPathStrip,
		IgnoreForNav: &defaultPathIgnoreForNav,
	}
	defaultPathConfig.Filename = &filenameStruct{
		Strip:           &defaultFilenameStrip,
		Ignore:          &defaultFilenameIgnore,
		OutputExtension: &defaultFilenameOutputExtension,
	}

	readContentDir(*inDir+"/content", *outDir, "", defaultPathConfig, contentConfig)
	//spew.Dump(contentConfig)

	//spew.Dump(navMap)

	processContent(contentConfig)

	processAssets()

}