mark2web/main.go

package main

import (
	"flag"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/imdario/mergo"

	"github.com/aymerick/raymond"
	"github.com/davecgh/go-spew/spew"
	"github.com/op/go-logging"
	cpy "github.com/otiai10/copy"
	"gopkg.in/russross/blackfriday.v2"
	"gopkg.in/yaml.v2"
)

var log = logging.MustGetLogger("myLogger")

var inDir *string
var outDir *string

var templateCache = make(map[string]*raymond.Template)

// GlobalConfig is config which is used only once in root dir
type GlobalConfig struct {
	Webserver struct {
		Type string `yaml:"Type"`
	} `yaml:"Webserver"`

	Assets struct {
		FromPath    string `yaml:"FromPath"`
		ToPath      string `yaml:"ToPath"`
		Action      string `yaml:"Action"`
		FixTemplate struct {
			Find    string `yaml:"Find"`
			Replace string `yaml:"Replace"`
		} `yaml:"FixTemplate"`
	} `yaml:"Assets"`

	OtherFiles struct {
		Action string `yaml:"Action"`
	} `yaml:"OtherFiles"`
}

var globalConfig = new(GlobalConfig)

// PathConfig of subdir
type PathConfig struct {
	This struct {
		Navname *string `yaml:"Navname"`
		GoTo    *string `yaml:"GoTo"`
	} `yaml:"This"`

	Template *string `yaml:"Template"`

	Index struct {
		InputFile  *string `yaml:"InputFile"`
		OutputFile *string `yaml:"OutputFile"`
	} `yaml:"Index"`

	Meta struct {
		Title       *string `yaml:"Title"`
		Description *string `yaml:"Description"`
		Keywords    *string `yaml:"Keywords"`
	} `yaml:"Meta"`

	Path struct {
		Strip        *string `yaml:"Strip"`
		IgnoreForNav *string `yaml:"IgnoreForNav"`
	} `yaml:"Path"`

	Filename struct {
		Strip           *string `yaml:"Strip"`
		Ignore          *string `yaml:"Ignore"`
		OutputExtension *string `yaml:"OutputExtension"`
	} `yaml:"Filename"`

	Data interface{} `yaml:"Data"`
}

// PathConfigTree is complete config tree of content dir
type PathConfigTree struct {
	InputPath  string
	OutputPath string

	InputFiles []string
	OtherFiles []string

	Config *PathConfig
	Sub    []*PathConfigTree
}

func mergeConfig(mergeInto *PathConfig, mergeFrom *PathConfig) {
	// navname, goto are individual, so no merging here

	if mergeInto.Template == nil || *mergeInto.Template == "" { // always require a template
		mergeInto.Template = mergeFrom.Template
	}

	if mergeInto.Index.InputFile == nil {
		mergeInto.Index.InputFile = mergeFrom.Index.InputFile
	}

	if mergeInto.Index.InputFile == nil {
		mergeInto.Index.InputFile = mergeFrom.Index.InputFile
	}

	if mergeInto.Index.OutputFile == nil {
		mergeInto.Index.OutputFile = mergeFrom.Index.OutputFile
	}

	if mergeInto.Meta.Title == nil {
		mergeInto.Meta.Title = mergeFrom.Meta.Title
	}
	if mergeInto.Meta.Description == nil {
		mergeInto.Meta.Description = mergeFrom.Meta.Description
	}
	if mergeInto.Meta.Keywords == nil {
		mergeInto.Meta.Keywords = mergeFrom.Meta.Keywords
	}

	if mergeInto.Path.IgnoreForNav == nil {
		mergeInto.Path.IgnoreForNav = mergeFrom.Path.IgnoreForNav
	}
	if mergeInto.Path.Strip == nil {
		mergeInto.Path.Strip = mergeFrom.Path.Strip
	}

	if mergeInto.Filename.Strip == nil {
		mergeInto.Filename.Strip = mergeFrom.Filename.Strip
	}
	if mergeInto.Filename.Ignore == nil {
		mergeInto.Filename.Ignore = mergeFrom.Filename.Ignore
	}
	if mergeInto.Filename.OutputExtension == nil {
		mergeInto.Filename.OutputExtension = mergeFrom.Filename.OutputExtension
	}
}

var contentConfig = new(PathConfigTree)

func readContentDir(inBase string, outBase string, dir string, conf *PathConfig, tree *PathConfigTree) {
	inPath := inBase
	if dir != "" {
		inPath += "/" + dir
	}

	log.Infof("reading input directory: %s", inPath)

	files, err := ioutil.ReadDir(inPath)
	if err != nil {
		log.Panic(err)
	}

	tree.InputPath = inPath

	// read config
	newConfig := new(PathConfig)
	log.Debug("looking for config.yml ...")
	configFile := inPath + "/config.yml"
	if _, err = os.Stat(configFile); os.IsNotExist(err) {
		log.Debug("no config.yml found in this directory, using upper configs")
		mergo.Merge(newConfig, conf)
		// remove this
		newConfig.This.GoTo = nil
		newConfig.This.Navname = nil
	} else {
		log.Debug("reading config...")
		data, err := ioutil.ReadFile(configFile)
		if err != nil {
			log.Panicf("could not read file '%s': %s", configFile, err)
		}
		err = yaml.Unmarshal(data, newConfig)
		if err != nil {
			log.Panicf("could not parse YAML file '%s': %s", configFile, err)
		}

		log.Debug("merging config with upper config")
		mergeConfig(newConfig, conf)
		log.Debug(spew.Sdump(newConfig))
	}

	tree.Config = newConfig

	// calc outDir
	stripedDir := dir
	regexStr := newConfig.Path.Strip
	if regexStr != nil && *regexStr != "" {
		if regex, err := regexp.Compile(*regexStr); err != nil {
			log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err)
		} else {
			stripedDir = regex.ReplaceAllString(stripedDir, "$1")
		}
	}
	regex := regexp.MustCompile("[^a-zA-Z]")
	stripedDir = regex.ReplaceAllString(stripedDir, "_")
	outPath := outBase + "/" + stripedDir
	outPath = path.Clean(outPath)

	log.Infof("calculated output directory: %s", outPath)
	tree.OutputPath = outPath

	if tree.Config.This.Navname == nil {
		navname := strings.Replace(stripedDir, "_", " ", -1)
		tree.Config.This.Navname = &navname
	}

	// first only files
	for _, f := range files {
		p := inPath + "/" + f.Name()
		if !f.IsDir() {
			switch path.Ext(f.Name()) {
			case ".md":
				log.Debugf(".MD %s", p)
				if tree.InputFiles == nil {
					tree.InputFiles = make([]string, 0)
				}
				tree.InputFiles = append(tree.InputFiles, f.Name())
				break
			default:
				log.Debugf("FIL %s", p)
				if tree.OtherFiles == nil {
					tree.OtherFiles = make([]string, 0)
				}
				tree.OtherFiles = append(tree.OtherFiles, f.Name())
			}
		}
	}

	// only directorys, needed config before
	for _, f := range files {
		p := inPath + "/" + f.Name()
		if f.IsDir() {
			log.Debugf("DIR %s", p)
			newTree := new(PathConfigTree)
			if tree.Sub == nil {
				tree.Sub = make([]*PathConfigTree, 0)
			}
			tree.Sub = append(tree.Sub, newTree)
			readContentDir(inPath, outPath, f.Name(), newConfig, newTree)
		}
	}
}

type navElement struct {
	Navname string
	GoTo    string

	SubMap   *map[string]*navElement
	SubSlice *[]*navElement
}

func buildNavigation(conf *PathConfigTree, curNavMap *map[string]*navElement, curNavSlice *[]*navElement, activeNav string) {
	for _, el := range conf.Sub {
		subMap := make(map[string]*navElement)
		subSlice := make([]*navElement, 0)
		navEl := navElement{
			SubMap:   &subMap,
			SubSlice: &subSlice,
		}
		elPath := strings.TrimPrefix(el.OutputPath, *outDir+"/")

		n := el.Config.This.Navname
		if n != nil {
			navEl.Navname = *n
		}
		g := el.Config.This.GoTo
		if g != nil {
			if strings.HasPrefix(*g, "/") {
				// abslute
				navEl.GoTo = *g
			} else {
				// relative
				navEl.GoTo = elPath + "/" + *g
			}
		} else {
			navEl.GoTo = elPath + "/"
		}

		if activeNav != "" && activeNav != "/" {
			// calculate relative path
			tmpPath := ""
			for i := len(strings.Split(activeNav, "/")); i > 0; i-- {
				tmpPath += "../"
			}
			navEl.GoTo = tmpPath + navEl.GoTo
			navEl.GoTo = path.Clean(navEl.GoTo)
		}

		(*curNavMap)[navEl.Navname] = &navEl
		if curNavSlice != nil {
			*curNavSlice = append(*curNavSlice, &navEl)
		}

		buildNavigation(el, &subMap, &subSlice, activeNav)
	}
}

func processContent(conf *PathConfigTree) {
	log.Debugf("trying to create output directory: %s", conf.OutputPath)

	if dirH, err := os.Stat(conf.OutputPath); os.IsNotExist(err) {
		err := os.MkdirAll(conf.OutputPath, 0755)
		if err != nil {
			log.Panicf("could not create output directory '%s': %s", conf.OutputPath, err)
		}
		log.Noticef("created output directory: %s", conf.OutputPath)
	} else if dirH != nil {
		if dirH.IsDir() {
			log.Noticef("output directory '%s' already exists", conf.OutputPath)
		} else {
			log.Panicf("output directory '%s' is no directory", conf.OutputPath)
		}
	} else {
		log.Panicf("unknown error for output directory '%s': %s", conf.OutputPath, err)
	}

	for _, file := range conf.InputFiles {
		inFile := conf.InputPath + "/" + file
		log.Debugf("reading file: %s", inFile)

		input, err := ioutil.ReadFile(inFile)
		if err != nil {
			log.Panicf("could not read '%s':%s", inFile, err)
		}
		log.Infof("processing input file '%s'", inFile)

		newConfig := new(PathConfig)

		regex := regexp.MustCompile("(?sm)^---(.*)^---")
		yamlData := regex.Find(input)
		if string(yamlData) != "" {
			log.Debugf("found yaml header in '%s', merging config", inFile)
			err = yaml.Unmarshal(yamlData, newConfig)
			if err != nil {
				log.Panicf("could not parse YAML header from '%s': %s", inFile, err)
			}

			log.Debug("merging config with upper config")
			mergeConfig(newConfig, conf.Config)
			log.Debug(spew.Sdump(newConfig))

			input = regex.ReplaceAll(input, []byte(""))
		} else {
			mergo.Merge(newConfig, conf.Config)
		}

		// ignore ???
		ignoreFile := false
		ignoreRegex := newConfig.Filename.Ignore
		if ignoreRegex != nil && *ignoreRegex != "" {
			regex, err := regexp.Compile(*ignoreRegex)
			if err != nil {
				log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
			}
			ignoreFile = regex.MatchString(file)
		}

		if ignoreFile {
			log.Infof("ignoring file '%s', because of filename.ignore", inFile)
		} else {

			// build output filename
			outputFilename := file

			indexInputFile := conf.Config.Index.InputFile
			indexOutputFile := conf.Config.Index.OutputFile

			if indexInputFile != nil && *indexInputFile == file && indexOutputFile != nil && *indexOutputFile != "" {
				outputFilename = *indexOutputFile
			} else {
				stripRegex := newConfig.Filename.Strip
				if stripRegex != nil && *stripRegex != "" {
					regex, err := regexp.Compile(*stripRegex)
					if err != nil {
						log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
					}
					outputFilename = regex.ReplaceAllString(outputFilename, "$1")
				}
				outputExt := newConfig.Filename.OutputExtension
				if outputExt != nil && *outputExt != "" {
					outputFilename += "." + *outputExt
				}
			}

			outFile := conf.OutputPath + "/" + outputFilename
			log.Debugf("using '%s' as output file", outFile)

			//html := blackfriday.Run(input, blackfriday.WithRenderer(bfchroma.NewRenderer()))
			html := blackfriday.Run(input)

			log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
			templateFile := *inDir + "/templates/" + *newConfig.Template
			template := templateCache[templateFile]
			if template == nil {
				var err error
				if template, err = raymond.ParseFile(templateFile); err != nil {
					log.Panicf("could not parse template '%s': %s", templateFile, err)
				} else {
					templateCache[templateFile] = template
				}
			}

			// build navigation
			curNavPath := strings.TrimPrefix(conf.OutputPath, *outDir+"/")
			var navMap = make(map[string]*navElement)
			buildNavigation(contentConfig, &navMap, nil, curNavPath)

			// read yaml header as data for template
			ctx := make(map[string]interface{})
			ctx["Meta"] = newConfig.Meta
			ctx["Data"] = newConfig.Data
			ctx["NavMap"] = navMap
			ctx["Body"] = string(html)

			result, err := template.Exec(ctx)
			if err != nil {
				log.Panicf("could not execute template '%s' for input file '%s': %s", templateFile, inFile, err)
			}

			if find := globalConfig.Assets.FixTemplate.Find; find != "" {
				log.Debugf("fixing assets paths in '%s' for '%s'", templateFile, inFile)
				tmpPath := ""
				for i := len(strings.Split(curNavPath, "/")); i > 0; i-- {
					tmpPath += "../"
				}
				regex, err := regexp.Compile(find)
				if err != nil {
					log.Panicf("could not compile regexp '%s' for assets path: %s", find, err)
				}
				repl := globalConfig.Assets.FixTemplate.Replace
				repl = tmpPath + "/" + globalConfig.Assets.ToPath + "/" + repl
				repl = path.Clean(repl) + "/"
				result = regex.ReplaceAllString(result, repl)
			}

			log.Noticef("writing to output file: %s", outFile)
			err = ioutil.WriteFile(outFile, []byte(result), 0644)
			if err != nil {
				log.Panicf("could not write to output file '%s': %s", outFile, err)
			}

			//fmt.Println(string(html))
		}
	}

	// process other files, copy...
	for _, file := range conf.OtherFiles {
		switch globalConfig.OtherFiles.Action {
		case "copy":
			from := conf.InputPath + "/" + file
			to := conf.OutputPath + "/" + file
			log.Noticef("copying file from '%s' to '%s'", from, to)
			err := cpy.Copy(from, to)
			if err != nil {
				log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
			}
		}
	}

	for _, el := range conf.Sub {
		processContent(el)
	}
}

func processAssets() {
	switch globalConfig.Assets.Action {
	case "copy":
		from := globalConfig.Assets.FromPath
		to := globalConfig.Assets.ToPath
		if !strings.HasPrefix(from, "/") {
			from = *inDir + "/" + from
		}
		if !strings.HasPrefix(to, "/") {
			to = *outDir + "/" + to
		}
		log.Noticef("copying assets from '%s' to '%s'", from, to)
		err := cpy.Copy(from, to)
		if err != nil {
			log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
		}
	}
}

func main() {
	spew.Config.DisablePointerAddresses = true
	spew.Config.DisableCapacities = true
	spew.Config.DisableMethods = true
	spew.Config.DisablePointerMethods = true

	inDir = flag.String("in", "./", "input directory")
	outDir = flag.String("out", "html", "output directory")
	createOutDir := flag.Bool("create", false, "create output directory if not existing")
	//clearOutDir := flag.Bool("clear", false, "clear output directory before generating website")
	logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error")

	flag.Parse()

	logBackend := logging.NewLogBackend(os.Stderr, "", 0)
	logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
		`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
	))
	logBackendLeveled := logging.AddModuleLevel(logBackendFormatter)
	logBackendLevel := logging.INFO
	if logLevel != nil {
		switch *logLevel {
		case "debug":
			logBackendLevel = logging.DEBUG
			break

		case "info":
			logBackendLevel = logging.INFO
			break

		case "warning":
			logBackendLevel = logging.WARNING
			break

		case "error":
			logBackendLevel = logging.ERROR
			break

		}
	}
	logBackendLeveled.SetLevel(logBackendLevel, "")
	logging.SetBackend(logBackendLeveled)

	if inDir == nil || *inDir == "" {
		log.Panic("input directory not specified")
	}
	log.Infof("input directory: %s", *inDir)

	if outDir == nil || *outDir == "" {
		log.Panic("output directory not specified")
	}
	log.Infof("output directory: %s", *outDir)

	if createOutDir != nil && *createOutDir {
		if _, err := os.Stat(*outDir); os.IsNotExist(err) {
			log.Debugf("output directory '%s' does not exist", *outDir)
			log.Debugf("trying to create output directory: %s", *outDir)
			err := os.MkdirAll(*outDir, 0755)
			if err != nil {
				log.Panic(err)
			}
			log.Noticef("created output directory: %s", *outDir)
		} else {
			log.Noticef("output directory '%s' already exists", *outDir)
		}
	}

	if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
		log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
	} else {
		if fD == nil {
			log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
		} else if !fD.IsDir() {
			log.Panicf("output directory '%s' is not a directory", *outDir)
		}
	}

	log.Debug("reading global config...")
	p := *inDir + "/config.yml"
	data, err := ioutil.ReadFile(p)
	if err != nil {
		log.Panicf("could not read file '%s': %s", p, err)
	}
	err = yaml.Unmarshal(data, globalConfig)
	if err != nil {
		log.Panicf("could not parse YAML file '%s': %s", p, err)
	}

	log.Debug(spew.Sdump(globalConfig))

	log.Debugf("reading input directory %s", *inDir)

	defaultTemplate := "index.html"
	defaultInputFile := "README.md"
	defaultOutputFile := "index.html"
	defaultPathStrip := "^[0-9]*_(.*)"
	defaultPathIgnoreForNav := "^_"
	defaultFilenameStrip := "(.*).md$"
	defaultFilenameIgnore := "^_"
	defaultFilenameOutputExtension := "html"

	defaultPathConfig := new(PathConfig)
	defaultPathConfig.Template = &defaultTemplate
	defaultPathConfig.Index.InputFile = &defaultInputFile
	defaultPathConfig.Index.OutputFile = &defaultOutputFile
	defaultPathConfig.Path.Strip = &defaultPathStrip
	defaultPathConfig.Path.IgnoreForNav = &defaultPathIgnoreForNav
	defaultPathConfig.Filename.Strip = &defaultFilenameStrip
	defaultPathConfig.Filename.Ignore = &defaultFilenameIgnore
	defaultPathConfig.Filename.OutputExtension = &defaultFilenameOutputExtension

	readContentDir(*inDir+"/content", *outDir, "", defaultPathConfig, contentConfig)
	//spew.Dump(contentConfig)

	//spew.Dump(navMap)

	partialsPath := *inDir + "/templates/partials"
	if d, err := os.Stat(partialsPath); !os.IsNotExist(err) {
		if d != nil && d.IsDir() {
			log.Debugf("register template partials from '%s'", partialsPath)
			if entries, err := ioutil.ReadDir(partialsPath); err == nil {
				for _, f := range entries {
					if !f.IsDir() {
						pFile := partialsPath + "/" + f.Name()
						log.Infof("registering partial: %s", pFile)
						pContent, err := ioutil.ReadFile(pFile)
						if err != nil {
							log.Panicf("could not read partial '%s': %s", pFile, err)
						}
						raymond.RegisterPartial(f.Name(), string(pContent))
					}
				}
			} else {
				log.Panicf("could not read from partials directory '%s': %s", partialsPath, err)
			}
		} else if err == nil {
			log.Panicf("template partials directory '%s' is not a directory", partialsPath)
		} else {
			log.Panicf("unknown error on partials directory '%s': %s", partialsPath, err)
		}

	}

	processContent(contentConfig)

	processAssets()

}