package main import ( "bytes" "flag" "fmt" "io/ioutil" "os" "path" "reflect" "regexp" "strings" "github.com/imdario/mergo" "gitbase.de/apairon/mark2web/helper" "github.com/Depado/bfchroma" "github.com/davecgh/go-spew/spew" "github.com/flosch/pongo2" "github.com/gosimple/slug" "github.com/op/go-logging" cpy "github.com/otiai10/copy" "gopkg.in/russross/blackfriday.v2" "gopkg.in/yaml.v2" ) var ( // Version is the app's version string Version = "UNKNOWN" // GitHash is the current git hash for this version GitHash = "UNKNOWN" // BuildTime is the time of build of this app BuildTime = "UNKNOWN" ) var log = logging.MustGetLogger("myLogger") var inDir *string var outDir *string var templateCache = make(map[string]*pongo2.Template) // GlobalConfig is config which is used only once in root dir type GlobalConfig struct { Webserver struct { Type string `yaml:"Type"` } `yaml:"Webserver"` Assets struct { FromPath string `yaml:"FromPath"` ToPath string `yaml:"ToPath"` Action string `yaml:"Action"` FixTemplate struct { Find string `yaml:"Find"` Replace string `yaml:"Replace"` } `yaml:"FixTemplate"` } `yaml:"Assets"` OtherFiles struct { Action string `yaml:"Action"` } `yaml:"OtherFiles"` } var globalConfig = new(GlobalConfig) // ThisPathConfig is struct for This in paths yaml type ThisPathConfig struct { Navname *string `yaml:"Navname"` GoTo *string `yaml:"GoTo"` Data interface{} `yaml:"Data"` } type indexStruct struct { InputFile *string `yaml:"InputFile"` OutputFile *string `yaml:"OutputFile"` } type metaStruct struct { Title *string `yaml:"Title"` Description *string `yaml:"Description"` Keywords *string `yaml:"Keywords"` } type pathStruct struct { Strip *string `yaml:"Strip"` IgnoreForNav *string `yaml:"IgnoreForNav"` } type filenameStruct struct { Strip *string `yaml:"Strip"` Ignore *string `yaml:"Ignore"` OutputExtension *string `yaml:"OutputExtension"` } type markdownStruct struct { ChromaRenderer *bool `yaml:"ChromaRenderer"` ChromaStyle *string `yaml:"ChromaStyle"` } // PathConfig of subdir type PathConfig struct { This ThisPathConfig `yaml:"This"` Template *string `yaml:"Template"` Index *indexStruct `yaml:"Index"` Meta *metaStruct `yaml:"Meta"` Path *pathStruct `yaml:"Path"` Filename *filenameStruct `yaml:"Filename"` Markdown *markdownStruct `yaml:"Markdown"` Data interface{} `yaml:"Data"` } // PathConfigTree is complete config tree of content dir type PathConfigTree struct { InputPath string OutputPath string InputFiles []string OtherFiles []string Config *PathConfig Sub []*PathConfigTree } var contentConfig = new(PathConfigTree) type ptrTransformer struct{} func (t ptrTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error { if typ.Kind() == reflect.Ptr { return func(dst, src reflect.Value) error { if dst.CanSet() { if dst.IsNil() { dst.Set(src) } } return nil } } return nil } func merge(dst, src interface{}) error { return mergo.Merge(dst, src, mergo.WithTransformers(ptrTransformer{})) } func backToRoot(curNavPath string) string { tmpPath := "" if curNavPath != "" { for i := strings.Count(curNavPath, "/") + 1; i > 0; i-- { tmpPath += "../" } } return tmpPath } func readContentDir(inBase string, outBase string, dir string, conf *PathConfig, tree *PathConfigTree) { inPath := inBase if dir != "" { inPath += "/" + dir } log.Infof("reading input directory: %s", inPath) files, err := ioutil.ReadDir(inPath) if err != nil { log.Panic(err) } tree.InputPath = inPath // read config newConfig := new(PathConfig) log.Debug("looking for config.yml ...") configFile := inPath + "/config.yml" if _, err = os.Stat(configFile); os.IsNotExist(err) { log.Debug("no config.yml found in this directory, using upper configs") merge(newConfig, conf) // remove this newConfig.This = ThisPathConfig{} } else { log.Debug("reading config...") data, err := ioutil.ReadFile(configFile) if err != nil { log.Panicf("could not read file '%s': %s", configFile, err) } err = yaml.Unmarshal(data, newConfig) if err != nil { log.Panicf("could not parse YAML file '%s': %s", configFile, err) } log.Debug("merging config with upper config") oldThis := newConfig.This merge(newConfig, conf) newConfig.This = oldThis log.Debug(spew.Sdump(newConfig)) } tree.Config = newConfig // calc outDir stripedDir := dir var regexStr *string if newConfig.Path != nil { regexStr = newConfig.Path.Strip } if regexStr != nil && *regexStr != "" { if regex, err := regexp.Compile(*regexStr); err != nil { log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err) } else { stripedDir = regex.ReplaceAllString(stripedDir, "$1") } } if tree.Config.This.Navname == nil { navname := strings.Replace(stripedDir, "_", " ", -1) tree.Config.This.Navname = &navname } stripedDir = slug.Make(stripedDir) outPath := outBase + "/" + stripedDir outPath = path.Clean(outPath) log.Infof("calculated output directory: %s", outPath) tree.OutputPath = outPath // first only files for _, f := range files { p := inPath + "/" + f.Name() if !f.IsDir() && f.Name() != "config.yml" { switch path.Ext(f.Name()) { case ".md": log.Debugf(".MD %s", p) if tree.InputFiles == nil { tree.InputFiles = make([]string, 0) } tree.InputFiles = append(tree.InputFiles, f.Name()) break default: log.Debugf("FIL %s", p) if tree.OtherFiles == nil { tree.OtherFiles = make([]string, 0) } tree.OtherFiles = append(tree.OtherFiles, f.Name()) } } } // only directorys, needed config before for _, f := range files { p := inPath + "/" + f.Name() if f.IsDir() { log.Debugf("DIR %s", p) newTree := new(PathConfigTree) if tree.Sub == nil { tree.Sub = make([]*PathConfigTree, 0) } tree.Sub = append(tree.Sub, newTree) readContentDir(inPath, outPath, f.Name(), newConfig, newTree) } } } type navElement struct { Navname string GoTo string Active bool Data interface{} This ThisPathConfig SubMap *map[string]*navElement SubSlice *[]*navElement } func buildNavigation(conf *PathConfigTree, curNavMap *map[string]*navElement, curNavSlice *[]*navElement, navActive *[]*navElement, activeNav string) { for _, el := range conf.Sub { var ignNav *string if p := el.Config.Path; p != nil { ignNav = p.IgnoreForNav } if ignNav != nil && *ignNav != "" { regex, err := regexp.Compile(*ignNav) if err != nil { log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err) } if regex.MatchString(path.Base(el.InputPath)) { log.Debugf("ignoring input directory '%s' in navigation", el.InputPath) continue } } elPath := strings.TrimPrefix(el.OutputPath, *outDir+"/") subMap := make(map[string]*navElement) subSlice := make([]*navElement, 0) navEl := navElement{ Active: strings.HasPrefix(activeNav, elPath), Data: el.Config.Data, SubMap: &subMap, SubSlice: &subSlice, } navEl.This = el.Config.This if navEl.Active { // add to navActive level navigation currentLevel := strings.Count(activeNav, "/") if len(*navActive) <= currentLevel { // not registered *navActive = append(*navActive, &navEl) } } n := el.Config.This.Navname if n != nil { navEl.Navname = *n } g := el.Config.This.GoTo if g != nil { if strings.HasPrefix(*g, "/") { // abslute navEl.GoTo = *g } else { // relative navEl.GoTo = elPath + "/" + *g } } else { navEl.GoTo = elPath + "/" } if activeNav != "" && activeNav != "/" { // calculate relative path bToRoot := backToRoot(activeNav) navEl.GoTo = bToRoot + navEl.GoTo navEl.GoTo = path.Clean(navEl.GoTo) } (*curNavMap)[navEl.Navname] = &navEl if curNavSlice != nil { *curNavSlice = append(*curNavSlice, &navEl) } buildNavigation(el, &subMap, &subSlice, navActive, activeNav) } } func processContent(conf *PathConfigTree) { log.Debugf("trying to create output directory: %s", conf.OutputPath) if dirH, err := os.Stat(conf.OutputPath); os.IsNotExist(err) { err := os.MkdirAll(conf.OutputPath, 0755) if err != nil { log.Panicf("could not create output directory '%s': %s", conf.OutputPath, err) } log.Noticef("created output directory: %s", conf.OutputPath) } else if dirH != nil { if dirH.IsDir() { log.Noticef("output directory '%s' already exists", conf.OutputPath) } else { log.Panicf("output directory '%s' is no directory", conf.OutputPath) } } else { log.Panicf("unknown error for output directory '%s': %s", conf.OutputPath, err) } curNavPath := strings.TrimPrefix(conf.OutputPath, *outDir) curNavPath = strings.TrimPrefix(curNavPath, "/") curNavPath = path.Clean(curNavPath) if curNavPath == "." { curNavPath = "" } goTo := conf.Config.This.GoTo if goTo != nil && *goTo != "" { goToFixed := *goTo if strings.HasPrefix(goToFixed, "/") { goToFixed = backToRoot(curNavPath) + goToFixed } goToFixed = path.Clean(goToFixed) switch globalConfig.Webserver.Type { case "apache": htaccessFile := conf.OutputPath + "/.htaccess" log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed) err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L] `), 0644) if err != nil { log.Panicf("could not write '%s': %s", htaccessFile, err) } break } } for _, file := range conf.InputFiles { inFile := conf.InputPath + "/" + file log.Debugf("reading file: %s", inFile) input, err := ioutil.ReadFile(inFile) if err != nil { log.Panicf("could not read '%s':%s", inFile, err) } log.Infof("processing input file '%s'", inFile) newConfig := new(PathConfig) regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?") yamlData := regex.Find(input) if string(yamlData) != "" { log.Debugf("found yaml header in '%s', merging config", inFile) err = yaml.Unmarshal(yamlData, newConfig) if err != nil { log.Panicf("could not parse YAML header from '%s': %s", inFile, err) } log.Debug("merging config with upper config") oldThis := newConfig.This merge(newConfig, conf.Config) newConfig.This = oldThis log.Debug(spew.Sdump(newConfig)) input = regex.ReplaceAll(input, []byte("")) } else { merge(newConfig, conf.Config) } // ignore ??? ignoreFile := false var ignoreRegex *string var stripRegex *string var outputExt *string if f := newConfig.Filename; f != nil { ignoreRegex = f.Ignore stripRegex = f.Strip outputExt = f.OutputExtension } if ignoreRegex != nil && *ignoreRegex != "" { regex, err := regexp.Compile(*ignoreRegex) if err != nil { log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err) } ignoreFile = regex.MatchString(file) } if ignoreFile { log.Infof("ignoring file '%s', because of filename.ignore", inFile) } else { // build output filename outputFilename := file var indexInputFile *string var indexOutputFile *string if i := newConfig.Index; i != nil { indexInputFile = i.InputFile indexOutputFile = i.OutputFile } if indexInputFile != nil && *indexInputFile == file && indexOutputFile != nil && *indexOutputFile != "" { outputFilename = *indexOutputFile } else { if stripRegex != nil && *stripRegex != "" { regex, err := regexp.Compile(*stripRegex) if err != nil { log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err) } outputFilename = regex.ReplaceAllString(outputFilename, "$1") } if outputExt != nil && *outputExt != "" { outputFilename += "." + *outputExt } } outFile := conf.OutputPath + "/" + outputFilename log.Debugf("using '%s' as output file", outFile) var options []blackfriday.Option var chromaRenderer *bool var chromaStyle *string if m := newConfig.Markdown; m != nil { chromaRenderer = m.ChromaRenderer chromaStyle = m.ChromaStyle } if chromaStyle == nil { style := "monokai" chromaStyle = &style } if chromaRenderer != nil && *chromaRenderer { options = []blackfriday.Option{ blackfriday.WithRenderer( bfchroma.NewRenderer( bfchroma.Style(*chromaStyle), ), ), } } // fix \r from markdown for blackfriday input = bytes.Replace(input, []byte("\r"), []byte(""), -1) html := blackfriday.Run(input, options...) // use --- for splitting document in markdown parts regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?") inputParts := regex.Split(string(input), -1) htmlParts := make([]*pongo2.Value, 0) for _, iPart := range inputParts { htmlParts = append(htmlParts, pongo2.AsSafeValue(string(blackfriday.Run([]byte(iPart), options...)))) } log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile) templateFile := *inDir + "/templates/" + *newConfig.Template template := templateCache[templateFile] if template == nil { var err error if template, err = pongo2.FromFile(templateFile); err != nil { log.Panicf("could not parse template '%s': %s", templateFile, err) } else { templateCache[templateFile] = template } } // build navigation navMap := make(map[string]*navElement) navSlice := make([]*navElement, 0) navActive := make([]*navElement, 0) buildNavigation(contentConfig, &navMap, &navSlice, &navActive, curNavPath) // read yaml header as data for template ctx := make(pongo2.Context) ctx["This"] = newConfig.This ctx["Meta"] = newConfig.Meta ctx["Data"] = newConfig.Data ctx["NavMap"] = navMap ctx["NavSlice"] = navSlice ctx["NavActive"] = navActive ctx["Body"] = pongo2.AsSafeValue(string(html)) ctx["BodyParts"] = htmlParts // register functions ctx["fnRequest"] = helper.Request result, err := template.Execute(ctx) if err != nil { log.Panicf("could not execute template '%s' for input file '%s': %s", templateFile, inFile, err) } if find := globalConfig.Assets.FixTemplate.Find; find != "" { log.Debugf("fixing assets paths in '%s' for '%s'", templateFile, inFile) bToRoot := backToRoot(curNavPath) regex, err := regexp.Compile(find) if err != nil { log.Panicf("could not compile regexp '%s' for assets path: %s", find, err) } repl := globalConfig.Assets.FixTemplate.Replace repl = bToRoot + globalConfig.Assets.ToPath + "/" + repl repl = path.Clean(repl) + "/" log.Debugf("new assets paths: %s", repl) result = regex.ReplaceAllString(result, repl) } log.Noticef("writing to output file: %s", outFile) err = ioutil.WriteFile(outFile, []byte(result), 0644) if err != nil { log.Panicf("could not write to output file '%s': %s", outFile, err) } //fmt.Println(string(html)) } } // process other files, copy... for _, file := range conf.OtherFiles { switch globalConfig.OtherFiles.Action { case "copy": from := conf.InputPath + "/" + file to := conf.OutputPath + "/" + file log.Noticef("copying file from '%s' to '%s'", from, to) err := cpy.Copy(from, to) if err != nil { log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err) } } } for _, el := range conf.Sub { processContent(el) } } func processAssets() { switch globalConfig.Assets.Action { case "copy": from := globalConfig.Assets.FromPath to := globalConfig.Assets.ToPath if !strings.HasPrefix(from, "/") { from = *inDir + "/" + from } if !strings.HasPrefix(to, "/") { to = *outDir + "/" + to } log.Noticef("copying assets from '%s' to '%s'", from, to) err := cpy.Copy(from, to) if err != nil { log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err) } } } func main() { spew.Config.DisablePointerAddresses = true spew.Config.DisableCapacities = true spew.Config.DisableMethods = true spew.Config.DisablePointerMethods = true inDir = flag.String("in", "./", "input directory") outDir = flag.String("out", "html", "output directory") createOutDir := flag.Bool("create", false, "create output directory if not existing") //clearOutDir := flag.Bool("clear", false, "clear output directory before generating website") logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error") version := flag.Bool("version", false, "print version of this executable") flag.Parse() if version != nil && *version { fmt.Printf(`%11s: %s %11s: %s %11s: %s `, "version", Version, "git hash", GitHash, "build time", BuildTime) os.Exit(0) } logBackend := logging.NewLogBackend(os.Stderr, "", 0) logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter( `%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`, )) logBackendLeveled := logging.AddModuleLevel(logBackendFormatter) logBackendLevel := logging.INFO if logLevel != nil { switch *logLevel { case "debug": logBackendLevel = logging.DEBUG break case "info": logBackendLevel = logging.INFO break case "notice": logBackendLevel = logging.NOTICE break case "warning": logBackendLevel = logging.WARNING break case "error": logBackendLevel = logging.ERROR break } } logBackendLeveled.SetLevel(logBackendLevel, "") logging.SetBackend(logBackendLeveled) if inDir == nil || *inDir == "" { log.Panic("input directory not specified") } log.Infof("input directory: %s", *inDir) if outDir == nil || *outDir == "" { log.Panic("output directory not specified") } log.Infof("output directory: %s", *outDir) if createOutDir != nil && *createOutDir { if _, err := os.Stat(*outDir); os.IsNotExist(err) { log.Debugf("output directory '%s' does not exist", *outDir) log.Debugf("trying to create output directory: %s", *outDir) err := os.MkdirAll(*outDir, 0755) if err != nil { log.Panic(err) } log.Noticef("created output directory: %s", *outDir) } else { log.Noticef("output directory '%s' already exists", *outDir) } } if fD, err := os.Stat(*outDir); os.IsNotExist(err) { log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir) } else { if fD == nil { log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir) } else if !fD.IsDir() { log.Panicf("output directory '%s' is not a directory", *outDir) } } log.Debug("reading global config...") p := *inDir + "/config.yml" data, err := ioutil.ReadFile(p) if err != nil { log.Panicf("could not read file '%s': %s", p, err) } err = yaml.Unmarshal(data, globalConfig) if err != nil { log.Panicf("could not parse YAML file '%s': %s", p, err) } log.Debug(spew.Sdump(globalConfig)) log.Debugf("reading input directory %s", *inDir) defaultTemplate := "base.html" defaultInputFile := "README.md" defaultOutputFile := "index.html" defaultPathStrip := "^[0-9]*_(.*)" defaultPathIgnoreForNav := "^_" defaultFilenameStrip := "(.*).md$" defaultFilenameIgnore := "^_" defaultFilenameOutputExtension := "html" defaultPathConfig := new(PathConfig) defaultPathConfig.Template = &defaultTemplate defaultPathConfig.Index = &indexStruct{ InputFile: &defaultInputFile, OutputFile: &defaultOutputFile, } defaultPathConfig.Path = &pathStruct{ Strip: &defaultPathStrip, IgnoreForNav: &defaultPathIgnoreForNav, } defaultPathConfig.Filename = &filenameStruct{ Strip: &defaultFilenameStrip, Ignore: &defaultFilenameIgnore, OutputExtension: &defaultFilenameOutputExtension, } readContentDir(*inDir+"/content", *outDir, "", defaultPathConfig, contentConfig) //spew.Dump(contentConfig) //spew.Dump(navMap) processContent(contentConfig) processAssets() }