package main import ( "flag" "io/ioutil" "os" "path" "regexp" "strings" "github.com/imdario/mergo" "github.com/davecgh/go-spew/spew" "github.com/flosch/pongo2" "github.com/op/go-logging" cpy "github.com/otiai10/copy" "gopkg.in/russross/blackfriday.v2" "gopkg.in/yaml.v2" ) var log = logging.MustGetLogger("myLogger") var inDir *string var outDir *string var templateCache = make(map[string]*pongo2.Template) // GlobalConfig is config which is used only once in root dir type GlobalConfig struct { Webserver struct { Type string `yaml:"Type"` } `yaml:"Webserver"` Assets struct { FromPath string `yaml:"FromPath"` ToPath string `yaml:"ToPath"` Action string `yaml:"Action"` FixTemplate struct { Find string `yaml:"Find"` Replace string `yaml:"Replace"` } `yaml:"FixTemplate"` } `yaml:"Assets"` OtherFiles struct { Action string `yaml:"Action"` } `yaml:"OtherFiles"` } var globalConfig = new(GlobalConfig) type ThisPathConfig struct { Navname *string `yaml:"Navname"` GoTo *string `yaml:"GoTo"` Data interface{} `yaml:"Data"` } // PathConfig of subdir type PathConfig struct { This ThisPathConfig `yaml:"This"` Template *string `yaml:"Template"` Index struct { InputFile *string `yaml:"InputFile"` OutputFile *string `yaml:"OutputFile"` } `yaml:"Index"` Meta struct { Title *string `yaml:"Title"` Description *string `yaml:"Description"` Keywords *string `yaml:"Keywords"` } `yaml:"Meta"` Path struct { Strip *string `yaml:"Strip"` IgnoreForNav *string `yaml:"IgnoreForNav"` } `yaml:"Path"` Filename struct { Strip *string `yaml:"Strip"` Ignore *string `yaml:"Ignore"` OutputExtension *string `yaml:"OutputExtension"` } `yaml:"Filename"` Data interface{} `yaml:"Data"` } // PathConfigTree is complete config tree of content dir type PathConfigTree struct { InputPath string OutputPath string InputFiles []string OtherFiles []string Config *PathConfig Sub []*PathConfigTree } var contentConfig = new(PathConfigTree) func readContentDir(inBase string, outBase string, dir string, conf *PathConfig, tree *PathConfigTree) { inPath := inBase if dir != "" { inPath += "/" + dir } log.Infof("reading input directory: %s", inPath) files, err := ioutil.ReadDir(inPath) if err != nil { log.Panic(err) } tree.InputPath = inPath // read config newConfig := new(PathConfig) log.Debug("looking for config.yml ...") configFile := inPath + "/config.yml" if _, err = os.Stat(configFile); os.IsNotExist(err) { log.Debug("no config.yml found in this directory, using upper configs") mergo.Merge(newConfig, conf) // remove this newConfig.This = ThisPathConfig{} } else { log.Debug("reading config...") data, err := ioutil.ReadFile(configFile) if err != nil { log.Panicf("could not read file '%s': %s", configFile, err) } err = yaml.Unmarshal(data, newConfig) if err != nil { log.Panicf("could not parse YAML file '%s': %s", configFile, err) } log.Debug("merging config with upper config") oldThis := newConfig.This mergo.Merge(newConfig, conf) newConfig.This = oldThis log.Debug(spew.Sdump(newConfig)) } tree.Config = newConfig // calc outDir stripedDir := dir regexStr := newConfig.Path.Strip if regexStr != nil && *regexStr != "" { if regex, err := regexp.Compile(*regexStr); err != nil { log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err) } else { stripedDir = regex.ReplaceAllString(stripedDir, "$1") } } regex := regexp.MustCompile("[^a-zA-Z]") stripedDir = regex.ReplaceAllString(stripedDir, "_") outPath := outBase + "/" + stripedDir outPath = path.Clean(outPath) log.Infof("calculated output directory: %s", outPath) tree.OutputPath = outPath if tree.Config.This.Navname == nil { navname := strings.Replace(stripedDir, "_", " ", -1) tree.Config.This.Navname = &navname } // first only files for _, f := range files { p := inPath + "/" + f.Name() if !f.IsDir() { switch path.Ext(f.Name()) { case ".md": log.Debugf(".MD %s", p) if tree.InputFiles == nil { tree.InputFiles = make([]string, 0) } tree.InputFiles = append(tree.InputFiles, f.Name()) break default: log.Debugf("FIL %s", p) if tree.OtherFiles == nil { tree.OtherFiles = make([]string, 0) } tree.OtherFiles = append(tree.OtherFiles, f.Name()) } } } // only directorys, needed config before for _, f := range files { p := inPath + "/" + f.Name() if f.IsDir() { log.Debugf("DIR %s", p) newTree := new(PathConfigTree) if tree.Sub == nil { tree.Sub = make([]*PathConfigTree, 0) } tree.Sub = append(tree.Sub, newTree) readContentDir(inPath, outPath, f.Name(), newConfig, newTree) } } } type navElement struct { Navname string GoTo string Active bool Data interface{} This ThisPathConfig SubMap *map[string]*navElement SubSlice *[]*navElement } func buildNavigation(conf *PathConfigTree, curNavMap *map[string]*navElement, curNavSlice *[]*navElement, navActive *[]*navElement, activeNav string) { for _, el := range conf.Sub { ignNav := el.Config.Path.IgnoreForNav if ignNav != nil && *ignNav != "" { regex, err := regexp.Compile(*ignNav) if err != nil { log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err) } if regex.MatchString(path.Base(el.InputPath)) { log.Debugf("ignoring input directory '%s' in navigation", el.InputPath) continue } } elPath := strings.TrimPrefix(el.OutputPath, *outDir+"/") subMap := make(map[string]*navElement) subSlice := make([]*navElement, 0) navEl := navElement{ Active: strings.HasPrefix(activeNav, elPath), Data: el.Config.Data, SubMap: &subMap, SubSlice: &subSlice, } navEl.This = el.Config.This if navEl.Active { // add to navActive level navigation currentLevel := strings.Count(activeNav, "/") if len(*navActive) <= currentLevel { // not registered *navActive = append(*navActive, &navEl) } } n := el.Config.This.Navname if n != nil { navEl.Navname = *n } g := el.Config.This.GoTo if g != nil { if strings.HasPrefix(*g, "/") { // abslute navEl.GoTo = *g } else { // relative navEl.GoTo = elPath + "/" + *g } } else { navEl.GoTo = elPath + "/" } if activeNav != "" && activeNav != "/" { // calculate relative path tmpPath := "" for i := len(strings.Split(activeNav, "/")); i > 0; i-- { tmpPath += "../" } navEl.GoTo = tmpPath + navEl.GoTo navEl.GoTo = path.Clean(navEl.GoTo) } (*curNavMap)[navEl.Navname] = &navEl if curNavSlice != nil { *curNavSlice = append(*curNavSlice, &navEl) } buildNavigation(el, &subMap, &subSlice, navActive, activeNav) } } func processContent(conf *PathConfigTree) { log.Debugf("trying to create output directory: %s", conf.OutputPath) if dirH, err := os.Stat(conf.OutputPath); os.IsNotExist(err) { err := os.MkdirAll(conf.OutputPath, 0755) if err != nil { log.Panicf("could not create output directory '%s': %s", conf.OutputPath, err) } log.Noticef("created output directory: %s", conf.OutputPath) } else if dirH != nil { if dirH.IsDir() { log.Noticef("output directory '%s' already exists", conf.OutputPath) } else { log.Panicf("output directory '%s' is no directory", conf.OutputPath) } } else { log.Panicf("unknown error for output directory '%s': %s", conf.OutputPath, err) } for _, file := range conf.InputFiles { inFile := conf.InputPath + "/" + file log.Debugf("reading file: %s", inFile) input, err := ioutil.ReadFile(inFile) if err != nil { log.Panicf("could not read '%s':%s", inFile, err) } log.Infof("processing input file '%s'", inFile) newConfig := new(PathConfig) regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?") yamlData := regex.Find(input) if string(yamlData) != "" { log.Debugf("found yaml header in '%s', merging config", inFile) err = yaml.Unmarshal(yamlData, newConfig) if err != nil { log.Panicf("could not parse YAML header from '%s': %s", inFile, err) } log.Debug("merging config with upper config") oldThis := newConfig.This mergo.Merge(newConfig, conf.Config) newConfig.This = oldThis log.Debug(spew.Sdump(newConfig)) input = regex.ReplaceAll(input, []byte("")) } else { mergo.Merge(newConfig, conf.Config) } // ignore ??? ignoreFile := false ignoreRegex := newConfig.Filename.Ignore if ignoreRegex != nil && *ignoreRegex != "" { regex, err := regexp.Compile(*ignoreRegex) if err != nil { log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err) } ignoreFile = regex.MatchString(file) } if ignoreFile { log.Infof("ignoring file '%s', because of filename.ignore", inFile) } else { // build output filename outputFilename := file indexInputFile := conf.Config.Index.InputFile indexOutputFile := conf.Config.Index.OutputFile if indexInputFile != nil && *indexInputFile == file && indexOutputFile != nil && *indexOutputFile != "" { outputFilename = *indexOutputFile } else { stripRegex := newConfig.Filename.Strip if stripRegex != nil && *stripRegex != "" { regex, err := regexp.Compile(*stripRegex) if err != nil { log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err) } outputFilename = regex.ReplaceAllString(outputFilename, "$1") } outputExt := newConfig.Filename.OutputExtension if outputExt != nil && *outputExt != "" { outputFilename += "." + *outputExt } } outFile := conf.OutputPath + "/" + outputFilename log.Debugf("using '%s' as output file", outFile) //html := blackfriday.Run(input, blackfriday.WithRenderer(bfchroma.NewRenderer())) html := blackfriday.Run(input) // use --- for splitting document in markdown parts regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?") inputParts := regex.Split(string(input), -1) htmlParts := make([]*pongo2.Value, 0) for _, iPart := range inputParts { htmlParts = append(htmlParts, pongo2.AsSafeValue(string(blackfriday.Run([]byte(iPart))))) } log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile) templateFile := *inDir + "/templates/" + *newConfig.Template template := templateCache[templateFile] if template == nil { var err error if template, err = pongo2.FromFile(templateFile); err != nil { log.Panicf("could not parse template '%s': %s", templateFile, err) } else { templateCache[templateFile] = template } } // build navigation curNavPath := strings.TrimPrefix(conf.OutputPath, *outDir) curNavPath = strings.TrimPrefix(curNavPath, "/") curNavPath = path.Clean(curNavPath) if curNavPath == "." { curNavPath = "" } navMap := make(map[string]*navElement) navSlice := make([]*navElement, 0) navActive := make([]*navElement, 0) buildNavigation(contentConfig, &navMap, &navSlice, &navActive, curNavPath) // read yaml header as data for template ctx := make(pongo2.Context) ctx["Meta"] = newConfig.Meta ctx["Data"] = newConfig.Data ctx["NavMap"] = navMap ctx["NavSlice"] = navSlice ctx["NavActive"] = navActive ctx["Body"] = pongo2.AsSafeValue(string(html)) ctx["BodyParts"] = htmlParts result, err := template.Execute(ctx) if err != nil { log.Panicf("could not execute template '%s' for input file '%s': %s", templateFile, inFile, err) } if find := globalConfig.Assets.FixTemplate.Find; find != "" { log.Debugf("fixing assets paths in '%s' for '%s'", templateFile, inFile) tmpPath := "" if curNavPath != "" { for i := len(strings.Split(curNavPath, "/")); i > 0; i-- { tmpPath += "../" } } regex, err := regexp.Compile(find) if err != nil { log.Panicf("could not compile regexp '%s' for assets path: %s", find, err) } repl := globalConfig.Assets.FixTemplate.Replace repl = tmpPath + globalConfig.Assets.ToPath + "/" + repl repl = path.Clean(repl) + "/" log.Debugf("new assets paths: %s", repl) result = regex.ReplaceAllString(result, repl) } log.Noticef("writing to output file: %s", outFile) err = ioutil.WriteFile(outFile, []byte(result), 0644) if err != nil { log.Panicf("could not write to output file '%s': %s", outFile, err) } //fmt.Println(string(html)) } } // process other files, copy... for _, file := range conf.OtherFiles { switch globalConfig.OtherFiles.Action { case "copy": from := conf.InputPath + "/" + file to := conf.OutputPath + "/" + file log.Noticef("copying file from '%s' to '%s'", from, to) err := cpy.Copy(from, to) if err != nil { log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err) } } } for _, el := range conf.Sub { processContent(el) } } func processAssets() { switch globalConfig.Assets.Action { case "copy": from := globalConfig.Assets.FromPath to := globalConfig.Assets.ToPath if !strings.HasPrefix(from, "/") { from = *inDir + "/" + from } if !strings.HasPrefix(to, "/") { to = *outDir + "/" + to } log.Noticef("copying assets from '%s' to '%s'", from, to) err := cpy.Copy(from, to) if err != nil { log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err) } } } func main() { spew.Config.DisablePointerAddresses = true spew.Config.DisableCapacities = true spew.Config.DisableMethods = true spew.Config.DisablePointerMethods = true inDir = flag.String("in", "./", "input directory") outDir = flag.String("out", "html", "output directory") createOutDir := flag.Bool("create", false, "create output directory if not existing") //clearOutDir := flag.Bool("clear", false, "clear output directory before generating website") logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error") flag.Parse() logBackend := logging.NewLogBackend(os.Stderr, "", 0) logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter( `%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`, )) logBackendLeveled := logging.AddModuleLevel(logBackendFormatter) logBackendLevel := logging.INFO if logLevel != nil { switch *logLevel { case "debug": logBackendLevel = logging.DEBUG break case "info": logBackendLevel = logging.INFO break case "warning": logBackendLevel = logging.WARNING break case "error": logBackendLevel = logging.ERROR break } } logBackendLeveled.SetLevel(logBackendLevel, "") logging.SetBackend(logBackendLeveled) if inDir == nil || *inDir == "" { log.Panic("input directory not specified") } log.Infof("input directory: %s", *inDir) if outDir == nil || *outDir == "" { log.Panic("output directory not specified") } log.Infof("output directory: %s", *outDir) if createOutDir != nil && *createOutDir { if _, err := os.Stat(*outDir); os.IsNotExist(err) { log.Debugf("output directory '%s' does not exist", *outDir) log.Debugf("trying to create output directory: %s", *outDir) err := os.MkdirAll(*outDir, 0755) if err != nil { log.Panic(err) } log.Noticef("created output directory: %s", *outDir) } else { log.Noticef("output directory '%s' already exists", *outDir) } } if fD, err := os.Stat(*outDir); os.IsNotExist(err) { log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir) } else { if fD == nil { log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir) } else if !fD.IsDir() { log.Panicf("output directory '%s' is not a directory", *outDir) } } log.Debug("reading global config...") p := *inDir + "/config.yml" data, err := ioutil.ReadFile(p) if err != nil { log.Panicf("could not read file '%s': %s", p, err) } err = yaml.Unmarshal(data, globalConfig) if err != nil { log.Panicf("could not parse YAML file '%s': %s", p, err) } log.Debug(spew.Sdump(globalConfig)) log.Debugf("reading input directory %s", *inDir) defaultTemplate := "base.html" defaultInputFile := "README.md" defaultOutputFile := "index.html" defaultPathStrip := "^[0-9]*_(.*)" defaultPathIgnoreForNav := "^_" defaultFilenameStrip := "(.*).md$" defaultFilenameIgnore := "^_" defaultFilenameOutputExtension := "html" defaultPathConfig := new(PathConfig) defaultPathConfig.Template = &defaultTemplate defaultPathConfig.Index.InputFile = &defaultInputFile defaultPathConfig.Index.OutputFile = &defaultOutputFile defaultPathConfig.Path.Strip = &defaultPathStrip defaultPathConfig.Path.IgnoreForNav = &defaultPathIgnoreForNav defaultPathConfig.Filename.Strip = &defaultFilenameStrip defaultPathConfig.Filename.Ignore = &defaultFilenameIgnore defaultPathConfig.Filename.OutputExtension = &defaultFilenameOutputExtension readContentDir(*inDir+"/content", *outDir, "", defaultPathConfig, contentConfig) //spew.Dump(contentConfig) //spew.Dump(navMap) processContent(contentConfig) processAssets() }