package helper import ( "bytes" "encoding/json" "io/ioutil" "os" "path" "regexp" "strings" "gitbase.de/apairon/mark2web/config" "github.com/Depado/bfchroma" "github.com/davecgh/go-spew/spew" "github.com/extemporalgenome/slug" "github.com/flosch/pongo2" cpy "github.com/otiai10/copy" "gopkg.in/russross/blackfriday.v2" "gopkg.in/yaml.v2" ) func newContext() map[string]interface{} { return map[string]interface{}{ "fnRequest": RequestFn, "fnRender": RenderFn, "AssetsPath": config.Config.Assets.ToPath, } } func fillNodeConfig(node *config.PathConfigTree, inBase, outBase, dir string, conf *config.PathConfig) { inPath := inBase if dir != "" { inPath += "/" + dir } Log.Infof("reading input directory: %s", inPath) node.InputPath = inPath // read config newConfig := new(config.PathConfig) Log.Debug("looking for config.yml ...") configFile := inPath + "/config.yml" if _, err := os.Stat(configFile); os.IsNotExist(err) { Log.Debug("no config.yml found in this directory, using upper configs") config.Merge(newConfig, conf) // remove this newConfig.This = config.ThisPathConfig{} } else { Log.Debug("reading config...") data, err := ioutil.ReadFile(configFile) if err != nil { Log.Panicf("could not read file '%s': %s", configFile, err) } err = yaml.Unmarshal(data, newConfig) if err != nil { Log.Panicf("could not parse YAML file '%s': %s", configFile, err) } Log.Debug("merging config with upper config") oldThis := newConfig.This config.Merge(newConfig, conf) newConfig.This = oldThis Log.Debug(spew.Sdump(newConfig)) } node.Config = newConfig // calc outDir stripedDir := dir var regexStr *string if newConfig.Path != nil { regexStr = newConfig.Path.Strip } if regexStr != nil && *regexStr != "" { if regex, err := regexp.Compile(*regexStr); err != nil { Log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err) } else { stripedDir = regex.ReplaceAllString(stripedDir, "$1") } } if node.Config.This.Navname == nil { navname := strings.Replace(stripedDir, "_", " ", -1) node.Config.This.Navname = &navname } stripedDir = slug.Slug(stripedDir) outPath := outBase + "/" + stripedDir outPath = path.Clean(outPath) Log.Infof("calculated output directory: %s", outPath) node.OutputPath = outPath // handle collections for _, colConfig := range newConfig.This.Collections { if colConfig != nil { if colConfig.Name == nil || *colConfig.Name == "" { Log.Panicf("missing Name in collection config in '%s'", inPath) } if colConfig.EntriesJSON == nil || *colConfig.EntriesJSON == "" { Log.Panicf("missing EntriesJSON in collection config in '%s'", inPath) } } if node.ColMap == nil { node.ColMap = make(map[string]interface{}) } ctx := newContext() jsonStr, err := pongo2.RenderTemplateString(*colConfig.EntriesJSON, ctx) if err != nil { Log.Panicf("invalid template string in '%s': %s", inPath, err) } var colSlice []map[string]interface{} err = json.Unmarshal([]byte(jsonStr), &colSlice) if err != nil { Log.Panicf("invalid JSON in EntriesJSON in '%s': %s", inPath, err) } node.ColMap[*colConfig.Name] = colSlice //node.ColMap[*colConfig.Name] = jsonWebRequest(*colConfig.URL) if navT := colConfig.NavTemplate; navT != nil { tpl := navT.Template if tpl == "" { tpl = *newConfig.Template } // build navigation with detail sites for _, colEl := range colSlice { ctxE := make(map[string]interface{}) err := config.Merge(&ctxE, ctx) if err != nil { Log.Panicf("could not merge context in '%s': %s", inPath, err) } err = config.Merge(&ctxE, colEl) if err != nil { Log.Panicf("could not merge context in '%s': %s", inPath, err) } goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE) if err != nil { Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", inPath, err) } goTo = strings.Trim(goTo, "/") goTo = path.Clean(goTo) if strings.Contains(goTo, "..") { Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", inPath, goTo) } if goTo == "." { Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", inPath) } if goTo == "" { Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", inPath) } navname := "" if navT.Navname != "" { navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE) if err != nil { Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", inPath, err) } } body := "" if navT.Body != "" { body, err = pongo2.RenderTemplateString(navT.Body, ctxE) if err != nil { Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", inPath, err) } } add2Nav(node, node.Config, tpl, goTo, navname, colEl, navT.DataKey, body, navT.Hidden) } } } } // ReadContentDir walks through content directory and builds the tree of configurations func ReadContentDir(inBase string, outBase string, dir string, conf *config.PathConfig, tree *config.PathConfigTree) { fillNodeConfig(tree, inBase, outBase, dir, conf) files, err := ioutil.ReadDir(tree.InputPath) if err != nil { Log.Panic(err) } // first only files for _, f := range files { p := tree.InputPath + "/" + f.Name() if !f.IsDir() && f.Name() != "config.yml" { switch path.Ext(f.Name()) { case ".md": Log.Debugf(".MD %s", p) if tree.InputFiles == nil { tree.InputFiles = make([]string, 0) } tree.InputFiles = append(tree.InputFiles, f.Name()) break default: Log.Debugf("FIL %s", p) if tree.OtherFiles == nil { tree.OtherFiles = make([]string, 0) } tree.OtherFiles = append(tree.OtherFiles, f.Name()) } } } // only directorys, needed config before for _, f := range files { p := tree.InputPath + "/" + f.Name() if f.IsDir() { Log.Debugf("DIR %s", p) newTree := new(config.PathConfigTree) if tree.Sub == nil { tree.Sub = make([]*config.PathConfigTree, 0) } tree.Sub = append(tree.Sub, newTree) ReadContentDir(tree.InputPath, tree.OutputPath, f.Name(), tree.Config, newTree) } } } // ProcessContent walks recursivly through the input paths and processes all files for output func ProcessContent(rootConf, conf *config.PathConfigTree) { CreateDirectory(conf.OutputPath) curNavPath := strings.TrimPrefix(conf.OutputPath, config.Config.Directories.Output) curNavPath = strings.TrimPrefix(curNavPath, "/") curNavPath = path.Clean(curNavPath) if curNavPath == "." { curNavPath = "" } goTo := conf.Config.This.GoTo if goTo != nil && *goTo != "" { goToFixed := *goTo if strings.HasPrefix(goToFixed, "/") { goToFixed = BackToRoot(curNavPath) + goToFixed } goToFixed = path.Clean(goToFixed) switch config.Config.Webserver.Type { case "apache": htaccessFile := conf.OutputPath + "/.htaccess" Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed) err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L] `), 0644) if err != nil { Log.Panicf("could not write '%s': %s", htaccessFile, err) } break } } for _, file := range conf.InputFiles { var input []byte inFile := "InputString" if file != "" { inFile = conf.InputPath + "/" + file Log.Debugf("reading file: %s", inFile) var err error input, err = ioutil.ReadFile(inFile) if err != nil { Log.Panicf("could not read '%s':%s", inFile, err) } Log.Infof("processing input file '%s'", inFile) } else { // use input string if available and input filename == "" var inputString *string if i := conf.Config.Index; i != nil { inputString = i.InputString } if inputString != nil { Log.Debugf("using input string instead of file") input = []byte(*inputString) } } newConfig := new(config.PathConfig) regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?") yamlData := regex.Find(input) if string(yamlData) != "" { // replace tabs yamlData = bytes.Replace(yamlData, []byte("\t"), []byte(" "), -1) Log.Debugf("found yaml header in '%s', merging config", inFile) err := yaml.Unmarshal(yamlData, newConfig) if err != nil { Log.Panicf("could not parse YAML header from '%s': %s", inFile, err) } Log.Debug("merging config with upper config") oldThis := newConfig.This config.Merge(newConfig, conf.Config) newConfig.This = oldThis Log.Debug(spew.Sdump(newConfig)) input = regex.ReplaceAll(input, []byte("")) } else { config.Merge(newConfig, conf.Config) } // ignore ??? ignoreFile := false var ignoreRegex *string var stripRegex *string var outputExt *string if f := newConfig.Filename; f != nil { ignoreRegex = f.Ignore stripRegex = f.Strip outputExt = f.OutputExtension } if ignoreRegex != nil && *ignoreRegex != "" { regex, err := regexp.Compile(*ignoreRegex) if err != nil { Log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err) } ignoreFile = regex.MatchString(file) } if ignoreFile { Log.Infof("ignoring file '%s', because of filename.ignore", inFile) } else { // build output filename outputFilename := file var indexInputFile *string var indexOutputFile *string if i := newConfig.Index; i != nil { indexInputFile = i.InputFile indexOutputFile = i.OutputFile } if indexInputFile != nil && *indexInputFile == file && indexOutputFile != nil && *indexOutputFile != "" { outputFilename = *indexOutputFile } else { if stripRegex != nil && *stripRegex != "" { regex, err := regexp.Compile(*stripRegex) if err != nil { Log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err) } outputFilename = regex.ReplaceAllString(outputFilename, "$1") } if outputExt != nil && *outputExt != "" { outputFilename += "." + *outputExt } } outFile := conf.OutputPath + "/" + outputFilename Log.Debugf("using '%s' as output file", outFile) // use --- for splitting document in markdown parts regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?") inputParts := regex.Split(string(input), -1) htmlParts := make([]*pongo2.Value, 0) for _, iPart := range inputParts { htmlParts = append(htmlParts, pongo2.AsSafeValue( string(renderMarkdown([]byte(iPart), newConfig.Markdown)))) } // build navigation navMap := make(map[string]*NavElement) navSlice := make([]*NavElement, 0) navActive := make([]*NavElement, 0) BuildNavigation(rootConf, &navMap, &navSlice, &navActive, curNavPath) // read yaml header as data for template ctx := newContext() ctx["This"] = newConfig.This ctx["Meta"] = newConfig.Meta ctx["Data"] = newConfig.Data ctx["ColMap"] = rootConf.ColMap // root as NavMap and NavSlice, for sub go to NavElement.ColMap ctx["NavMap"] = navMap ctx["NavSlice"] = navSlice ctx["NavActive"] = navActive ctx["Body"] = pongo2.AsSafeValue(string(renderMarkdown(input, newConfig.Markdown))) ctx["BodyParts"] = htmlParts ctx["CurrentPath"] = curNavPath // set active nav element if len(navActive) > 0 { ctx["NavElement"] = navActive[len(navActive)-1] } else { // if no active path to content, we are in root dir ctx["NavElement"] = &NavElement{ GoTo: BackToRoot(curNavPath), Active: true, ColMap: rootConf.ColMap, Data: rootConf.Config.Data, This: rootConf.Config.This, SubMap: &navMap, SubSlice: &navSlice, } } Log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile) templateFilename := *newConfig.Template result, err := RenderTemplate(*newConfig.Template, conf, newConfig, &ctx) if err != nil { Log.Panicf("could not execute template '%s' for input file '%s': %s", templateFilename, inFile, err) } result = FixAssetsPath(result, curNavPath) Log.Noticef("writing to output file: %s", outFile) err = ioutil.WriteFile(outFile, []byte(result), 0644) if err != nil { Log.Panicf("could not write to output file '%s': %s", outFile, err) } //fmt.Println(string(html)) } } // process other files, copy... for _, file := range conf.OtherFiles { switch config.Config.OtherFiles.Action { case "copy": from := conf.InputPath + "/" + file to := conf.OutputPath + "/" + file Log.Noticef("copying file from '%s' to '%s'", from, to) err := cpy.Copy(from, to) if err != nil { Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err) } } } i := 0 for i < len(conf.Sub) { ProcessContent(rootConf, conf.Sub[i]) i++ } } func renderMarkdown(input []byte, markdownConf *config.MarkdownConfig) []byte { var options []blackfriday.Option var chromaRenderer *bool var chromaStyle *string if m := markdownConf; m != nil { chromaRenderer = m.ChromaRenderer chromaStyle = m.ChromaStyle } if chromaStyle == nil { style := "monokai" chromaStyle = &style } if chromaRenderer != nil && *chromaRenderer { options = []blackfriday.Option{ blackfriday.WithRenderer( bfchroma.NewRenderer( bfchroma.Style(*chromaStyle), ), ), } } // fix \r from markdown for blackfriday input = bytes.Replace(input, []byte("\r"), []byte(""), -1) return blackfriday.Run(input, options...) }