apairon/mark2web
1
1
Fork 0
Code Issues 8 Pull Requests Releases 5 Activity

reorganized code
Some checks failed
continuous-integration/drone/push Build is failing
Details

Browse Source
This commit is contained in:
Sebastian Frank
2019-03-18 13:34:52 +01:00
parent 6b34509d9a
commit b2e0d78a2c
120 changed files with 1034 additions and 1067 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
helper/assets.go
View File

@@ -1,28 +0,0 @@
package helper
import (
"strings"
"gitbase.de/apairon/mark2web/config"
cpy "github.com/otiai10/copy"
)
// ProcessAssets copies the assets from input to output dir
func ProcessAssets() {
switch config.Config.Assets.Action {
case "copy":
from := config.Config.Assets.FromPath
to := config.Config.Assets.ToPath
if !strings.HasPrefix(from, "/") {
from = config.Config.Directories.Input + "/" + from
}
if !strings.HasPrefix(to, "/") {
to = config.Config.Directories.Output + "/" + to
}
Log.Noticef("copying assets from '%s' to '%s'", from, to)
err := cpy.Copy(from, to)
if err != nil {
Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
}
}
}

514
helper/content.go
View File

@@ -1,514 +0,0 @@
package helper
import (
"bytes"
"io/ioutil"
"os"
"path"
"regexp"
"strings"
"time"
"gitbase.de/apairon/mark2web/config"
"github.com/Depado/bfchroma"
"github.com/davecgh/go-spew/spew"
"github.com/extemporalgenome/slug"
"github.com/flosch/pongo2"
cpy "github.com/otiai10/copy"
"gopkg.in/russross/blackfriday.v2"
"gopkg.in/yaml.v2"
)
func newContext() pongo2.Context {
return pongo2.Context{
"fnRequest": RequestFn,
"fnRender": RenderFn,
"AssetsPath": config.Config.Assets.ToPath,
"Timestamp": time.Now().Unix,
}
}
func fillNodeConfig(node *config.PathConfigTree, inBase, outBase, dir string, conf *config.PathConfig) {
inPath := inBase
if dir != "" {
inPath += "/" + dir
}
Log.Infof("reading input directory: %s", inPath)
node.InputPath = inPath
// read config
newConfig := new(config.PathConfig)
Log.Debug("looking for config.yml ...")
configFile := inPath + "/config.yml"
if _, err := os.Stat(configFile); os.IsNotExist(err) {
Log.Debug("no config.yml found in this directory, using upper configs")
config.Merge(newConfig, conf)
// remove this
newConfig.This = config.ThisPathConfig{}
} else {
Log.Debug("reading config...")
data, err := ioutil.ReadFile(configFile)
if err != nil {
Log.Panicf("could not read file '%s': %s", configFile, err)
}
err = yaml.Unmarshal(data, newConfig)
if err != nil {
Log.Panicf("could not parse YAML file '%s': %s", configFile, err)
}
Log.Debug("merging config with upper config")
oldThis := newConfig.This
config.Merge(newConfig, conf)
newConfig.This = oldThis
Log.Debug(spew.Sdump(newConfig))
}
node.Config = newConfig
// calc outDir
stripedDir := dir
var regexStr *string
if newConfig.Path != nil {
regexStr = newConfig.Path.Strip
}
if regexStr != nil && *regexStr != "" {
if regex, err := regexp.Compile(*regexStr); err != nil {
Log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+dir, err)
} else {
stripedDir = regex.ReplaceAllString(stripedDir, "$1")
}
}
if node.Config.This.Navname == nil {
navname := strings.Replace(stripedDir, "_", " ", -1)
node.Config.This.Navname = &navname
}
stripedDir = slug.Slug(stripedDir)
outPath := outBase + "/" + stripedDir
outPath = path.Clean(outPath)
Log.Infof("calculated output directory: %s", outPath)
node.OutputPath = outPath
// handle collections
for _, colConfig := range newConfig.This.Collections {
if colConfig != nil {
if colConfig.Name == nil || *colConfig.Name == "" {
Log.Panicf("missing Name in collection config in '%s'", inPath)
}
if colConfig.URL == nil || *colConfig.URL == "" {
Log.Panicf("missing EntriesJSON in collection config in '%s'", inPath)
}
}
if node.ColMap == nil {
node.ColMap = make(config.MapString)
}
ctx := newContext()
ctx["This"] = node.Config.This
ctx["Data"] = node.Config.Data
url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
if err != nil {
Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", inPath, err)
}
colData := jsonWebRequest(url)
node.ColMap[*colConfig.Name] = colData
if navT := colConfig.NavTemplate; navT != nil {
var entries []interface{}
var ok bool
if navT.EntriesAttribute != "" {
var colDataMap map[string]interface{}
if colDataMap, ok = colData.(map[string]interface{}); ok {
entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
if !ok {
Log.Debug(spew.Sdump(colDataMap))
Log.Panicf("invalid json data in [%s] from url '%s' for entries", navT.EntriesAttribute, url)
}
}
} else {
entries, ok = colData.([]interface{})
}
if !ok {
Log.Debug(spew.Sdump(colData))
Log.Panicf("invalid json data from url '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", url)
}
// build navigation with detail sites
for idx, colEl := range entries {
ctxE := make(pongo2.Context)
err := config.Merge(&ctxE, ctx)
if err != nil {
Log.Panicf("could not merge context in '%s': %s", inPath, err)
}
var jsonCtx map[string]interface{}
if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
Log.Debug(spew.Sdump(colEl))
Log.Panicf("no json object for entry index %d from url '%s'", idx, url)
}
err = config.Merge(&ctxE, pongo2.Context(jsonCtx))
if err != nil {
Log.Panicf("could not merge context in '%s': %s", inPath, err)
}
tpl := ""
if navT.Template != "" {
tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
if err != nil {
Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", inPath, err)
}
}
if tpl == "" {
tpl = *newConfig.Template
}
dataKey := ""
if navT.DataKey != "" {
dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
if err != nil {
Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", inPath, err)
}
}
goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
if err != nil {
Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", inPath, err)
}
goTo = strings.Trim(goTo, "/")
goTo = path.Clean(goTo)
if strings.Contains(goTo, "..") {
Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", inPath, goTo)
}
if goTo == "." {
Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", inPath)
}
if goTo == "" {
Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", inPath)
}
navname := ""
if navT.Navname != "" {
navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
if err != nil {
Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", inPath, err)
}
}
body := ""
if navT.Body != "" {
body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
if err != nil {
Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", inPath, err)
}
}
add2Nav(node, node.Config, tpl, goTo, navname, colEl, dataKey, body, navT.Hidden)
}
}
}
}
// ReadContentDir walks through content directory and builds the tree of configurations
func ReadContentDir(inBase string, outBase string, dir string, conf *config.PathConfig, tree *config.PathConfigTree) {
fillNodeConfig(tree, inBase, outBase, dir, conf)
files, err := ioutil.ReadDir(tree.InputPath)
if err != nil {
Log.Panic(err)
}
// first only files
for _, f := range files {
p := tree.InputPath + "/" + f.Name()
if !f.IsDir() && f.Name() != "config.yml" {
switch path.Ext(f.Name()) {
case ".md":
Log.Debugf(".MD %s", p)
if tree.InputFiles == nil {
tree.InputFiles = make([]string, 0)
}
tree.InputFiles = append(tree.InputFiles, f.Name())
break
default:
Log.Debugf("FIL %s", p)
if tree.OtherFiles == nil {
tree.OtherFiles = make([]string, 0)
}
tree.OtherFiles = append(tree.OtherFiles, f.Name())
}
}
}
// only directorys, needed config before
for _, f := range files {
p := tree.InputPath + "/" + f.Name()
if f.IsDir() {
Log.Debugf("DIR %s", p)
newTree := new(config.PathConfigTree)
if tree.Sub == nil {
tree.Sub = make([]*config.PathConfigTree, 0)
}
tree.Sub = append(tree.Sub, newTree)
ReadContentDir(tree.InputPath, tree.OutputPath, f.Name(), tree.Config, newTree)
}
}
}
// ProcessContent walks recursivly through the input paths and processes all files for output
func ProcessContent(rootConf, conf *config.PathConfigTree) {
CreateDirectory(conf.OutputPath)
curNavPath := strings.TrimPrefix(conf.OutputPath, config.Config.Directories.Output)
curNavPath = strings.TrimPrefix(curNavPath, "/")
curNavPath = path.Clean(curNavPath)
if curNavPath == "." {
curNavPath = ""
}
goTo := conf.Config.This.GoTo
if goTo != nil && *goTo != "" {
goToFixed := *goTo
if strings.HasPrefix(goToFixed, "/") {
goToFixed = BackToRoot(curNavPath) + goToFixed
}
goToFixed = path.Clean(goToFixed)
switch config.Config.Webserver.Type {
case "apache":
htaccessFile := conf.OutputPath + "/.htaccess"
Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goToFixed)
err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
RewriteRule ^$ %{REQUEST_URI}`+goToFixed+`/ [R,L]
`), 0644)
if err != nil {
Log.Panicf("could not write '%s': %s", htaccessFile, err)
}
break
}
}
for _, file := range conf.InputFiles {
var input []byte
inFile := "InputString"
if file != "" {
inFile = conf.InputPath + "/" + file
Log.Debugf("reading file: %s", inFile)
var err error
input, err = ioutil.ReadFile(inFile)
if err != nil {
Log.Panicf("could not read '%s':%s", inFile, err)
}
Log.Infof("processing input file '%s'", inFile)
} else {
// use input string if available and input filename == ""
var inputString *string
if i := conf.Config.Index; i != nil {
inputString = i.InputString
}
if inputString != nil {
Log.Debugf("using input string instead of file")
input = []byte(*inputString)
}
}
newConfig := new(config.PathConfig)
regex := regexp.MustCompile("(?s)^---(.*?)\\r?\\n\\r?---\\r?\\n\\r?")
yamlData := regex.Find(input)
if string(yamlData) != "" {
// replace tabs
yamlData = bytes.Replace(yamlData, []byte("\t"), []byte(" "), -1)
Log.Debugf("found yaml header in '%s', merging config", inFile)
err := yaml.Unmarshal(yamlData, newConfig)
if err != nil {
Log.Panicf("could not parse YAML header from '%s': %s", inFile, err)
}
Log.Debug("merging config with upper config")
oldThis := newConfig.This
config.Merge(newConfig, conf.Config)
newConfig.This = oldThis
Log.Debug(spew.Sdump(newConfig))
input = regex.ReplaceAll(input, []byte(""))
} else {
config.Merge(newConfig, conf.Config)
}
// ignore ???
ignoreFile := false
var ignoreRegex *string
var stripRegex *string
var outputExt *string
if f := newConfig.Filename; f != nil {
ignoreRegex = f.Ignore
stripRegex = f.Strip
outputExt = f.OutputExtension
}
if ignoreRegex != nil && *ignoreRegex != "" {
regex, err := regexp.Compile(*ignoreRegex)
if err != nil {
Log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
}
ignoreFile = regex.MatchString(file)
}
if ignoreFile {
Log.Infof("ignoring file '%s', because of filename.ignore", inFile)
} else {
// build output filename
outputFilename := file
var indexInputFile *string
var indexOutputFile *string
if i := newConfig.Index; i != nil {
indexInputFile = i.InputFile
indexOutputFile = i.OutputFile
}
if indexInputFile != nil &&
*indexInputFile == file &&
indexOutputFile != nil &&
*indexOutputFile != "" {
outputFilename = *indexOutputFile
} else {
if stripRegex != nil && *stripRegex != "" {
regex, err := regexp.Compile(*stripRegex)
if err != nil {
Log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
}
outputFilename = regex.ReplaceAllString(outputFilename, "$1")
}
if outputExt != nil && *outputExt != "" {
outputFilename += "." + *outputExt
}
}
outFile := conf.OutputPath + "/" + outputFilename
Log.Debugf("using '%s' as output file", outFile)
// use --- for splitting document in markdown parts
regex := regexp.MustCompile("\\r?\\n\\r?---\\r?\\n\\r?")
inputParts := regex.Split(string(input), -1)
htmlParts := make([]*pongo2.Value, 0)
for _, iPart := range inputParts {
htmlParts = append(htmlParts,
pongo2.AsSafeValue(
string(renderMarkdown([]byte(iPart), newConfig.Markdown))))
}
// build navigation
navMap := make(map[string]*NavElement)
navSlice := make([]*NavElement, 0)
navActive := make([]*NavElement, 0)
BuildNavigation(rootConf, &navMap, &navSlice, &navActive, curNavPath)
// read yaml header as data for template
ctx := newContext()
ctx["This"] = newConfig.This
ctx["Meta"] = newConfig.Meta
ctx["Data"] = newConfig.Data
ctx["ColMap"] = rootConf.ColMap // root as NavMap and NavSlice, for sub go to NavElement.ColMap
ctx["NavMap"] = navMap
ctx["NavSlice"] = navSlice
ctx["NavActive"] = navActive
ctx["Body"] = pongo2.AsSafeValue(string(renderMarkdown(input, newConfig.Markdown)))
ctx["BodyParts"] = htmlParts
ctx["CurrentPath"] = curNavPath
// set active nav element
if len(navActive) > 0 {
ctx["NavElement"] = navActive[len(navActive)-1]
} else {
// if no active path to content, we are in root dir
ctx["NavElement"] = &NavElement{
GoTo: BackToRoot(curNavPath),
Active: true,
ColMap: rootConf.ColMap,
Data: rootConf.Config.Data,
This: rootConf.Config.This,
SubMap: &navMap,
SubSlice: &navSlice,
}
}
Log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
templateFilename := *newConfig.Template
result, err := RenderTemplate(*newConfig.Template, conf, newConfig, &ctx)
if err != nil {
Log.Panicf("could not execute template '%s' for input file '%s': %s", templateFilename, inFile, err)
}
result = FixAssetsPath(result, curNavPath)
Log.Noticef("writing to output file: %s", outFile)
err = ioutil.WriteFile(outFile, []byte(result), 0644)
if err != nil {
Log.Panicf("could not write to output file '%s': %s", outFile, err)
}
//fmt.Println(string(html))
}
}
// process other files, copy...
for _, file := range conf.OtherFiles {
switch config.Config.OtherFiles.Action {
case "copy":
from := conf.InputPath + "/" + file
to := conf.OutputPath + "/" + file
Log.Noticef("copying file from '%s' to '%s'", from, to)
err := cpy.Copy(from, to)
if err != nil {
Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
}
}
}
i := 0
for i < len(conf.Sub) {
ProcessContent(rootConf, conf.Sub[i])
i++
}
}
func renderMarkdown(input []byte, markdownConf *config.MarkdownConfig) []byte {
var options []blackfriday.Option
var chromaRenderer *bool
var chromaStyle *string
if m := markdownConf; m != nil {
chromaRenderer = m.ChromaRenderer
chromaStyle = m.ChromaStyle
}
if chromaStyle == nil {
style := "monokai"
chromaStyle = &style
}
if chromaRenderer != nil && *chromaRenderer {
options = []blackfriday.Option{
blackfriday.WithRenderer(
bfchroma.NewRenderer(
bfchroma.Style(*chromaStyle),
),
),
}
}
// fix \r from markdown for blackfriday
input = bytes.Replace(input, []byte("\r"), []byte(""), -1)
return blackfriday.Run(input, options...)
}

16
helper/dir.go
View File

@@ -1,6 +1,9 @@
package helper
import "os"
import (
"os"
"strings"
)
// CreateDirectory creates direcory with all missing parents and panic if error
func CreateDirectory(dir string) {
@@ -22,3 +25,14 @@ func CreateDirectory(dir string) {
Log.Panicf("unknown error for output directory '%s': %s", dir, err)
}
}
// BackToRoot builds ../../ string
func BackToRoot(curNavPath string) string {
tmpPath := ""
if curNavPath != "" {
for i := strings.Count(curNavPath, "/") + 1; i > 0; i-- {
tmpPath += "../"
}
}
return tmpPath
}

29
helper/markdown.go Normal file
View File

@@ -0,0 +1,29 @@
package helper
import (
"bytes"
"github.com/Depado/bfchroma"
"gopkg.in/russross/blackfriday.v2"
)
func RenderMarkdown(input []byte, chromaRenderer bool, chromaStyle string) []byte {
var options []blackfriday.Option
if chromaStyle == "" {
chromaStyle = "monokai"
}
if chromaRenderer {
options = []blackfriday.Option{
blackfriday.WithRenderer(
bfchroma.NewRenderer(
bfchroma.Style(chromaStyle),
),
),
}
}
// fix \r from markdown for blackfriday
input = bytes.Replace(input, []byte("\r"), []byte(""), -1)
return blackfriday.Run(input, options...)
}

28
helper/merge.go Normal file
View File

@@ -0,0 +1,28 @@
package helper
import (
"reflect"
"github.com/imdario/mergo"
)
type ptrTransformer struct{}
func (t ptrTransformer) Transformer(typ reflect.Type) func(dst, src reflect.Value) error {
if typ.Kind() == reflect.Ptr {
return func(dst, src reflect.Value) error {
if dst.CanSet() {
if dst.IsNil() {
dst.Set(src)
}
}
return nil
}
}
return nil
}
// Merge merges 2 objects or maps
func Merge(dst, src interface{}) error {
return mergo.Merge(dst, src, mergo.WithTransformers(ptrTransformer{}))
}

103
helper/navigation.go
View File

@@ -1,103 +0,0 @@
package helper
import (
"path"
"regexp"
"strings"
"gitbase.de/apairon/mark2web/config"
)
// NavElement is one element with ist attributes and subs
type NavElement struct {
Navname string
GoTo string
Active bool
ColMap config.MapString
Data interface{}
This config.ThisPathConfig
SubMap *map[string]*NavElement
SubSlice *[]*NavElement
}
// BuildNavigation builds the navigation trees for use in templates
func BuildNavigation(conf *config.PathConfigTree, curNavMap *map[string]*NavElement, curNavSlice *[]*NavElement, navActive *[]*NavElement, activeNav string) {
for _, el := range conf.Sub {
if el.Hidden {
continue // ignore hidden nav points from collections
}
var ignNav *string
if p := el.Config.Path; p != nil {
ignNav = p.IgnoreForNav
}
if ignNav != nil && *ignNav != "" {
regex, err := regexp.Compile(*ignNav)
if err != nil {
Log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err)
}
if regex.MatchString(path.Base(el.InputPath)) {
Log.Debugf("ignoring input directory '%s' in navigation", el.InputPath)
continue
}
}
elPath := strings.TrimPrefix(el.OutputPath, config.Config.Directories.Output+"/")
subMap := make(map[string]*NavElement)
subSlice := make([]*NavElement, 0)
navEl := NavElement{
Active: strings.HasPrefix(activeNav, elPath),
Data: el.Config.Data,
ColMap: el.ColMap,
SubMap: &subMap,
SubSlice: &subSlice,
}
navEl.This = el.Config.This
if navEl.Active {
// add to navActive level navigation
currentLevel := strings.Count(activeNav, "/")
if len(*navActive) <= currentLevel {
// not registered
*navActive = append(*navActive, &navEl)
}
}
n := el.Config.This.Navname
if n != nil {
navEl.Navname = *n
}
g := el.Config.This.GoTo
if g != nil {
if strings.HasPrefix(*g, "/") {
// abslute
navEl.GoTo = *g
} else {
// relative
navEl.GoTo = elPath + "/" + *g
}
} else {
navEl.GoTo = elPath + "/"
}
if activeNav != "" && activeNav != "/" {
// calculate relative path
bToRoot := BackToRoot(activeNav)
navEl.GoTo = bToRoot + navEl.GoTo
navEl.GoTo = path.Clean(navEl.GoTo)
}
(*curNavMap)[path.Base(el.OutputPath)] = &navEl
if curNavSlice != nil {
*curNavSlice = append(*curNavSlice, &navEl)
}
BuildNavigation(el, &subMap, &subSlice, navActive, activeNav)
}
}

103
helper/render.go
View File

@@ -1,103 +0,0 @@
package helper
import (
"log"
"path"
"regexp"
"strings"
"gitbase.de/apairon/mark2web/config"
"github.com/flosch/pongo2"
)
var templateCache = make(map[string]*pongo2.Template)
var currentContext *pongo2.Context
var currentTreeNodeConfig *config.PathConfigTree
var currentPathConfig *config.PathConfig
var templateDir string
// BackToRoot builds ../../ string
func BackToRoot(curNavPath string) string {
tmpPath := ""
if curNavPath != "" {
for i := strings.Count(curNavPath, "/") + 1; i > 0; i-- {
tmpPath += "../"
}
}
return tmpPath
}
// ResolveNavPath fixes nav target relative to current navigation path
func ResolveNavPath(target string) string {
curNavPath := (*currentContext)["CurrentPath"].(string)
if strings.HasPrefix(target, "/") {
target = BackToRoot(curNavPath) + target
}
target = path.Clean(target)
return target
}
// ResolveOutputPath fixes output directory relative to current navigation path
func ResolveOutputPath(target string) string {
if strings.HasPrefix(target, "/") {
target = config.Config.Directories.Output + "/" + target
} else {
target = currentTreeNodeConfig.OutputPath + "/" + target
}
return path.Clean(target)
}
// ResolveInputPath fixes input directory relative to current navigation path
func ResolveInputPath(target string) string {
if strings.HasPrefix(target, "/") {
target = config.Config.Directories.Input + "/" + target
} else {
target = currentTreeNodeConfig.InputPath + "/" + target
}
return path.Clean(target)
}
// SetTemplateDir sets base directory for searching template files
func SetTemplateDir(dir string) {
templateDir = dir
}
// RenderTemplate renders a pongo2 template with context
func RenderTemplate(filename string, treeNodeConfig *config.PathConfigTree, pathConfig *config.PathConfig, ctx *pongo2.Context) (string, error) {
currentContext = ctx
currentTreeNodeConfig = treeNodeConfig
currentPathConfig = pathConfig
templateFile := templateDir + "/" + filename
template := templateCache[templateFile]
if template == nil {
var err error
if template, err = pongo2.FromFile(templateFile); err != nil {
log.Panicf("could not parse template '%s': %s", templateFile, err)
} else {
templateCache[templateFile] = template
}
}
return template.Execute(*ctx)
}
// FixAssetsPath replaces assets path based on current path
func FixAssetsPath(str, curNavPath string) string {
if find := config.Config.Assets.FixTemplate.Find; find != "" {
Log.Debugf("fixing assets paths for path '%s'", curNavPath)
repl := config.Config.Assets.FixTemplate.Replace
toPath := config.Config.Assets.ToPath
bToRoot := BackToRoot(curNavPath)
regex, err := regexp.Compile(find)
if err != nil {
log.Panicf("could not compile regexp '%s' for assets path: %s", find, err)
}
repl = bToRoot + toPath + "/" + repl
repl = path.Clean(repl) + "/"
Log.Debugf("new assets paths: %s", repl)
return regex.ReplaceAllString(str, repl)
}
return str
}

377
helper/template_filters.go
View File

@@ -1,377 +0,0 @@
package helper
import (
"crypto/md5"
"encoding/json"
"errors"
"fmt"
"image"
"io/ioutil"
"net/http"
"net/url"
"os"
"path"
"strconv"
"strings"
"gitbase.de/apairon/mark2web/config"
"github.com/davecgh/go-spew/spew"
"github.com/ddliu/motto"
"github.com/disintegration/imaging"
"github.com/flosch/pongo2"
_ "github.com/flosch/pongo2-addons"
_ "github.com/robertkrimen/otto/underscore"
)
func init() {
err := pongo2.ReplaceFilter("markdown", MarkdownFilter)
if err != nil {
panic(err)
}
newFilters := map[string]pongo2.FilterFunction{
"image_process": ImageProcessFilter,
"relative_path": RelativePathFilter,
"json": JSONFilter,
"dump": DumpFilter,
}
for name, fn := range newFilters {
err := pongo2.RegisterFilter(name, fn)
if err != nil {
panic(err)
}
}
}
// DumpFilter is a pongo2 filter, which returns a spew.Dump of the input
func DumpFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
dumpString := spew.Sdump(in.Interface())
return pongo2.AsValue(string(dumpString)), nil
}
// JSONFilter is a pongo2 filter, which returns a json string of the input
func JSONFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
pretty := false
for _, s := range strings.Split(param.String(), ",") {
switch s {
case "pretty":
pretty = true
}
}
var err error
var jsonBytes []byte
if pretty {
jsonBytes, err = json.MarshalIndent(in.Interface(), "", " ")
} else {
jsonBytes, err = json.Marshal(in.Interface())
}
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:json",
OrigError: err,
}
}
return pongo2.AsSafeValue(string(jsonBytes)), nil
}
// MarkdownFilter is a pongo2 filter, which converts markdown to html
func MarkdownFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
return pongo2.AsSafeValue(
string(
renderMarkdown(
[]byte(in.String()),
currentPathConfig.Markdown,
))),
nil
}
func parseImageParams(str string) (*config.ImagingConfig, error) {
p := config.ImagingConfig{}
if str == "" {
config.Merge(&p, currentPathConfig.Imaging)
// Filename and Format are only valid for current image
p.Filename = ""
p.Format = ""
return &p, nil
}
for _, s := range strings.Split(str, ",") {
e := strings.Split(s, "=")
if len(e) < 2 {
return nil, fmt.Errorf("invalid image parameter: %s", s)
}
var err error
switch e[0] {
case "w":
p.Width, err = strconv.Atoi(e[1])
case "h":
p.Height, err = strconv.Atoi(e[1])
case "f":
p.Filename = e[1]
case "t":
p.TargetDir = e[1]
case "p":
p.Process = e[1]
case "a":
p.Anchor = e[1]
case "q":
p.Quality, err = strconv.Atoi(e[1])
if p.Quality < 0 || p.Quality > 100 {
err = errors.New("q= must be between 1 and 100")
}
default:
return nil, fmt.Errorf("invalid image parameter: %s", s)
}
if err != nil {
return nil, fmt.Errorf("could not convert image parameter to correct value type for '%s': %s", s, err)
}
}
return &p, nil
}
func getImageFromURL(url string) (image.Image, string, error) {
resp, err := http.Get(url)
if err != nil {
return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
}
img, format, err := image.Decode(resp.Body)
if err != nil {
return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
}
return img, format, nil
}
// ImageProcessFilter read the image url and process parameters and saves the resized/processed image
// param: w=WITDH,h=HEIGHT
func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
imgSource := in.String()
p, err := parseImageParams(param.String())
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: err,
}
}
if p == nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: errors.New("no imaging config defined"),
}
}
var img image.Image
if p.Process == "" {
p.Process = "resize"
}
filePrefix := fmt.Sprintf(
"%s_%dx%d_q%03d",
p.Process,
p.Width,
p.Height,
p.Quality,
)
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
// remote file
img, p.Format, err = getImageFromURL(imgSource)
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
}
}
// build filename
if p.Filename == "" {
var fBase string
if u, _ := url.Parse(imgSource); u != nil {
fBase = strings.Split(path.Base(u.Path), ".")[0]
}
p.Filename = fmt.Sprintf(
"%s_%x_%s.%s",
filePrefix,
md5.Sum([]byte(imgSource)),
fBase,
p.Format,
)
}
} else {
// local file
imgSource = ResolveInputPath(imgSource)
if p.Filename == "" {
p.Filename = fmt.Sprintf(
"%s_%s",
filePrefix,
path.Base(imgSource),
)
}
}
var imgTarget string
if p.TargetDir != "" {
imgTarget = ResolveOutputPath(
path.Clean(p.TargetDir) + "/" +
p.Filename,
)
pt := path.Dir(imgTarget)
if _, err := os.Stat(pt); os.IsNotExist(err) {
Log.Infof("create image target dir: %s", pt)
if err := os.MkdirAll(pt, 0755); err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("could not create image target dir '%s': %s", pt, err),
}
}
}
p.Filename = ResolveNavPath(p.TargetDir + "/" + p.Filename)
} else {
imgTarget = ResolveOutputPath(p.Filename)
}
if f, err := os.Stat(imgTarget); err == nil && !f.IsDir() {
Log.Noticef("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
} else {
Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
// webrequest before finding target filename, because of file format in filename
} else {
img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
}
}
}
switch p.Process {
case "resize":
img = imaging.Resize(img, p.Width, p.Height, imaging.Lanczos)
case "fit":
img = imaging.Fit(img, p.Width, p.Height, imaging.Lanczos)
case "fill":
var anchor imaging.Anchor
switch strings.ToLower(p.Anchor) {
case "":
fallthrough
case "center":
anchor = imaging.Center
case "topleft":
anchor = imaging.TopLeft
case "top":
anchor = imaging.Top
case "topright":
anchor = imaging.TopRight
case "left":
anchor = imaging.Left
case "right":
anchor = imaging.Right
case "bottomleft":
anchor = imaging.BottomLeft
case "bottom":
anchor = imaging.Bottom
case "bottomright":
anchor = imaging.BottomRight
default:
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("unknown anchor a=%s definition", p.Anchor),
}
}
img = imaging.Fill(img, p.Width, p.Height, anchor, imaging.Lanczos)
default:
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("invalid p parameter '%s'", p.Process),
}
}
var encodeOptions = make([]imaging.EncodeOption, 0)
if p.Quality > 0 {
encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
}
err = imaging.Save(img, imgTarget, encodeOptions...)
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("could save image '%s': %s", imgTarget, err),
}
}
}
return pongo2.AsValue(ResolveNavPath(p.Filename)), nil
}
// RelativePathFilter returns the relative path to navpoint based on current nav
func RelativePathFilter(in, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
return pongo2.AsValue(
ResolveNavPath(
in.String(),
),
), nil
}
// RegisterFilters reads a directory and register filters from files within it
func RegisterFilters(dir string) {
files, err := ioutil.ReadDir(dir)
if err != nil {
Log.Panicf("could not read from template filters dir '%s': %s", dir, err)
}
for _, f := range files {
if !f.IsDir() {
switch path.Ext(f.Name()) {
case ".js":
fileBase := strings.TrimSuffix(f.Name(), ".js")
jsFile := dir + "/" + f.Name()
Log.Debugf("trying to register filter from: %s", jsFile)
/*
jsStr, err := ioutil.ReadFile(jsFile)
if err != nil {
Log.Panicf("could not read '%s': %s", jsFile, err)
}
*/
vm := motto.New()
fn, err := vm.Run(jsFile)
if err != nil {
Log.Panicf("error in javascript vm for '%s': %s", jsFile, err)
}
if !fn.IsFunction() {
Log.Panicf("%s does not contain a function code", jsFile)
}
err = pongo2.RegisterFilter(
fileBase,
func(in, param *pongo2.Value) (out *pongo2.Value, erro *pongo2.Error) {
thisObj, _ := vm.Object("({})")
if currentContext != nil {
thisObj.Set("context", *currentContext)
}
if err != nil {
Log.Panicf("could not set context as in '%s': %s", jsFile, err)
}
ret, err := fn.Call(thisObj.Value(), in.Interface(), param.Interface())
if err != nil {
Log.Panicf("error in javascript file '%s' while calling returned function: %s", jsFile, err)
}
retGo, err := ret.Export()
if err != nil {
Log.Panicf("export error for '%s': %s", jsFile, err)
}
return pongo2.AsValue(retGo), nil
},
)
if err != nil {
Log.Panicf("could not register filter from '%s': %s", jsFile, err)
}
}
}
}
}

125
helper/template_functions.go
View File

@@ -1,125 +0,0 @@
package helper
import (
"encoding/json"
"io/ioutil"
"net/http"
"strings"
"gitbase.de/apairon/mark2web/config"
"github.com/flosch/pongo2"
)
func jsonWebRequest(url string) interface{} {
Log.Noticef("requesting url via GET %s", url)
resp, err := http.Get(url)
if err != nil {
Log.Panicf("could not get url '%s': %s", url, err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Log.Panicf("could not read body from url '%s': %s", url, err)
}
Log.Debugf("output from url '%s':\n%s", url, string(body))
if resp.StatusCode >= 400 {
Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
}
contentType := resp.Header.Get("Content-Type")
if strings.Contains(contentType, "json") {
} else {
Log.Panicf("is not json '%s' from url '%s'", contentType, url)
}
jsonMap := make(map[string]interface{})
err = json.Unmarshal(body, &jsonMap)
if err == nil {
return jsonMap
}
jsonArrayMap := make([]map[string]interface{}, 0)
err = json.Unmarshal(body, &jsonArrayMap)
if err == nil {
return jsonArrayMap
}
Log.Panicf("could not read json from '%s': invalid type", url)
return nil
}
// RequestFn will make a web request and returns map[string]interface form pongo2
func RequestFn(url *pongo2.Value, args ...*pongo2.Value) *pongo2.Value {
u := url.String()
return pongo2.AsValue(jsonWebRequest(u))
}
func add2Nav(currentNode *config.PathConfigTree, pathConfig *config.PathConfig, tplFilename, outDir string, navname string, ctx interface{}, dataMapKey string, body string, hidden bool) {
newNodeConfig := new(config.PathConfigTree)
fillNodeConfig(
newNodeConfig,
currentNode.InputPath,
currentNode.OutputPath,
outDir,
pathConfig,
)
if navname != "" {
newNodeConfig.Config.This = config.ThisPathConfig{
Navname: &navname,
}
}
if dataMapKey != "" {
if newNodeConfig.Config.Data == nil {
newNodeConfig.Config.Data = make(config.MapString)
}
// as submap in Data
newNodeConfig.Config.Data[dataMapKey] = ctx
} else if m, ok := ctx.(map[string]interface{}); ok {
// direct set data
newNodeConfig.Config.Data = m
}
// fake via normal file behavior
newNodeConfig.Config.Template = &tplFilename
newNodeConfig.InputFiles = []string{""} // empty file is special for use InputString
indexInFile := ""
indexOutFile := "index.html"
if idx := newNodeConfig.Config.Index; idx != nil {
if idx.OutputFile != nil && *idx.OutputFile != "" {
indexOutFile = *idx.OutputFile
}
}
newNodeConfig.Config.Index = &config.IndexConfig{
InputFile: &indexInFile,
OutputFile: &indexOutFile,
InputString: &body,
}
newNodeConfig.Hidden = hidden
currentNode.Sub = append(currentNode.Sub, newNodeConfig)
}
// RenderFn renders a pongo2 template with additional context
func RenderFn(templateFilename, outDir, ctx *pongo2.Value, param ...*pongo2.Value) *pongo2.Value {
dataMapKey := ""
body := ""
for i, p := range param {
switch i {
case 0:
dataMapKey = p.String()
case 1:
body = p.String()
}
}
add2Nav(currentTreeNodeConfig, currentPathConfig, templateFilename.String(), outDir.String(), "", ctx.Interface(), dataMapKey, body, true)
return pongo2.AsValue(nil)
}

52
helper/webrequest.go Normal file
View File

@@ -0,0 +1,52 @@
package helper
import (
"encoding/json"
"io/ioutil"
"net/http"
"strings"
)
func JSONWebRequest(url string) interface{} {
Log.Noticef("requesting url via GET %s", url)
resp, err := http.Get(url)
if err != nil {
Log.Panicf("could not get url '%s': %s", url, err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Log.Panicf("could not read body from url '%s': %s", url, err)
}
Log.Debugf("output from url '%s':\n%s", url, string(body))
if resp.StatusCode >= 400 {
Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
}
contentType := resp.Header.Get("Content-Type")
if strings.Contains(contentType, "json") {
} else {
Log.Panicf("is not json '%s' from url '%s'", contentType, url)
}
jsonMap := make(map[string]interface{})
err = json.Unmarshal(body, &jsonMap)
if err == nil {
return jsonMap
}
jsonArrayMap := make([]map[string]interface{}, 0)
err = json.Unmarshal(body, &jsonArrayMap)
if err == nil {
return jsonArrayMap
}
Log.Panicf("could not read json from '%s': invalid type", url)
return nil
}