Compare commits
10 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
daed37587e
|
||
|
|
7695f42e20
|
||
|
|
5d6d03702e
|
||
|
|
4cb09fb81f
|
||
|
|
234137f22f
|
||
|
|
267d1010bb
|
||
|
|
740fb94556
|
||
|
|
a17926f54b
|
||
|
|
b9c4553577
|
||
|
|
745c886cec
|
12
.gitmodules
vendored
12
.gitmodules
vendored
@@ -79,3 +79,15 @@
|
||||
[submodule "vendor/github.com/itchio/go-brotli"]
|
||||
path = vendor/github.com/itchio/go-brotli
|
||||
url = https://github.com/itchio/go-brotli
|
||||
[submodule "vendor/github.com/gosuri/uiprogress"]
|
||||
path = vendor/github.com/gosuri/uiprogress
|
||||
url = https://github.com/gosuri/uiprogress
|
||||
[submodule "vendor/github.com/gosuri/uilive"]
|
||||
path = vendor/github.com/gosuri/uilive
|
||||
url = https://github.com/gosuri/uilive
|
||||
[submodule "vendor/github.com/mattn/go-tty"]
|
||||
path = vendor/github.com/mattn/go-tty
|
||||
url = https://github.com/mattn/go-tty
|
||||
[submodule "vendor/golang.org/x/sys"]
|
||||
path = vendor/golang.org/x/sys
|
||||
url = https://go.googlesource.com/sys
|
||||
|
||||
2
.vscode/settings.json
vendored
2
.vscode/settings.json
vendored
@@ -6,7 +6,7 @@
|
||||
"commands": [
|
||||
{
|
||||
"match": "website/.*",
|
||||
"cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create",
|
||||
"cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create -logLevel warning -progress",
|
||||
"silent": false
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1 +1 @@
|
||||
1.2.0-pre
|
||||
1.2.0
|
||||
@@ -7,8 +7,9 @@ import (
|
||||
"path"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/filter"
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"gitbase.de/apairon/mark2web/pkg/mark2web"
|
||||
"gitbase.de/apairon/mark2web/pkg/progress"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -25,7 +26,8 @@ func main() {
|
||||
outDir := flag.String("out", "html", "output directory")
|
||||
createOutDir := flag.Bool("create", false, "create output directory if not existing")
|
||||
//clearOutDir := flag.Bool("clear", false, "clear output directory before generating website")
|
||||
logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error")
|
||||
logLevel := flag.String("logLevel", "", "log level: debug, info, notice, warning, error")
|
||||
progressBars := flag.Bool("progress", false, "show progress bars for jobs")
|
||||
version := flag.Bool("version", false, "print version of this executable")
|
||||
|
||||
flag.Parse()
|
||||
@@ -37,60 +39,66 @@ func main() {
|
||||
os.Exit(0)
|
||||
}
|
||||
|
||||
level := "info"
|
||||
if logLevel != nil {
|
||||
level := "notice"
|
||||
|
||||
if progressBars != nil && *progressBars {
|
||||
progress.Start()
|
||||
level = "warning" // disable log for progressbars
|
||||
}
|
||||
|
||||
if logLevel != nil && *logLevel != "" {
|
||||
level = *logLevel
|
||||
}
|
||||
helper.ConfigureLogger(level)
|
||||
logger.SetLogLevel(level)
|
||||
|
||||
if inDir == nil || *inDir == "" {
|
||||
helper.Log.Panic("input directory not specified")
|
||||
logger.Exit("input directory not specified")
|
||||
}
|
||||
iDir := path.Clean(*inDir)
|
||||
inDir = &iDir
|
||||
helper.Log.Infof("input directory: %s", *inDir)
|
||||
logger.I("input directory: %s", *inDir)
|
||||
|
||||
if outDir == nil || *outDir == "" {
|
||||
helper.Log.Panic("output directory not specified")
|
||||
logger.Exit("output directory not specified")
|
||||
}
|
||||
oDir := path.Clean(*outDir)
|
||||
outDir = &oDir
|
||||
helper.Log.Infof("output directory: %s", *outDir)
|
||||
logger.I("output directory: %s", *outDir)
|
||||
|
||||
if createOutDir != nil && *createOutDir {
|
||||
if _, err := os.Stat(*outDir); os.IsNotExist(err) {
|
||||
helper.Log.Debugf("output directory '%s' does not exist", *outDir)
|
||||
helper.Log.Debugf("trying to create output directory: %s", *outDir)
|
||||
logger.D("output directory '%s' does not exist", *outDir)
|
||||
logger.D("trying to create output directory: %s", *outDir)
|
||||
err := os.MkdirAll(*outDir, 0755)
|
||||
if err != nil {
|
||||
helper.Log.Panic(err)
|
||||
logger.Log.Panic(err)
|
||||
}
|
||||
helper.Log.Noticef("created output directory: %s", *outDir)
|
||||
logger.I("created output directory: %s", *outDir)
|
||||
} else {
|
||||
helper.Log.Noticef("output directory '%s' already exists", *outDir)
|
||||
logger.I("output directory '%s' already exists", *outDir)
|
||||
}
|
||||
}
|
||||
|
||||
if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
|
||||
helper.Log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
|
||||
logger.Eexit(err, "output directory '%s' does not exist, try -create parameter or create manually", *outDir)
|
||||
} else {
|
||||
if fD == nil {
|
||||
helper.Log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
|
||||
logger.P("something went wrong, could not get file handle for output dir %s", *outDir)
|
||||
} else if !fD.IsDir() {
|
||||
helper.Log.Panicf("output directory '%s' is not a directory", *outDir)
|
||||
logger.Exit("output directory '%s' is not a directory", *outDir)
|
||||
}
|
||||
}
|
||||
|
||||
helper.Log.Debug("reading global config...")
|
||||
logger.D("reading global config...")
|
||||
configFilename := *inDir + "/config.yml"
|
||||
err := mark2web.Config.ReadFromFile(configFilename)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not read file '%s': %s", configFilename, err)
|
||||
logger.Eexit(err, "could not read file '%s'", configFilename)
|
||||
}
|
||||
mark2web.Config.Directories.Input = *inDir
|
||||
mark2web.Config.Directories.Output = *outDir
|
||||
|
||||
helper.Log.Debugf("reading input directory %s", *inDir)
|
||||
logger.D("reading input directory %s", *inDir)
|
||||
|
||||
defaultTemplate := "base.html"
|
||||
defaultInputFile := "README.md"
|
||||
@@ -130,4 +138,6 @@ func main() {
|
||||
|
||||
mark2web.Run(*inDir, *outDir, defaultPathConfig)
|
||||
|
||||
logger.N("done")
|
||||
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"gitbase.de/apairon/mark2web/pkg/mark2web"
|
||||
"github.com/ddliu/motto"
|
||||
"github.com/flosch/pongo2"
|
||||
@@ -15,16 +15,14 @@ import (
|
||||
// RegisterFilters reads a directory and register filters from files within it
|
||||
func RegisterFilters(dir string) {
|
||||
files, err := ioutil.ReadDir(dir)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not read from template filters dir '%s': %s", dir, err)
|
||||
}
|
||||
logger.Eexit(err, "could not read from template filters dir '%s'", dir)
|
||||
for _, f := range files {
|
||||
if !f.IsDir() {
|
||||
switch path.Ext(f.Name()) {
|
||||
case ".js":
|
||||
fileBase := strings.TrimSuffix(f.Name(), ".js")
|
||||
jsFile := dir + "/" + f.Name()
|
||||
helper.Log.Debugf("trying to register filter from: %s", jsFile)
|
||||
logger.D("trying to register filter from: %s", jsFile)
|
||||
/*
|
||||
jsStr, err := ioutil.ReadFile(jsFile)
|
||||
if err != nil {
|
||||
@@ -33,38 +31,30 @@ func RegisterFilters(dir string) {
|
||||
*/
|
||||
vm := motto.New()
|
||||
fn, err := vm.Run(jsFile)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("error in javascript vm for '%s': %s", jsFile, err)
|
||||
}
|
||||
logger.Eexit(err, "error in javascript vm for '%s'", jsFile)
|
||||
if !fn.IsFunction() {
|
||||
helper.Log.Panicf("%s does not contain a function code", jsFile)
|
||||
logger.Exit("%s does not contain a function code", jsFile)
|
||||
}
|
||||
|
||||
err = pongo2.RegisterFilter(
|
||||
fileBase,
|
||||
func(in, param *pongo2.Value) (out *pongo2.Value, erro *pongo2.Error) {
|
||||
thisObj, _ := vm.Object("({})")
|
||||
var err error
|
||||
if mark2web.CurrentContext != nil {
|
||||
thisObj.Set("context", *mark2web.CurrentContext)
|
||||
err = thisObj.Set("context", *mark2web.CurrentContext)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not set context as in '%s': %s", jsFile, err)
|
||||
}
|
||||
logger.Perr(err, "could not set context in '%s': %s", jsFile)
|
||||
ret, err := fn.Call(thisObj.Value(), in.Interface(), param.Interface())
|
||||
if err != nil {
|
||||
helper.Log.Panicf("error in javascript file '%s' while calling returned function: %s", jsFile, err)
|
||||
}
|
||||
logger.Eexit(err, "error in javascript file '%s' while calling returned function", jsFile)
|
||||
|
||||
retGo, err := ret.Export()
|
||||
if err != nil {
|
||||
helper.Log.Panicf("export error for '%s': %s", jsFile, err)
|
||||
}
|
||||
logger.Perr(err, "export error for '%s'", jsFile)
|
||||
return pongo2.AsValue(retGo), nil
|
||||
},
|
||||
)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not register filter from '%s': %s", jsFile, err)
|
||||
}
|
||||
logger.Perr(err, "could not register filter from '%s'", jsFile)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,9 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"image"
|
||||
"net/http"
|
||||
"image/gif"
|
||||
"image/jpeg"
|
||||
"image/png"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
@@ -13,7 +15,10 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/jobm"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"gitbase.de/apairon/mark2web/pkg/mark2web"
|
||||
"gitbase.de/apairon/mark2web/pkg/webrequest"
|
||||
"github.com/disintegration/imaging"
|
||||
"github.com/flosch/pongo2"
|
||||
)
|
||||
@@ -61,20 +66,6 @@ func parseImageParams(str string) (*mark2web.ImagingConfig, error) {
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
func getImageFromURL(url string) (image.Image, string, error) {
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
|
||||
}
|
||||
|
||||
img, format, err := image.Decode(resp.Body)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
|
||||
}
|
||||
|
||||
return img, format, nil
|
||||
}
|
||||
|
||||
// ImageProcessFilter read the image url and process parameters and saves the resized/processed image
|
||||
// param: w=WITDH,h=HEIGHT
|
||||
func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
|
||||
@@ -105,27 +96,18 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
|
||||
p.Quality,
|
||||
)
|
||||
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
|
||||
// remote file
|
||||
img, p.Format, err = getImageFromURL(imgSource)
|
||||
if err != nil {
|
||||
return nil, &pongo2.Error{
|
||||
Sender: "filter:image_resize",
|
||||
OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
|
||||
}
|
||||
}
|
||||
// build filename
|
||||
if p.Filename == "" {
|
||||
var fBase string
|
||||
if u, _ := url.Parse(imgSource); u != nil {
|
||||
fBase = strings.Split(path.Base(u.Path), ".")[0]
|
||||
fBase = path.Base(u.Path)
|
||||
}
|
||||
|
||||
p.Filename = fmt.Sprintf(
|
||||
"%s_%x_%s.%s",
|
||||
"%s_%x_%s",
|
||||
filePrefix,
|
||||
md5.Sum([]byte(imgSource)),
|
||||
fBase,
|
||||
p.Format,
|
||||
)
|
||||
}
|
||||
} else {
|
||||
@@ -149,7 +131,7 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
|
||||
|
||||
pt := path.Dir(imgTarget)
|
||||
if _, err := os.Stat(pt); os.IsNotExist(err) {
|
||||
helper.Log.Infof("create image target dir: %s", pt)
|
||||
logger.I("create image target dir: %s", pt)
|
||||
if err := os.MkdirAll(pt, 0755); err != nil {
|
||||
return nil, &pongo2.Error{
|
||||
Sender: "filter:image_resize",
|
||||
@@ -165,18 +147,18 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
|
||||
}
|
||||
|
||||
if f, err := os.Stat(imgTarget); err == nil && !f.IsDir() {
|
||||
helper.Log.Noticef("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
|
||||
logger.N("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
|
||||
} else {
|
||||
mark2web.ThreadStart(func() {
|
||||
helper.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
|
||||
jobm.Enqueue(jobm.Job{
|
||||
Function: func() {
|
||||
logger.N("processing image from %s to %s", imgSource, imgTarget)
|
||||
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
|
||||
// webrequest before finding target filename, because of file format in filename
|
||||
// remote file
|
||||
img, p.Format, err = webrequest.GetImage(imgSource)
|
||||
} else {
|
||||
img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
|
||||
if err != nil {
|
||||
helper.Log.Panicf("filter:image_resize, could not open image '%s': %s", imgSource, err)
|
||||
}
|
||||
}
|
||||
logger.Eexit(err, "filter:image_resize, could not open image '%s'", imgSource)
|
||||
|
||||
switch p.Process {
|
||||
case "resize":
|
||||
@@ -207,23 +189,54 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
|
||||
case "bottomright":
|
||||
anchor = imaging.BottomRight
|
||||
default:
|
||||
helper.Log.Panicf("filter:image_resize, unknown anchor a=%s definition", p.Anchor)
|
||||
logger.Exit("filter:image_resize, unknown anchor a=%s definition", p.Anchor)
|
||||
}
|
||||
img = imaging.Fill(img, p.Width, p.Height, anchor, imaging.Lanczos)
|
||||
default:
|
||||
helper.Log.Panicf("filter:image_resize, invalid p parameter '%s'", p.Process)
|
||||
logger.Exit("filter:image_resize, invalid p parameter '%s'", p.Process)
|
||||
}
|
||||
|
||||
if p.Format == "" {
|
||||
switch strings.ToLower(path.Ext(imgTarget)) {
|
||||
case ".jpg", ".jpeg", ".gif", ".png":
|
||||
var encodeOptions = make([]imaging.EncodeOption, 0)
|
||||
if p.Quality > 0 {
|
||||
encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
|
||||
}
|
||||
|
||||
err = imaging.Save(img, imgTarget, encodeOptions...)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("filter:image_resize, could save image '%s': %s", imgTarget, err)
|
||||
logger.Eerr(err, "filter:image_resize, could save image '%s'", imgTarget)
|
||||
default:
|
||||
logger.Exit("filter:image_resize, invalid filename extension for image: %s", imgTarget)
|
||||
}
|
||||
helper.Log.Noticef("finished image: %s", imgTarget)
|
||||
} else {
|
||||
out, err := os.Create(imgTarget)
|
||||
defer out.Close()
|
||||
|
||||
logger.Eexit(err, "filter:image_resize, could not create image file '%s'", imgTarget)
|
||||
switch p.Format {
|
||||
case "jpeg", "jpg":
|
||||
var jpegOpt *jpeg.Options
|
||||
if p.Quality > 0 {
|
||||
jpegOpt = &jpeg.Options{
|
||||
Quality: p.Quality,
|
||||
}
|
||||
}
|
||||
err = jpeg.Encode(out, img, jpegOpt)
|
||||
case "png":
|
||||
err = png.Encode(out, img)
|
||||
case "gif":
|
||||
err = gif.Encode(out, img, nil)
|
||||
default:
|
||||
logger.Exit("filter:image_resize, unknown format '%s' for '%s'", p.Format, imgSource)
|
||||
}
|
||||
logger.Eexit(err, "filter:image_resize, could not encode image file '%s'", imgTarget)
|
||||
}
|
||||
|
||||
logger.N("finished image: %s", imgTarget)
|
||||
},
|
||||
Description: imgSource,
|
||||
Category: "image process",
|
||||
})
|
||||
}
|
||||
return pongo2.AsValue(mark2web.CurrentTreeNode.ResolveNavPath(p.Filename)), nil
|
||||
|
||||
@@ -4,6 +4,7 @@ import (
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/jobm"
|
||||
"gitbase.de/apairon/mark2web/pkg/mark2web"
|
||||
|
||||
"github.com/flosch/pongo2"
|
||||
@@ -19,7 +20,7 @@ func TestImageProcessFilter(t *testing.T) {
|
||||
}
|
||||
|
||||
// we want to check files after function calls, so no multithreading
|
||||
mark2web.SetNumCPU(1)
|
||||
jobm.SetNumCPU(1)
|
||||
|
||||
mark2web.Config.Directories.Input = "../../test/in"
|
||||
mark2web.Config.Directories.Output = "../../test/out"
|
||||
|
||||
@@ -2,25 +2,26 @@ package helper
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
)
|
||||
|
||||
// CreateDirectory creates direcory with all missing parents and panic if error
|
||||
func CreateDirectory(dir string) {
|
||||
Log.Debugf("trying to create output directory: %s", dir)
|
||||
logger.D("trying to create output directory: %s", dir)
|
||||
|
||||
if dirH, err := os.Stat(dir); os.IsNotExist(err) {
|
||||
err := os.MkdirAll(dir, 0755)
|
||||
if err != nil {
|
||||
Log.Panicf("could not create output directory '%s': %s", dir, err)
|
||||
}
|
||||
Log.Noticef("created output directory: %s", dir)
|
||||
logger.Eexit(err, "could not create output directory '%s'", dir)
|
||||
|
||||
logger.I("created output directory: %s", dir)
|
||||
} else if dirH != nil {
|
||||
if dirH.IsDir() {
|
||||
Log.Noticef("output directory '%s' already exists", dir)
|
||||
logger.I("output directory '%s' already exists", dir)
|
||||
} else {
|
||||
Log.Panicf("output directory '%s' is no directory", dir)
|
||||
logger.Exit("output directory '%s' is no directory", dir)
|
||||
}
|
||||
} else {
|
||||
Log.Panicf("unknown error for output directory '%s': %s", dir, err)
|
||||
logger.Perr(err, "unknown error for output directory '%s'", dir)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,52 +0,0 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/op/go-logging"
|
||||
)
|
||||
|
||||
// Log is global logger
|
||||
var Log = logging.MustGetLogger("myLogger")
|
||||
|
||||
// ConfigureLogger sets logger backend and level
|
||||
func ConfigureLogger(level string) {
|
||||
logBackend := logging.NewLogBackend(os.Stderr, "", 0)
|
||||
logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
|
||||
`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
|
||||
))
|
||||
logBackendLeveled := logging.AddModuleLevel(logBackendFormatter)
|
||||
logBackendLevel := logging.INFO
|
||||
switch level {
|
||||
case "debug":
|
||||
logBackendLevel = logging.DEBUG
|
||||
break
|
||||
|
||||
case "info":
|
||||
logBackendLevel = logging.INFO
|
||||
break
|
||||
|
||||
case "notice":
|
||||
logBackendLevel = logging.NOTICE
|
||||
break
|
||||
|
||||
case "warning":
|
||||
logBackendLevel = logging.WARNING
|
||||
break
|
||||
|
||||
case "error":
|
||||
logBackendLevel = logging.ERROR
|
||||
break
|
||||
|
||||
}
|
||||
logBackendLeveled.SetLevel(logBackendLevel, "")
|
||||
logging.SetBackend(logBackendLeveled)
|
||||
}
|
||||
|
||||
func init() {
|
||||
spew.Config.DisablePointerAddresses = true
|
||||
spew.Config.DisableCapacities = true
|
||||
spew.Config.DisableMethods = true
|
||||
spew.Config.DisablePointerMethods = true
|
||||
}
|
||||
16
pkg/helper/string.go
Normal file
16
pkg/helper/string.go
Normal file
@@ -0,0 +1,16 @@
|
||||
package helper
|
||||
|
||||
// ShortenStringLeft shortens a string
|
||||
func ShortenStringLeft(str string, num int) string {
|
||||
if num <= 4 {
|
||||
return ""
|
||||
}
|
||||
tstr := str
|
||||
if len(str) > num {
|
||||
if num > 3 {
|
||||
num -= 3
|
||||
}
|
||||
tstr = "..." + str[len(str)-num:len(str)]
|
||||
}
|
||||
return tstr
|
||||
}
|
||||
@@ -1,53 +0,0 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// JSONWebRequest will GET a json object/array from a given URL
|
||||
func JSONWebRequest(url string) interface{} {
|
||||
Log.Noticef("requesting url via GET %s", url)
|
||||
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
Log.Panicf("could not get url '%s': %s", url, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
Log.Panicf("could not read body from url '%s': %s", url, err)
|
||||
}
|
||||
|
||||
Log.Debugf("output from url '%s':\n%s", url, string(body))
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
|
||||
if strings.Contains(contentType, "json") {
|
||||
|
||||
} else {
|
||||
Log.Panicf("is not json '%s' from url '%s'", contentType, url)
|
||||
}
|
||||
|
||||
jsonMap := make(map[string]interface{})
|
||||
err = json.Unmarshal(body, &jsonMap)
|
||||
if err == nil {
|
||||
return jsonMap
|
||||
}
|
||||
|
||||
jsonArrayMap := make([]map[string]interface{}, 0)
|
||||
err = json.Unmarshal(body, &jsonArrayMap)
|
||||
if err == nil {
|
||||
return jsonArrayMap
|
||||
}
|
||||
|
||||
Log.Panicf("could not read json from '%s': invalid type", url)
|
||||
return nil
|
||||
}
|
||||
61
pkg/jobm/jobmanager.go
Normal file
61
pkg/jobm/jobmanager.go
Normal file
@@ -0,0 +1,61 @@
|
||||
package jobm
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/progress"
|
||||
)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var numCPU = runtime.NumCPU()
|
||||
|
||||
// Job is a wrapper to descripe a Job function
|
||||
type Job struct {
|
||||
Function func()
|
||||
Description string
|
||||
Category string
|
||||
}
|
||||
|
||||
var jobChan = make(chan []Job)
|
||||
|
||||
func worker(jobChan <-chan []Job) {
|
||||
defer wg.Done()
|
||||
|
||||
for jobs := range jobChan {
|
||||
for _, job := range jobs {
|
||||
progress.DescribeCurrent(job.Category, job.Description)
|
||||
job.Function()
|
||||
progress.IncrDone(job.Category)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func init() {
|
||||
//logger.I("number of CPU core: %d", numCPU)
|
||||
// one core for main thread
|
||||
for i := 0; i < numCPU; i++ {
|
||||
wg.Add(1)
|
||||
go worker(jobChan)
|
||||
}
|
||||
}
|
||||
|
||||
// Enqueue enqueues a job to the job queue
|
||||
func Enqueue(jobs ...Job) {
|
||||
for _, job := range jobs {
|
||||
progress.IncrTotal(job.Category)
|
||||
}
|
||||
jobChan <- jobs
|
||||
}
|
||||
|
||||
// Wait will wait for all jobs to finish
|
||||
func Wait() {
|
||||
close(jobChan)
|
||||
progress.Stop()
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// SetNumCPU is for testing package without threading
|
||||
func SetNumCPU(i int) {
|
||||
numCPU = i
|
||||
}
|
||||
138
pkg/logger/logger.go
Normal file
138
pkg/logger/logger.go
Normal file
@@ -0,0 +1,138 @@
|
||||
package logger
|
||||
|
||||
import (
|
||||
"os"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/op/go-logging"
|
||||
)
|
||||
|
||||
// Log is global logger
|
||||
var Log = logging.MustGetLogger("myLogger")
|
||||
var logBackendLeveled logging.LeveledBackend
|
||||
|
||||
var Prefix = ""
|
||||
|
||||
// SetLogLevel sets log level for global logger (debug, info, notice, warning, error)
|
||||
func SetLogLevel(level string) {
|
||||
logBackendLevel := logging.INFO
|
||||
switch level {
|
||||
case "debug":
|
||||
logBackendLevel = logging.DEBUG
|
||||
break
|
||||
|
||||
case "info":
|
||||
logBackendLevel = logging.INFO
|
||||
break
|
||||
|
||||
case "notice":
|
||||
logBackendLevel = logging.NOTICE
|
||||
break
|
||||
|
||||
case "warning":
|
||||
logBackendLevel = logging.WARNING
|
||||
break
|
||||
|
||||
case "error":
|
||||
logBackendLevel = logging.ERROR
|
||||
break
|
||||
|
||||
}
|
||||
|
||||
logBackendLeveled.SetLevel(logBackendLevel, "")
|
||||
}
|
||||
|
||||
// configureLogger sets logger backend and level
|
||||
func configureLogger() {
|
||||
logBackend := logging.NewLogBackend(os.Stderr, "", 0)
|
||||
logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
|
||||
`%{color}%{time:2006-01-02 15:04:05.000} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
|
||||
))
|
||||
logBackendLeveled = logging.AddModuleLevel(logBackendFormatter)
|
||||
logBackendLeveled.SetLevel(logging.NOTICE, "")
|
||||
logging.SetBackend(logBackendLeveled)
|
||||
}
|
||||
|
||||
func init() {
|
||||
spew.Config.DisablePointerAddresses = true
|
||||
spew.Config.DisableCapacities = true
|
||||
spew.Config.DisableMethods = true
|
||||
spew.Config.DisablePointerMethods = true
|
||||
configureLogger()
|
||||
}
|
||||
|
||||
func prefix() string {
|
||||
return Prefix
|
||||
}
|
||||
|
||||
// D is shorthand for Debugf
|
||||
func D(format string, args ...interface{}) {
|
||||
Log.Debugf(prefix()+format, args...)
|
||||
}
|
||||
|
||||
// I is shorthand for Infof
|
||||
func I(format string, args ...interface{}) {
|
||||
Log.Infof(prefix()+format, args...)
|
||||
}
|
||||
|
||||
// N is shorthand for Noticef
|
||||
func N(format string, args ...interface{}) {
|
||||
Log.Noticef(prefix()+format, args...)
|
||||
}
|
||||
|
||||
// W is shorthand for Warningf
|
||||
func W(format string, args ...interface{}) {
|
||||
Log.Warningf(prefix()+format, args...)
|
||||
}
|
||||
|
||||
// E is shorthand for Errorf
|
||||
func E(format string, args ...interface{}) {
|
||||
Log.Errorf(prefix()+format, args...)
|
||||
}
|
||||
|
||||
// P is shorthand for Panicf
|
||||
func P(format string, args ...interface{}) {
|
||||
Log.Panicf(prefix()+format, args...)
|
||||
}
|
||||
|
||||
// Eerr is shorthand for
|
||||
// if err != nil {
|
||||
// Log.Errorf(...)
|
||||
// }
|
||||
func Eerr(err error, format string, args ...interface{}) {
|
||||
if err != nil {
|
||||
args = append(args, err)
|
||||
Log.Errorf(prefix()+format+" (Error: %s)", args...)
|
||||
}
|
||||
}
|
||||
|
||||
// Eexit is shorthand for
|
||||
// if err != nil {
|
||||
// Log.Errorf(...)
|
||||
// os.Exit(1)
|
||||
// }
|
||||
func Eexit(err error, format string, args ...interface{}) {
|
||||
Eerr(err, format, args...)
|
||||
if err != nil {
|
||||
os.Exit(1)
|
||||
}
|
||||
}
|
||||
|
||||
// Exit is shorthand for
|
||||
// Log.Errorf(...)
|
||||
// os.Exit(1)
|
||||
func Exit(format string, args ...interface{}) {
|
||||
E(format, args...)
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
// Perr is shorthand for
|
||||
// if err != nil {
|
||||
// Log.Panicf(...)
|
||||
// }
|
||||
func Perr(err error, format string, args ...interface{}) {
|
||||
if err != nil {
|
||||
args = append(args, err)
|
||||
Log.Panicf(prefix()+format+" (Error: %s)", args...)
|
||||
}
|
||||
}
|
||||
@@ -1,12 +1,11 @@
|
||||
package mark2web
|
||||
|
||||
import (
|
||||
"log"
|
||||
"path"
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
cpy "github.com/otiai10/copy"
|
||||
)
|
||||
|
||||
@@ -22,11 +21,10 @@ func ProcessAssets() {
|
||||
if !strings.HasPrefix(to, "/") {
|
||||
to = Config.Directories.Output + "/" + to
|
||||
}
|
||||
helper.Log.Noticef("copying assets from '%s' to '%s'", from, to)
|
||||
logger.N("copying assets from '%s' to '%s'", from, to)
|
||||
err := cpy.Copy(from, to)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
|
||||
}
|
||||
|
||||
logger.Perr(err, "could not copy assets from '%s' to '%s'", from, to)
|
||||
|
||||
if Config.Assets.Compress {
|
||||
compressFilesInDir(to)
|
||||
@@ -37,18 +35,17 @@ func ProcessAssets() {
|
||||
// fixAssetsPath replaces assets path based on current path
|
||||
func (node *TreeNode) fixAssetsPath(str string) string {
|
||||
if find := Config.Assets.FixTemplate.Find; find != "" {
|
||||
helper.Log.Debugf("fixing assets paths for path '%s'", node.CurrentNavPath())
|
||||
logger.D("fixing assets paths for path '%s'", node.CurrentNavPath())
|
||||
repl := Config.Assets.FixTemplate.Replace
|
||||
toPath := Config.Assets.ToPath
|
||||
|
||||
bToRoot := node.BackToRootPath()
|
||||
regex, err := regexp.Compile(find)
|
||||
if err != nil {
|
||||
log.Panicf("could not compile regexp '%s' for assets path: %s", find, err)
|
||||
}
|
||||
logger.Eexit(err, "could not compile regexp '%s' for assets path", find)
|
||||
|
||||
repl = bToRoot + toPath + "/" + repl
|
||||
repl = path.Clean(repl) + "/"
|
||||
helper.Log.Debugf("new assets paths: %s", repl)
|
||||
logger.D("new assets paths: %s", repl)
|
||||
return regex.ReplaceAllString(str, repl)
|
||||
}
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"io"
|
||||
"os"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"github.com/itchio/go-brotli/enc"
|
||||
)
|
||||
|
||||
@@ -15,12 +15,11 @@ var brotliSupported = true
|
||||
func handleBrotliCompression(filename string, content []byte) {
|
||||
brFilename := filename + ".br"
|
||||
|
||||
helper.Log.Infof("writing to compressed output file: %s", brFilename)
|
||||
logger.I("writing to compressed output file: %s", brFilename)
|
||||
|
||||
f, err := os.Create(brFilename)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not create file '%s': %s", brFilename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not create file '%s'", brFilename)
|
||||
|
||||
defer f.Close()
|
||||
|
||||
bw := enc.NewBrotliWriter(f, nil)
|
||||
@@ -29,20 +28,14 @@ func handleBrotliCompression(filename string, content []byte) {
|
||||
if content != nil {
|
||||
// content given
|
||||
_, err = bw.Write(content)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not write brotli content for '%s': %s", filename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not write brotli content for '%s'", filename)
|
||||
} else {
|
||||
// read file
|
||||
r, err := os.Open(filename)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not open file '%s': %s", filename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not open file '%s'", filename)
|
||||
defer r.Close()
|
||||
|
||||
_, err = io.Copy(bw, r)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not write brotli file for '%s': %s", filename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not write brotli file for '%s'", filename)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"gitbase.de/apairon/mark2web/pkg/webrequest"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/flosch/pongo2"
|
||||
)
|
||||
@@ -23,11 +25,11 @@ func (node *TreeNode) handleCollections() {
|
||||
collections := append(node.Config.Collections, node.Config.This.Collections...)
|
||||
for _, colConfig := range collections {
|
||||
if colConfig.Name == nil || *colConfig.Name == "" {
|
||||
helper.Log.Panicf("missing Name in collection config in '%s'", node.InputPath)
|
||||
logger.Exit("missing Name in collection config in '%s'", node.InputPath)
|
||||
}
|
||||
if (colConfig.URL == nil || *colConfig.URL == "") &&
|
||||
(colConfig.Directory == nil) {
|
||||
helper.Log.Panicf("missing URL and Directory in collection config in '%s'", node.InputPath)
|
||||
logger.Exit("missing URL and Directory in collection config in '%s'", node.InputPath)
|
||||
}
|
||||
|
||||
if node.ColMap == nil {
|
||||
@@ -44,9 +46,7 @@ func (node *TreeNode) handleCollections() {
|
||||
|
||||
if colConfig.URL != nil {
|
||||
url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "invalid template string for Collection Element.URL in '%s'", node.InputPath)
|
||||
|
||||
errSrcText = "URL " + url
|
||||
cacheKey = url
|
||||
@@ -55,8 +55,8 @@ func (node *TreeNode) handleCollections() {
|
||||
colData = cacheEntry.data
|
||||
cacheEntry.hit++
|
||||
} else {
|
||||
helper.Log.Noticef("reading collection from: %s", errSrcText)
|
||||
colData = helper.JSONWebRequest(url)
|
||||
logger.N("reading collection from: %s", errSrcText)
|
||||
colData = webrequest.GetJSON(url)
|
||||
colCache[url] = &colCacheEntry{
|
||||
data: colData,
|
||||
navnames: make([]string, 0),
|
||||
@@ -67,20 +67,16 @@ func (node *TreeNode) handleCollections() {
|
||||
path := node.ResolveInputPath(colConfig.Directory.Path)
|
||||
errSrcText = "DIR " + path
|
||||
|
||||
helper.Log.Noticef("reading collection from: %s", errSrcText)
|
||||
logger.N("reading collection from: %s", errSrcText)
|
||||
d, err := ioutil.ReadDir(path)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not read directory '%s': %s", path, err)
|
||||
}
|
||||
logger.Eexit(err, "could not read directory '%s'", path)
|
||||
|
||||
mStr := "."
|
||||
if colConfig.Directory.MatchFilename != "" {
|
||||
mStr = colConfig.Directory.MatchFilename
|
||||
}
|
||||
matcher, err := regexp.Compile(mStr)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not compile regex for MatchFilename '%s' in '%s': %s", mStr, path, err)
|
||||
}
|
||||
logger.Eexit(err, "could not compile regex for MatchFilename '%s' in '%s'", mStr, path)
|
||||
|
||||
if colConfig.Directory.ReverseOrder {
|
||||
for i := len(d)/2 - 1; i >= 0; i-- {
|
||||
@@ -94,9 +90,8 @@ func (node *TreeNode) handleCollections() {
|
||||
if !fh.IsDir() && matcher.MatchString(fh.Name()) {
|
||||
inFile := path + "/" + fh.Name()
|
||||
md, err := ioutil.ReadFile(inFile)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not read file '%s': %s", inFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not read file '%s'", inFile)
|
||||
|
||||
_, ctx := node.processMarkdownWithHeader(md, inFile)
|
||||
(*ctx)["FilenameMatch"] = helper.GetRegexpParams(matcher, fh.Name())
|
||||
fcolData = append(fcolData, *ctx)
|
||||
@@ -116,41 +111,36 @@ func (node *TreeNode) handleCollections() {
|
||||
if colDataMap, ok = colData.(map[string]interface{}); ok {
|
||||
entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
|
||||
if !ok {
|
||||
helper.Log.Debug(spew.Sdump(colDataMap))
|
||||
helper.Log.Panicf("invalid json data in [%s] from '%s' for entries", navT.EntriesAttribute, errSrcText)
|
||||
logger.D(spew.Sdump(colDataMap))
|
||||
logger.Exit("invalid json data in [%s] from '%s' for entries", navT.EntriesAttribute, errSrcText)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
entries, ok = colData.([]interface{})
|
||||
}
|
||||
if !ok {
|
||||
helper.Log.Debug(spew.Sdump(colData))
|
||||
helper.Log.Panicf("invalid json data from '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", errSrcText)
|
||||
logger.D(spew.Sdump(colData))
|
||||
logger.Exit("invalid json data from '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", errSrcText)
|
||||
}
|
||||
|
||||
// build navigation with detail sites
|
||||
for idx, colEl := range entries {
|
||||
ctxE := make(pongo2.Context)
|
||||
err := helper.Merge(&ctxE, ctx)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "could not merge context in '%s'", node.InputPath)
|
||||
|
||||
var jsonCtx map[string]interface{}
|
||||
if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
|
||||
helper.Log.Debug(spew.Sdump(colEl))
|
||||
helper.Log.Panicf("no json object for entry index %d from '%s'", idx, errSrcText)
|
||||
logger.D(spew.Sdump(colEl))
|
||||
logger.Exit("no json object for entry index %d from '%s'", idx, errSrcText)
|
||||
}
|
||||
err = helper.Merge(&ctxE, pongo2.Context(jsonCtx))
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "could not merge context in '%s'", node.InputPath)
|
||||
|
||||
tpl := ""
|
||||
if navT.Template != "" {
|
||||
tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "invalid template string for NavTemplate.Template in '%s'", node.InputPath)
|
||||
}
|
||||
if tpl == "" {
|
||||
tpl = *node.Config.Template
|
||||
@@ -159,48 +149,41 @@ func (node *TreeNode) handleCollections() {
|
||||
dataKey := ""
|
||||
if navT.DataKey != "" {
|
||||
dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "invalid template string for NavTemplate.DataKey in '%s'", node.InputPath)
|
||||
}
|
||||
|
||||
goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "invalid template string for NavTemplate.GoTo in '%s'", node.InputPath)
|
||||
|
||||
goTo = strings.Trim(goTo, "/")
|
||||
goTo = path.Clean(goTo)
|
||||
|
||||
if strings.Contains(goTo, "..") {
|
||||
helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
|
||||
logger.Exit("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
|
||||
}
|
||||
if goTo == "." {
|
||||
helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
|
||||
logger.Exit("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
|
||||
}
|
||||
if goTo == "" {
|
||||
helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
|
||||
logger.Exit("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
|
||||
}
|
||||
|
||||
navname := ""
|
||||
if navT.Navname != "" {
|
||||
navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "invalid template string for NavTemplate.Navname in '%s'", node.InputPath)
|
||||
}
|
||||
body := ""
|
||||
if navT.Body != "" {
|
||||
body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", node.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "invalid template string for NavTemplate.Body in '%s'", node.InputPath)
|
||||
}
|
||||
|
||||
if l := len(colCache[cacheKey].navnames); colCache[cacheKey].hit > 1 &&
|
||||
l > 0 &&
|
||||
navname == colCache[cacheKey].navnames[l-1] {
|
||||
// navname before used same url, so recursion loop
|
||||
helper.Log.Panicf("collection request loop detected for in '%s' for : %s", node.InputPath, errSrcText)
|
||||
logger.Exit("collection request loop detected for in '%s' for : %s", node.InputPath, errSrcText)
|
||||
}
|
||||
|
||||
colCache[cacheKey].navnames = append(colCache[cacheKey].navnames, navname)
|
||||
|
||||
@@ -7,11 +7,14 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/jobm"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
)
|
||||
|
||||
func handleCompression(filename string, content []byte) {
|
||||
ThreadStart(func() {
|
||||
jobm.Enqueue(jobm.Job{
|
||||
Function: func() {
|
||||
if _, ok := Config.Compress.Extensions[path.Ext(filename)]; ok {
|
||||
|
||||
if Config.Compress.Brotli {
|
||||
@@ -21,54 +24,45 @@ func handleCompression(filename string, content []byte) {
|
||||
if Config.Compress.GZIP {
|
||||
gzFilename := filename + ".gz"
|
||||
|
||||
helper.Log.Infof("writing to compressed output file: %s", gzFilename)
|
||||
logger.I("writing to compressed output file: %s", gzFilename)
|
||||
|
||||
f, err := os.Create(gzFilename)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not create file '%s': %s", gzFilename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not create file '%s'", gzFilename)
|
||||
defer f.Close()
|
||||
|
||||
zw, err := gzip.NewWriterLevel(f, gzip.BestCompression)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not initialize gzip writer for '%s': %s", filename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not initialize gzip writer for '%s'", filename)
|
||||
defer zw.Close()
|
||||
|
||||
if content != nil {
|
||||
// content given
|
||||
_, err = zw.Write(content)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not write gziped content for '%s': %s", filename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not write gziped content for '%s'", filename)
|
||||
} else {
|
||||
// read file
|
||||
r, err := os.Open(filename)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not open file '%s': %s", filename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not open file '%s'", filename)
|
||||
defer r.Close()
|
||||
|
||||
_, err = io.Copy(zw, r)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not gzip file '%s': %s", filename, err)
|
||||
}
|
||||
logger.Eexit(err, "could not gzip file '%s'", filename)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
},
|
||||
Description: filename,
|
||||
Category: "compress",
|
||||
})
|
||||
}
|
||||
|
||||
func compressFilesInDir(dir string) {
|
||||
helper.Log.Noticef("compressing configured files in: %s", dir)
|
||||
logger.N("compressing configured files in: %s", dir)
|
||||
|
||||
var _processDir func(string)
|
||||
_processDir = func(d string) {
|
||||
entries, err := ioutil.ReadDir(d)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not read dir '%s': %s", d, err)
|
||||
}
|
||||
logger.Eexit(err, "could not read dir '%s'", d)
|
||||
|
||||
for _, entry := range entries {
|
||||
if entry.IsDir() {
|
||||
|
||||
@@ -7,7 +7,10 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/progress"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/flosch/pongo2"
|
||||
cpy "github.com/otiai10/copy"
|
||||
@@ -16,6 +19,9 @@ import (
|
||||
|
||||
// ReadContentDir walks through content directory and builds the tree of configurations
|
||||
func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string, conf *PathConfig) {
|
||||
progress.IncrTotal("content dir")
|
||||
progress.DescribeCurrent("content dir", "found "+inBase)
|
||||
|
||||
if node.root == nil {
|
||||
// first node is root
|
||||
node.root = node
|
||||
@@ -23,9 +29,7 @@ func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string,
|
||||
node.fillConfig(inBase, outBase, dir, conf)
|
||||
|
||||
files, err := ioutil.ReadDir(node.InputPath)
|
||||
if err != nil {
|
||||
helper.Log.Panic(err)
|
||||
}
|
||||
logger.Eexit(err, "could not read dir '%s'", node.InputPath)
|
||||
|
||||
// first only files
|
||||
for _, f := range files {
|
||||
@@ -33,14 +37,14 @@ func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string,
|
||||
if !f.IsDir() && f.Name() != "config.yml" {
|
||||
switch path.Ext(f.Name()) {
|
||||
case ".md":
|
||||
helper.Log.Debugf(".MD %s", p)
|
||||
logger.D(".MD %s", p)
|
||||
if node.InputFiles == nil {
|
||||
node.InputFiles = make([]string, 0)
|
||||
}
|
||||
node.InputFiles = append(node.InputFiles, f.Name())
|
||||
break
|
||||
default:
|
||||
helper.Log.Debugf("FIL %s", p)
|
||||
logger.D("FIL %s", p)
|
||||
if node.OtherFiles == nil {
|
||||
node.OtherFiles = make([]string, 0)
|
||||
}
|
||||
@@ -53,7 +57,7 @@ func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string,
|
||||
for _, f := range files {
|
||||
p := node.InputPath + "/" + f.Name()
|
||||
if f.IsDir() {
|
||||
helper.Log.Debugf("DIR %s", p)
|
||||
logger.D("DIR %s", p)
|
||||
newTree := new(TreeNode)
|
||||
newTree.root = node.root
|
||||
if node.Sub == nil {
|
||||
@@ -75,18 +79,16 @@ func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*Pa
|
||||
// replace tabs
|
||||
yamlData = bytes.Replace(yamlData, []byte("\t"), []byte(" "), -1)
|
||||
|
||||
helper.Log.Debugf("found yaml header in '%s', merging config", errorRef)
|
||||
logger.D("found yaml header in '%s', merging config", errorRef)
|
||||
err := yaml.Unmarshal(yamlData, newConfig)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not parse YAML header from '%s': %s", errorRef, err)
|
||||
}
|
||||
logger.Eexit(err, "could not parse YAML header from '%s'", errorRef)
|
||||
|
||||
helper.Log.Debug("merging config with upper config")
|
||||
logger.D("merging config with upper config")
|
||||
oldThis := newConfig.This
|
||||
helper.Merge(newConfig, node.Config)
|
||||
newConfig.This = oldThis
|
||||
|
||||
helper.Log.Debug(spew.Sdump(newConfig))
|
||||
logger.D(spew.Sdump(newConfig))
|
||||
|
||||
md = headerRegex.ReplaceAll(md, []byte(""))
|
||||
} else {
|
||||
@@ -154,6 +156,8 @@ func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*Pa
|
||||
|
||||
// ProcessContent walks recursivly through the input paths and processes all files for output
|
||||
func (node *TreeNode) ProcessContent() {
|
||||
progress.DescribeCurrent("content dir", "processing "+node.InputPath)
|
||||
|
||||
helper.CreateDirectory(node.OutputPath)
|
||||
|
||||
if node.root != node {
|
||||
@@ -183,27 +187,24 @@ func (node *TreeNode) ProcessContent() {
|
||||
}
|
||||
if ignoreRegex != nil && *ignoreRegex != "" {
|
||||
regex, err := regexp.Compile(*ignoreRegex)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not compile filename.ignore regexp '%s' for file '%s'", *ignoreRegex, inFile)
|
||||
ignoreFile = regex.MatchString(file)
|
||||
}
|
||||
}
|
||||
|
||||
if ignoreFile {
|
||||
helper.Log.Infof("ignoring file '%s', because of filename.ignore", inFile)
|
||||
logger.I("ignoring file '%s', because of filename.ignore", inFile)
|
||||
} else {
|
||||
var input []byte
|
||||
|
||||
if file != "" {
|
||||
helper.Log.Debugf("reading file: %s", inFile)
|
||||
logger.D("reading file: %s", inFile)
|
||||
|
||||
var err error
|
||||
input, err = ioutil.ReadFile(inFile)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not read '%s':%s", inFile, err)
|
||||
}
|
||||
helper.Log.Infof("processing input file '%s'", inFile)
|
||||
logger.Eexit(err, "could not read '%s'", inFile)
|
||||
|
||||
logger.I("processing input file '%s'", inFile)
|
||||
} else {
|
||||
// use input string if available and input filename == ""
|
||||
var inputString *string
|
||||
@@ -211,7 +212,7 @@ func (node *TreeNode) ProcessContent() {
|
||||
inputString = i.InputString
|
||||
}
|
||||
if inputString != nil {
|
||||
helper.Log.Debugf("using input string instead of file")
|
||||
logger.D("using input string instead of file")
|
||||
input = []byte(*inputString)
|
||||
}
|
||||
}
|
||||
@@ -243,9 +244,8 @@ func (node *TreeNode) ProcessContent() {
|
||||
} else {
|
||||
if stripRegex != nil && *stripRegex != "" {
|
||||
regex, err := regexp.Compile(*stripRegex)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not compile filename.strip regexp '%s' for file '%s'", *stripRegex, inFile)
|
||||
|
||||
outputFilename = regex.ReplaceAllString(outputFilename, "$1")
|
||||
}
|
||||
if outputExt != nil && *outputExt != "" {
|
||||
@@ -254,21 +254,17 @@ func (node *TreeNode) ProcessContent() {
|
||||
}
|
||||
|
||||
outFile := node.OutputPath + "/" + outputFilename
|
||||
helper.Log.Debugf("using '%s' as output file", outFile)
|
||||
helper.Log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
|
||||
logger.D("using '%s' as output file", outFile)
|
||||
logger.D("rendering template '%s' for '%s'", *newConfig.Template, outFile)
|
||||
templateFilename := *newConfig.Template
|
||||
result, err := renderTemplate(*newConfig.Template, node, newConfig, ctx)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not execute template '%s' for input file '%s': %s", templateFilename, inFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not execute template '%s' for input file '%s': %s", templateFilename, inFile)
|
||||
|
||||
result = node.fixAssetsPath(result)
|
||||
|
||||
helper.Log.Noticef("writing to output file: %s", outFile)
|
||||
logger.N("writing to output file: %s", outFile)
|
||||
err = ioutil.WriteFile(outFile, []byte(result), 0644)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not write to output file '%s': %s", outFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not write to output file '%s'", outFile)
|
||||
|
||||
handleCompression(outFile, []byte(result))
|
||||
|
||||
@@ -282,16 +278,16 @@ func (node *TreeNode) ProcessContent() {
|
||||
case "copy":
|
||||
from := node.InputPath + "/" + file
|
||||
to := node.OutputPath + "/" + file
|
||||
helper.Log.Noticef("copying file from '%s' to '%s'", from, to)
|
||||
logger.N("copying file from '%s' to '%s'", from, to)
|
||||
err := cpy.Copy(from, to)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
|
||||
}
|
||||
logger.Eexit(err, "could not copy file from '%s' to '%s': %s", from, to)
|
||||
|
||||
handleCompression(to, nil)
|
||||
}
|
||||
}
|
||||
|
||||
progress.IncrDone("content dir")
|
||||
|
||||
i := 0
|
||||
// sub can dynamically increase, so no for range
|
||||
for i < len(node.Sub) {
|
||||
|
||||
@@ -9,6 +9,8 @@ import (
|
||||
"time"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"gitbase.de/apairon/mark2web/pkg/progress"
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/extemporalgenome/slug"
|
||||
"github.com/flosch/pongo2"
|
||||
@@ -41,36 +43,33 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
|
||||
inPath += "/" + subDir
|
||||
}
|
||||
|
||||
helper.Log.Infof("reading input directory: %s", inPath)
|
||||
logger.I("reading input directory: %s", inPath)
|
||||
|
||||
node.InputPath = inPath
|
||||
|
||||
// read config
|
||||
newConfig := new(PathConfig)
|
||||
helper.Log.Debug("looking for config.yml ...")
|
||||
logger.D("looking for config.yml ...")
|
||||
configFile := inPath + "/config.yml"
|
||||
if _, err := os.Stat(configFile); os.IsNotExist(err) {
|
||||
helper.Log.Debug("no config.yml found in this directory, using upper configs")
|
||||
logger.D("no config.yml found in this directory, using upper configs")
|
||||
helper.Merge(newConfig, conf)
|
||||
// remove this
|
||||
newConfig.This = ThisPathConfig{}
|
||||
} else {
|
||||
helper.Log.Debug("reading config...")
|
||||
logger.D("reading config...")
|
||||
data, err := ioutil.ReadFile(configFile)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not read file '%s': %s", configFile, err)
|
||||
}
|
||||
err = yaml.Unmarshal(data, newConfig)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not parse YAML file '%s': %s", configFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not read file '%s'", configFile)
|
||||
|
||||
helper.Log.Debug("merging config with upper config")
|
||||
err = yaml.Unmarshal(data, newConfig)
|
||||
logger.Eexit(err, "could not parse YAML file '%s'", configFile)
|
||||
|
||||
logger.D("merging config with upper config")
|
||||
oldThis := newConfig.This
|
||||
helper.Merge(newConfig, conf)
|
||||
newConfig.This = oldThis
|
||||
|
||||
helper.Log.Debug(spew.Sdump(newConfig))
|
||||
logger.D(spew.Sdump(newConfig))
|
||||
}
|
||||
|
||||
node.Config = newConfig
|
||||
@@ -83,7 +82,7 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
|
||||
}
|
||||
if regexStr != nil && *regexStr != "" {
|
||||
if regex, err := regexp.Compile(*regexStr); err != nil {
|
||||
helper.Log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+subDir, err)
|
||||
logger.Eexit(err, "error compiling path.strip regex '%s' from '%s'", *regexStr, inBase+"/"+subDir)
|
||||
} else {
|
||||
stripedDir = regex.ReplaceAllString(stripedDir, "$1")
|
||||
}
|
||||
@@ -98,7 +97,7 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
|
||||
outPath := outBase + "/" + stripedDir
|
||||
outPath = path.Clean(outPath)
|
||||
|
||||
helper.Log.Infof("calculated output directory: %s", outPath)
|
||||
logger.I("calculated output directory: %s", outPath)
|
||||
node.OutputPath = outPath
|
||||
|
||||
// handle collections
|
||||
@@ -106,6 +105,9 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
|
||||
}
|
||||
|
||||
func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx interface{}, dataMapKey string, body string, hideInNav bool) {
|
||||
progress.IncrTotal("content dir")
|
||||
progress.DescribeCurrent("content dir", "subdir "+node.InputPath+"/"+subDir)
|
||||
|
||||
newNode := new(TreeNode)
|
||||
newNode.root = node.root
|
||||
|
||||
@@ -128,9 +130,8 @@ func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx
|
||||
|
||||
mergedConfig := new(PathConfig)
|
||||
err := helper.Merge(mergedConfig, node.Config)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("merge of path config failed: %s", err)
|
||||
}
|
||||
logger.Eexit(err, "merge of path config failed")
|
||||
|
||||
// dont merge Data[DataKey]
|
||||
if dataMapKey != "" {
|
||||
mergedConfig.Data[dataMapKey] = nil
|
||||
@@ -138,9 +139,7 @@ func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx
|
||||
mergedConfig.Data = make(helper.MapString)
|
||||
}
|
||||
err = helper.Merge(mergedConfig, newPathConfig)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("merge of path config failed: %s", err)
|
||||
}
|
||||
logger.Eexit(err, "merge of path config failed")
|
||||
|
||||
newNode.fillConfig(
|
||||
node.InputPath,
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
package mark2web
|
||||
|
||||
import (
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/webrequest"
|
||||
"github.com/flosch/pongo2"
|
||||
)
|
||||
|
||||
// RequestFn will make a web request and returns map[string]interface form pongo2
|
||||
func RequestFn(url *pongo2.Value, args ...*pongo2.Value) *pongo2.Value {
|
||||
u := url.String()
|
||||
return pongo2.AsValue(helper.JSONWebRequest(u))
|
||||
return pongo2.AsValue(webrequest.GetJSON(u))
|
||||
}
|
||||
|
||||
// RenderFn renders a pongo2 template with additional context
|
||||
|
||||
@@ -5,20 +5,18 @@ import (
|
||||
"regexp"
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
)
|
||||
|
||||
func htaccessRedirect(outDir, goTo string) {
|
||||
switch Config.Webserver.Type {
|
||||
case "apache":
|
||||
htaccessFile := outDir + "/.htaccess"
|
||||
helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goTo)
|
||||
logger.N("writing '%s' with redirect to: %s", htaccessFile, goTo)
|
||||
err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
|
||||
RewriteRule ^$ %{REQUEST_URI}`+goTo+`/ [R,L]
|
||||
`), 0644)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not write '%s'", htaccessFile)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -90,11 +88,9 @@ RemoveLanguage .br
|
||||
|
||||
if configStr != "" {
|
||||
htaccessFile := Config.Directories.Output + "/.htaccess"
|
||||
helper.Log.Noticef("writing webserver config to: %s", htaccessFile)
|
||||
logger.N("writing webserver config to: %s", htaccessFile)
|
||||
err := ioutil.WriteFile(htaccessFile, []byte(configStr), 0644)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
|
||||
}
|
||||
logger.Eexit(err, "could not write '%s'", htaccessFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
)
|
||||
|
||||
// NavElement is one element with ist attributes and subs
|
||||
@@ -41,11 +42,10 @@ func buildNavigationRecursive(tree *TreeNode, curNavMap *map[string]*NavElement,
|
||||
}
|
||||
if ignNav != nil && *ignNav != "" {
|
||||
regex, err := regexp.Compile(*ignNav)
|
||||
if err != nil {
|
||||
helper.Log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err)
|
||||
}
|
||||
logger.Eexit(err, "could not compile IngoreForNav regexp '%s' in '%s'", *ignNav, el.InputPath)
|
||||
|
||||
if regex.MatchString(path.Base(el.InputPath)) {
|
||||
helper.Log.Debugf("ignoring input directory '%s' in navigation", el.InputPath)
|
||||
logger.D("ignoring input directory '%s' in navigation", el.InputPath)
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
package mark2web
|
||||
|
||||
import "gitbase.de/apairon/mark2web/pkg/jobm"
|
||||
|
||||
// Run will do a complete run of mark2web
|
||||
func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
|
||||
SetTemplateDir(inDir + "/templates")
|
||||
@@ -12,5 +14,5 @@ func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
|
||||
|
||||
tree.WriteWebserverConfig()
|
||||
|
||||
Wait()
|
||||
jobm.Wait()
|
||||
}
|
||||
|
||||
@@ -1,60 +0,0 @@
|
||||
package mark2web
|
||||
|
||||
import (
|
||||
"runtime"
|
||||
"sync"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
)
|
||||
|
||||
var wg sync.WaitGroup
|
||||
var numCPU = runtime.NumCPU()
|
||||
var curNumThreads = 1 // main thread is 1
|
||||
|
||||
func init() {
|
||||
helper.Log.Infof("number of CPU core: %d", numCPU)
|
||||
}
|
||||
|
||||
// Wait will wait for all our internal go threads
|
||||
func Wait() {
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// ThreadSetup adds 1 to wait group
|
||||
func ThreadSetup() {
|
||||
curNumThreads++
|
||||
wg.Add(1)
|
||||
}
|
||||
|
||||
// ThreadDone removes 1 from wait group
|
||||
func ThreadDone() {
|
||||
curNumThreads--
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
// ThreadStart will start a thread an manages the wait group
|
||||
func ThreadStart(f func(), forceNewThread ...bool) {
|
||||
force := false
|
||||
if len(forceNewThread) > 0 && forceNewThread[0] {
|
||||
force = true
|
||||
}
|
||||
|
||||
if numCPU > curNumThreads || force {
|
||||
// only new thread if empty CPU core available or forced
|
||||
threadF := func() {
|
||||
f()
|
||||
ThreadDone()
|
||||
}
|
||||
|
||||
ThreadSetup()
|
||||
go threadF()
|
||||
} else {
|
||||
helper.Log.Debugf("no more CPU core (%d used), staying in main thread", curNumThreads)
|
||||
f()
|
||||
}
|
||||
}
|
||||
|
||||
// SetNumCPU is for testing package without threading
|
||||
func SetNumCPU(i int) {
|
||||
numCPU = i
|
||||
}
|
||||
96
pkg/progress/bar.go
Normal file
96
pkg/progress/bar.go
Normal file
@@ -0,0 +1,96 @@
|
||||
package progress
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/helper"
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
"github.com/gosuri/uiprogress"
|
||||
"github.com/mattn/go-tty"
|
||||
)
|
||||
|
||||
type bar struct {
|
||||
Bar *uiprogress.Bar
|
||||
Description string
|
||||
}
|
||||
|
||||
var bars = make(map[string]*bar)
|
||||
var initialized = false
|
||||
var terminalWidth = 80
|
||||
|
||||
// OverallTotal is number of total jobs
|
||||
var OverallTotal = 0
|
||||
|
||||
// OverallDone is number of done jobs
|
||||
var OverallDone = 0
|
||||
|
||||
func init() {
|
||||
updateLoggerPrefix()
|
||||
}
|
||||
|
||||
// Start initializes the bar drawing
|
||||
func Start() {
|
||||
if t, err := tty.Open(); err == nil && t != nil {
|
||||
terminalWidth, _, _ = t.Size()
|
||||
t.Close()
|
||||
}
|
||||
uiprogress.Start() // start rendering
|
||||
initialized = true
|
||||
}
|
||||
|
||||
func updateLoggerPrefix() {
|
||||
logger.Prefix = fmt.Sprintf("%3d/%3d: ", OverallDone, OverallTotal)
|
||||
}
|
||||
|
||||
// IncrTotal increases the total jobs for the bar
|
||||
func IncrTotal(barname string) {
|
||||
OverallTotal++
|
||||
updateLoggerPrefix()
|
||||
|
||||
if initialized {
|
||||
_bar := bars[barname]
|
||||
if _bar == nil {
|
||||
_bar = new(bar)
|
||||
_bar.Bar = uiprogress.AddBar(1)
|
||||
_bar.Bar.Width = 25
|
||||
|
||||
_bar.Bar.PrependFunc(func(b *uiprogress.Bar) string {
|
||||
return fmt.Sprintf("%15s: %3d/%3d", helper.ShortenStringLeft(barname, 15), b.Current(), b.Total)
|
||||
})
|
||||
_bar.Bar.AppendFunc(func(b *uiprogress.Bar) string {
|
||||
return fmt.Sprintf("%s", helper.ShortenStringLeft(_bar.Description, terminalWidth-80))
|
||||
})
|
||||
|
||||
bars[barname] = _bar
|
||||
} else {
|
||||
_bar.Bar.Total++
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IncrDone increases to done jobs counter
|
||||
func IncrDone(barname string) {
|
||||
OverallDone++
|
||||
updateLoggerPrefix()
|
||||
|
||||
if initialized {
|
||||
bars[barname].Bar.Incr()
|
||||
bars[barname].Description = ""
|
||||
}
|
||||
}
|
||||
|
||||
// DescribeCurrent describes the current job
|
||||
func DescribeCurrent(barname, description string) {
|
||||
if initialized {
|
||||
bars[barname].Description = description
|
||||
}
|
||||
}
|
||||
|
||||
// Stop stops the bar drawing
|
||||
func Stop() {
|
||||
if initialized {
|
||||
time.Sleep(time.Millisecond * 200)
|
||||
uiprogress.Stop()
|
||||
}
|
||||
}
|
||||
109
pkg/webrequest/request.go
Normal file
109
pkg/webrequest/request.go
Normal file
@@ -0,0 +1,109 @@
|
||||
package webrequest
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"image"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/progress"
|
||||
|
||||
"gitbase.de/apairon/mark2web/pkg/logger"
|
||||
)
|
||||
|
||||
type wrImageEntry struct {
|
||||
img image.Image
|
||||
format string
|
||||
}
|
||||
|
||||
type wrJSONEntry struct {
|
||||
data interface{}
|
||||
}
|
||||
|
||||
var wrImageCache = make(map[string]*wrImageEntry)
|
||||
var wrJSONCache = make(map[string]*wrJSONEntry)
|
||||
|
||||
// Get will fetch an url and returns reponse
|
||||
func Get(url string, opts interface{}) (resp *http.Response, err error) {
|
||||
logger.N("requesting url via GET %s", url)
|
||||
|
||||
progress.IncrTotal("web request")
|
||||
progress.DescribeCurrent("web request", url)
|
||||
resp, err = http.Get(url)
|
||||
progress.IncrDone("web request")
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// GetJSON will GET a json object/array from a given URL
|
||||
func GetJSON(url string) interface{} {
|
||||
cached := wrJSONCache[url]
|
||||
if cached == nil {
|
||||
resp, err := Get(url, nil)
|
||||
logger.Eexit(err, "could not get url '%s'", url)
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
logger.Eexit(err, "could not read body from url '%s'", url)
|
||||
|
||||
logger.D("output from url '%s':\n%s", url, string(body))
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
logger.Exit("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
|
||||
}
|
||||
|
||||
contentType := resp.Header.Get("Content-Type")
|
||||
|
||||
if strings.Contains(contentType, "json") {
|
||||
|
||||
} else {
|
||||
logger.Exit("is not json '%s' from url '%s'", contentType, url)
|
||||
}
|
||||
|
||||
cached = new(wrJSONEntry)
|
||||
|
||||
jsonMap := make(map[string]interface{})
|
||||
err = json.Unmarshal(body, &jsonMap)
|
||||
if err == nil {
|
||||
cached.data = jsonMap
|
||||
} else {
|
||||
jsonArrayMap := make([]map[string]interface{}, 0)
|
||||
err = json.Unmarshal(body, &jsonArrayMap)
|
||||
if err == nil {
|
||||
cached.data = jsonArrayMap
|
||||
} else {
|
||||
logger.Exit("could not read json from '%s': invalid type", url)
|
||||
}
|
||||
}
|
||||
|
||||
wrJSONCache[url] = cached
|
||||
|
||||
}
|
||||
return cached.data
|
||||
}
|
||||
|
||||
// GetImage gets an image from an url
|
||||
func GetImage(url string) (image.Image, string, error) {
|
||||
cached := wrImageCache[url]
|
||||
if cached == nil {
|
||||
resp, err := Get(url, nil)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
|
||||
}
|
||||
|
||||
img, format, err := image.Decode(resp.Body)
|
||||
if err != nil {
|
||||
return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
|
||||
}
|
||||
|
||||
cached = &wrImageEntry{
|
||||
img: img,
|
||||
format: format,
|
||||
}
|
||||
|
||||
wrImageCache[url] = cached
|
||||
}
|
||||
|
||||
return cached.img, cached.format, nil
|
||||
}
|
||||
1
vendor/github.com/gosuri/uilive
generated
vendored
Submodule
1
vendor/github.com/gosuri/uilive
generated
vendored
Submodule
Submodule vendor/github.com/gosuri/uilive added at ac356e6e42
1
vendor/github.com/gosuri/uiprogress
generated
vendored
Submodule
1
vendor/github.com/gosuri/uiprogress
generated
vendored
Submodule
Submodule vendor/github.com/gosuri/uiprogress added at d0567a9d84
1
vendor/github.com/mattn/go-tty
generated
vendored
Submodule
1
vendor/github.com/mattn/go-tty
generated
vendored
Submodule
Submodule vendor/github.com/mattn/go-tty added at 5518497423
1
vendor/golang.org/x/sys
generated
vendored
Submodule
1
vendor/golang.org/x/sys
generated
vendored
Submodule
Submodule vendor/golang.org/x/sys added at f49334f85d
@@ -1,4 +1,6 @@
|
||||
---
|
||||
Template: base_doc.html
|
||||
|
||||
Data:
|
||||
background: /img/folder.jpg
|
||||
|
||||
@@ -9,9 +11,7 @@ Data:
|
||||
|
||||
---
|
||||
|
||||
# Benutzung
|
||||
|
||||
## Ordnerstruktur
|
||||
# Ordnerstruktur
|
||||
|
||||
Das Ausgangsmaterial für die zu generierende Website wird in folgender Ordnerstruktur organisiert:
|
||||
|
||||
@@ -58,25 +58,3 @@ FIL config.yml (globale Konfiguration, enthält andere Anweisungen als individue
|
||||
```
|
||||
|
||||
In der Minimal-Variante sind nur die Ordner `content` und `templates` nötig.
|
||||
|
||||
---
|
||||
|
||||
### `content`
|
||||
|
||||
- enthält die Markdown-Dateien und Konfigurationen für die Navigationsstruktur und Einzelseiten
|
||||
- voranestellte Nummer mit Unterstrich wie z.B. `01_` dienen nur der Sortierung und gehen nicht in den eigentlichen Navigationspfad mit ein
|
||||
- zur Bildung des Navigationspfades werden die Verzeichnisnamen in Kleinschreibung konvertiert
|
||||
- Navigationsnamen für die Website werden aus dem Pfad gebildet, wobei `_`(Unterstriche) in Leerzeichen umgewandelt werden
|
||||
- Navigationsnamen können durch die `config.yml` überschrieben werden
|
||||
|
||||
#### Medien und Downloads
|
||||
|
||||
- Mediendateien werden neben den Inhalten in `content` abgelegt und müssen dementsprechend relativ verlinkt werden
|
||||
|
||||
### `assets`
|
||||
|
||||
- Bilder/CSS/JS die in Templates oder mehrfach in den Content-Seiten benötigt werden liegen in `assets`
|
||||
|
||||
### `templates`
|
||||
|
||||
- Template-Dateien für die Generierung der Website liegen hier
|
||||
@@ -0,0 +1,11 @@
|
||||
---
|
||||
Data:
|
||||
Version: "ab v1.0"
|
||||
|
||||
---
|
||||
|
||||
- enthält die Markdown-Dateien und Konfigurationen für die Navigationsstruktur und Einzelseiten
|
||||
- voranestellte Nummer mit Unterstrich wie z.B. `01_` dienen nur der Sortierung und gehen nicht in den eigentlichen Navigationspfad mit ein
|
||||
- zur Bildung des Navigationspfades werden die Verzeichnisnamen in Kleinschreibung konvertiert
|
||||
- Navigationsnamen für die Website werden aus dem Pfad gebildet, wobei `_`(Unterstriche) in Leerzeichen umgewandelt werden
|
||||
- Navigationsnamen können durch die `config.yml` überschrieben werden
|
||||
@@ -0,0 +1 @@
|
||||
- Mediendateien werden neben den Inhalten in `content` abgelegt und müssen dementsprechend relativ verlinkt werden
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
Data:
|
||||
Version: "ab v1.0"
|
||||
|
||||
---
|
||||
|
||||
- Bilder/CSS/JS die in Templates oder mehrfach in den Content-Seiten benötigt werden liegen in `assets`
|
||||
@@ -0,0 +1,7 @@
|
||||
---
|
||||
Data:
|
||||
Version: "ab v1.0"
|
||||
|
||||
---
|
||||
|
||||
- Template-Dateien für die Generierung der Website liegen hier
|
||||
@@ -1,3 +1,9 @@
|
||||
This:
|
||||
Data:
|
||||
teaser: Wie werden die Inhalte und Templates organisiert?
|
||||
Collections:
|
||||
- Name: doccoll
|
||||
Directory:
|
||||
Path: "."
|
||||
MatchFilename: "^_\\d+(?P<lowdash>_*)(?P<title>.+)\\.md"
|
||||
ReverseOrder: False
|
||||
|
||||
@@ -1,4 +1,6 @@
|
||||
---
|
||||
Template: base_doc.html
|
||||
|
||||
Data:
|
||||
background: /img/design.jpg
|
||||
|
||||
@@ -15,361 +17,3 @@ Templates werden über das pongo2-Paket gerendert. Dieses nutzt die Template-Spr
|
||||
|
||||
Sämtliche Template-Dateien sind im Ordner `templates` zu speichern.
|
||||
Die Endung kann frei gewählt werden. Für diese Dokumentation und auch als Grundlage für Beispiele wurde `.html` gewählt, da somit auch das Syntax-Highlighting gegeben ist.
|
||||
|
||||
## grober Überblick
|
||||
|
||||
Nachfolgend ist ein Beispiel eines Templates:
|
||||
|
||||
```django
|
||||
<html>
|
||||
<meta>
|
||||
<meta charset="UTF-8">
|
||||
|
||||
<title>{{ Meta.Title }}</title>
|
||||
<meta name="description" content="{{ Meta.Description }}" />
|
||||
<meta name="keywords" content="{{ Meta.Keywords }}" />
|
||||
<link rel="stylesheet" type="text/css" href="../assets/css/main.css">
|
||||
</meta>
|
||||
|
||||
<body>
|
||||
{% block header %}
|
||||
<header>
|
||||
<div class="langSelect">
|
||||
{% for nav in NavSlice %}
|
||||
<a href="{{ nav.GoTo }}" {% if nav.Active %}class="active"{% endif %}>
|
||||
<img src="../assets/img/{{ nav.Navname }}.png" alt="{{ nav.Navname }}" style="height: 20px;">
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div id="logoDiv"><img src="../assets/img/logo.png"></div>
|
||||
<div id="mainNavDiv" class="nav">
|
||||
<table border="0" style="width: 100%">
|
||||
<tr>
|
||||
<td>
|
||||
<div><b>main Level 1 ...</b></div>
|
||||
<ul>
|
||||
{% for nav in NavActive.0.SubMap.main.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
{% if nav.SubSlice %}
|
||||
<ul>
|
||||
{% for nav2 in nav.SubSlice %}
|
||||
<li {% if nav2.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
|
||||
{{ nav2.Navname }}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</td>
|
||||
<td>
|
||||
<div><b>main/service Level 2</b></div>
|
||||
<ul>
|
||||
{% for nav in NavActive.2.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</header>
|
||||
{% endblock %}
|
||||
|
||||
|
||||
{% block breadcrumb %}
|
||||
<div id="breadcrumb">
|
||||
{% for nav in NavActive %}
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div id="content">
|
||||
{{ Body }}
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block footer %}
|
||||
<footer class="nav">
|
||||
<div><b>service Level 1</b></div>
|
||||
<ul>
|
||||
{% for nav in NavActive.0.SubMap.service.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</footer>
|
||||
{% endblock %}
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
```
|
||||
|
||||
Wie im Beispiel zu sehen ist, werden einfache **Variables** über:
|
||||
|
||||
```django
|
||||
{{ Variable }}
|
||||
```
|
||||
|
||||
eingebunden. Variablen können außerdem speziell weiterverarbeitet werden. Dies geschieht mit sogenannten Filtern oder Filterfunktionen. Die Syntax dafür ist folgendermaßen:
|
||||
|
||||
```django
|
||||
{{ Variable|Filter }}
|
||||
```
|
||||
|
||||
Blockanweisungen dagegen verwenden zum Beispiel folgende Platzhalter:
|
||||
|
||||
```django
|
||||
{% if Variable %}
|
||||
...
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
Eine Liste der in Django möglichen Anweisungen finden Sie unter [Django builtins](https://docs.djangoproject.com/en/2.1/ref/templates/builtins/).
|
||||
|
||||
---
|
||||
|
||||
## mark2web Variablen
|
||||
|
||||
Der mark2web-Generator liefert für die Template-Verarbeitung Variablen für die Navigation und den Inhalt.
|
||||
|
||||
### Website-Inhalt
|
||||
|
||||
Das rohe HTML, welches aus einer Markdown-Datei generiert wird steht über folgende Variablen zur Verfügung.
|
||||
|
||||
```django
|
||||
{{ Body }} = komplettes HTML aus der Markdown-Datei
|
||||
{{ BodyParts.0 }} = erster HTML-Block
|
||||
{{ BodyParts.1 }} = zweiter HTML-Block
|
||||
usw.
|
||||
```
|
||||
|
||||
Ist die Markdown-Datei durch `---` auf einer Zeile (nach den Kopfdaten) geteilt, stehen die Einzelteile im Slice/Array `{{ BodyParts }}` zur Verfügung.
|
||||
|
||||
Aus folgender Markdown-Datei `README.md` in einem `content`-Unterverzeichnis:
|
||||
|
||||
```markdown
|
||||
# Titel 1
|
||||
|
||||
Text 1
|
||||
|
||||
---
|
||||
|
||||
## Titel 2
|
||||
|
||||
Text 2
|
||||
```
|
||||
|
||||
wird für `{{ Body }}` folgendes HTML:
|
||||
|
||||
```html
|
||||
<h1>Titel 1</h1>
|
||||
<p>Text 1</p>
|
||||
<hr>
|
||||
<h2>Titel 2</h2>
|
||||
<p>Text 2</p>
|
||||
```
|
||||
|
||||
`BodyParts` erklärt sich an folgendem Template:
|
||||
|
||||
```django
|
||||
<table>
|
||||
<tr>
|
||||
{% for part in BodyParts %}
|
||||
<td>
|
||||
{{ part }}
|
||||
</td>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
</table>
|
||||
```
|
||||
|
||||
Aus dem Template wird nach dem Rendern mit obiger Markdown-Datei also folgendes HTML:
|
||||
|
||||
```html
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<h1>Titel 1</h1>
|
||||
<p>Text 1</p>
|
||||
</td>
|
||||
<td>
|
||||
<h2>Titel 2</h2>
|
||||
<p>Text 2</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
```
|
||||
|
||||
Die Einrückungen im HTML wurden für die bessere Lesbarkeit angepasst. Wie zu sehen ist, wird `---` in `{{ Body }}` laut Markdown-Syntax zu `<br>`. In `{{ BodyParts.N }}` ist es jedoch nicht enthalten, da es hier nur zur Trennung des Dokuments dient.
|
||||
|
||||
### Navigation
|
||||
|
||||
Jedes Navigationselement steht intern in folgender go-Struktur zur Verfügung:
|
||||
|
||||
```go
|
||||
type navElement struct {
|
||||
Navname string
|
||||
GoTo string
|
||||
Active bool
|
||||
|
||||
Data interface{}
|
||||
|
||||
This ThisPathConfig
|
||||
|
||||
SubMap *map[string]*navElement
|
||||
SubSlice *[]*navElement
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
Diese erste Navigationsebene wird mit seinen Unternavigationspunkten zum einen auf die Variable `{{ NavMap }}` in Form einer Map (assoziatives Array) mit dem umgeformten Namen (wie im Zielverzeichnis) abgebildet.
|
||||
Außerdem steht die erste Navigationsebene als Liste, bzw. Slice (Array) über die Variable `{{ NavSlice }}` zur verfügung.
|
||||
|
||||
Wird z.B. folgende Navigation als Zielverzeichnis-Struktur angenommen:
|
||||
|
||||
```plain
|
||||
de
|
||||
main
|
||||
home
|
||||
leistungen
|
||||
referenzen
|
||||
service
|
||||
impressum
|
||||
en
|
||||
main
|
||||
home
|
||||
...
|
||||
```
|
||||
|
||||
Der Teasertext aus folgender `config.yml` im `content`-Verzeichnis `de/main/02_Leistungen`
|
||||
|
||||
```yaml
|
||||
This:
|
||||
Data:
|
||||
teaser: Teasertext
|
||||
```
|
||||
|
||||
welcher zum Navigationspunkt im Zielpfad *de/main/leistungen* gehört, ist über folgende Template-Variablen erreichbar:
|
||||
|
||||
```django
|
||||
{{ NavMap.de.SubMap.main.SubMap.leistungen.This.Data.teaser }}
|
||||
|
||||
oder
|
||||
|
||||
{{ NavSlice.0.SubSlice.0.SubSlice.1.This.Data.teaser}}
|
||||
|
||||
oder auch eine Kombination
|
||||
|
||||
{{ NavMap.de.SubMap.main.SubSlice.1.This.Data.teaser }}
|
||||
```
|
||||
|
||||
Natürlich wird diese Variable in der Form so nie verwendet, sondern soll nur den Aufbau der Struktur verdeutlichen. Üblicherweise werden Schleifenblöcke verwendet um die Navigationsbäume auszugeben, wie z.B. eine Liste als Sprachwähler, wenn man annimmt, dass die erste Navigationsebene die Website-Sprache ist:
|
||||
|
||||
```django
|
||||
<ul>
|
||||
{% for lang in NavMap %}
|
||||
<li {% if lang.Active %}class="active"{% endif %}>
|
||||
<a href="{{ lang.GoTo }}">{{ lang.Navname }}</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
```
|
||||
|
||||
Wie im Beispiel zu sehen ist, wird das aktive Navigationselement mit `class="active"` über die Variable `Active` aus der Struktur markiert.
|
||||
|
||||
#### aktiver Navigationspfad
|
||||
|
||||
Der aktive Navigationspfad ist über eine weitere vorbelegte Variable zu erfahren:
|
||||
|
||||
```django
|
||||
{{ NavActive }}
|
||||
```
|
||||
|
||||
Ähnlich wie `{{ NavSlice }}` oder `{{ ...SubSlice }}` ist dies ein Slice/Array welches als Elemnte Navigationselemente aus oben angegebener Struktur enthält.
|
||||
Im Gegensatz zu `{{ NavSlice }}` besteht die Liste nicht aus Elementen einer Ebene, sonder aus allen aktiven Elemtenten in des aktuellen Pfads.
|
||||
|
||||
Geht man also wieder vom obigen Beispiel aus und der aktive Pfad ist *de/main/leistungen*, so würden folgendes zutreffen:
|
||||
|
||||
```django
|
||||
{{ NavActive.0 }} ist das Navigationselement für "de"
|
||||
{{ NavActive.1 }} ist das Navigationselement für "main"
|
||||
{{ NavActive.2 }} ist das Navigationselement für "Leistungen"
|
||||
```
|
||||
|
||||
Somit lassen sich leicht Pfade anzeigen, bzw. Breadcrumbs in die Website einbinden, wie im folgenden Beispiel:
|
||||
|
||||
```django
|
||||
aktiver Pfad:
|
||||
{% for nav in NavActive %}
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">{{ nav.Navname }}</a>
|
||||
{% endfor %}
|
||||
```
|
||||
|
||||
Ebenso lässt sich bei mehrsprachigen Seite immer die richte Hauptnavigation zur aktuelle Sprache laden:
|
||||
|
||||
```django
|
||||
<h3>Hauptnavigation</h3>
|
||||
<ul>
|
||||
{% for nav in NavActive.0.SubMap.main.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
{% if nav.SubSlice %}
|
||||
<ul>
|
||||
{% for nav2 in nav.SubSlice %}
|
||||
<li {% if nav2.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
|
||||
{{ nav2.Navname }}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
```
|
||||
|
||||
### Meta-Angaben
|
||||
|
||||
Über die Variablen
|
||||
|
||||
```django
|
||||
{{ Meta.Title }}
|
||||
{{ Meta.Description }}
|
||||
{{ Meta.Keywords }}
|
||||
```
|
||||
|
||||
stehen die üblichen Meta-Angaben für die Verwendung im `<head>` Tag zur Verfügung.
|
||||
|
||||
### weitere Daten
|
||||
|
||||
Die Variablen
|
||||
|
||||
```django
|
||||
{{ This.Navname }}
|
||||
{{ This.Data }}
|
||||
{{ Data }}
|
||||
```
|
||||
|
||||
stehen ebenfalls zur Verfügung und spiegeln die Daten aus den Konfig-Dateien `config.yml` und den Kopfdaten der Markdown-Datei wieder.
|
||||
@@ -0,0 +1,125 @@
|
||||
Nachfolgend ist ein Beispiel eines Templates:
|
||||
|
||||
```django
|
||||
<html>
|
||||
<meta>
|
||||
<meta charset="UTF-8">
|
||||
|
||||
<title>{{ Meta.Title }}</title>
|
||||
<meta name="description" content="{{ Meta.Description }}" />
|
||||
<meta name="keywords" content="{{ Meta.Keywords }}" />
|
||||
<link rel="stylesheet" type="text/css" href="../assets/css/main.css">
|
||||
</meta>
|
||||
|
||||
<body>
|
||||
{% block header %}
|
||||
<header>
|
||||
<div class="langSelect">
|
||||
{% for nav in NavSlice %}
|
||||
<a href="{{ nav.GoTo }}" {% if nav.Active %}class="active"{% endif %}>
|
||||
<img src="../assets/img/{{ nav.Navname }}.png" alt="{{ nav.Navname }}" style="height: 20px;">
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
<div id="logoDiv"><img src="../assets/img/logo.png"></div>
|
||||
<div id="mainNavDiv" class="nav">
|
||||
<table border="0" style="width: 100%">
|
||||
<tr>
|
||||
<td>
|
||||
<div><b>main Level 1 ...</b></div>
|
||||
<ul>
|
||||
{% for nav in NavActive.0.SubMap.main.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
{% if nav.SubSlice %}
|
||||
<ul>
|
||||
{% for nav2 in nav.SubSlice %}
|
||||
<li {% if nav2.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
|
||||
{{ nav2.Navname }}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</td>
|
||||
<td>
|
||||
<div><b>main/service Level 2</b></div>
|
||||
<ul>
|
||||
{% for nav in NavActive.2.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
</div>
|
||||
</header>
|
||||
{% endblock %}
|
||||
|
||||
|
||||
{% block breadcrumb %}
|
||||
<div id="breadcrumb">
|
||||
{% for nav in NavActive %}
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block content %}
|
||||
<div id="content">
|
||||
{{ Body }}
|
||||
</div>
|
||||
{% endblock %}
|
||||
|
||||
{% block footer %}
|
||||
<footer class="nav">
|
||||
<div><b>service Level 1</b></div>
|
||||
<ul>
|
||||
{% for nav in NavActive.0.SubMap.service.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
</footer>
|
||||
{% endblock %}
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
||||
```
|
||||
|
||||
Wie im Beispiel zu sehen ist, werden einfache **Variables** über:
|
||||
|
||||
```django
|
||||
{{ Variable }}
|
||||
```
|
||||
|
||||
eingebunden. Variablen können außerdem speziell weiterverarbeitet werden. Dies geschieht mit sogenannten Filtern oder Filterfunktionen. Die Syntax dafür ist folgendermaßen:
|
||||
|
||||
```django
|
||||
{{ Variable|Filter }}
|
||||
```
|
||||
|
||||
Blockanweisungen dagegen verwenden zum Beispiel folgende Platzhalter:
|
||||
|
||||
```django
|
||||
{% if Variable %}
|
||||
...
|
||||
{% endif %}
|
||||
```
|
||||
|
||||
Eine Liste der in Django möglichen Anweisungen finden Sie unter [Django builtins](https://docs.djangoproject.com/en/2.1/ref/templates/builtins/).
|
||||
@@ -0,0 +1 @@
|
||||
Der mark2web-Generator liefert für die Template-Verarbeitung Variablen für die Navigation und den Inhalt.
|
||||
@@ -0,0 +1,67 @@
|
||||
Das rohe HTML, welches aus einer Markdown-Datei generiert wird steht über folgende Variablen zur Verfügung.
|
||||
|
||||
```django
|
||||
{{ Body }} = komplettes HTML aus der Markdown-Datei
|
||||
{{ BodyParts.0 }} = erster HTML-Block
|
||||
{{ BodyParts.1 }} = zweiter HTML-Block
|
||||
usw.
|
||||
```
|
||||
|
||||
Ist die Markdown-Datei durch `---` auf einer Zeile (nach den Kopfdaten) geteilt, stehen die Einzelteile im Slice/Array `{{ BodyParts }}` zur Verfügung.
|
||||
|
||||
Aus folgender Markdown-Datei `README.md` in einem `content`-Unterverzeichnis:
|
||||
|
||||
```markdown
|
||||
# Titel 1
|
||||
|
||||
Text 1
|
||||
|
||||
---
|
||||
|
||||
## Titel 2
|
||||
|
||||
Text 2
|
||||
```
|
||||
|
||||
wird für `{{ Body }}` folgendes HTML:
|
||||
|
||||
```html
|
||||
<h1>Titel 1</h1>
|
||||
<p>Text 1</p>
|
||||
<hr>
|
||||
<h2>Titel 2</h2>
|
||||
<p>Text 2</p>
|
||||
```
|
||||
|
||||
`BodyParts` erklärt sich an folgendem Template:
|
||||
|
||||
```django
|
||||
<table>
|
||||
<tr>
|
||||
{% for part in BodyParts %}
|
||||
<td>
|
||||
{{ part }}
|
||||
</td>
|
||||
{% endfor %}
|
||||
</tr>
|
||||
</table>
|
||||
```
|
||||
|
||||
Aus dem Template wird nach dem Rendern mit obiger Markdown-Datei also folgendes HTML:
|
||||
|
||||
```html
|
||||
<table>
|
||||
<tr>
|
||||
<td>
|
||||
<h1>Titel 1</h1>
|
||||
<p>Text 1</p>
|
||||
</td>
|
||||
<td>
|
||||
<h2>Titel 2</h2>
|
||||
<p>Text 2</p>
|
||||
</td>
|
||||
</tr>
|
||||
</table>
|
||||
```
|
||||
|
||||
Die Einrückungen im HTML wurden für die bessere Lesbarkeit angepasst. Wie zu sehen ist, wird `---` in `{{ Body }}` laut Markdown-Syntax zu `<br>`. In `{{ BodyParts.N }}` ist es jedoch nicht enthalten, da es hier nur zur Trennung des Dokuments dient.
|
||||
@@ -0,0 +1,72 @@
|
||||
Jedes Navigationselement steht intern in folgender go-Struktur zur Verfügung:
|
||||
|
||||
```go
|
||||
type navElement struct {
|
||||
Navname string
|
||||
GoTo string
|
||||
Active bool
|
||||
|
||||
Data interface{}
|
||||
|
||||
This ThisPathConfig
|
||||
|
||||
SubMap *map[string]*navElement
|
||||
SubSlice *[]*navElement
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
Diese erste Navigationsebene wird mit seinen Unternavigationspunkten zum einen auf die Variable `{{ NavMap }}` in Form einer Map (assoziatives Array) mit dem umgeformten Namen (wie im Zielverzeichnis) abgebildet.
|
||||
Außerdem steht die erste Navigationsebene als Liste, bzw. Slice (Array) über die Variable `{{ NavSlice }}` zur verfügung.
|
||||
|
||||
Wird z.B. folgende Navigation als Zielverzeichnis-Struktur angenommen:
|
||||
|
||||
```plain
|
||||
de
|
||||
main
|
||||
home
|
||||
leistungen
|
||||
referenzen
|
||||
service
|
||||
impressum
|
||||
en
|
||||
main
|
||||
home
|
||||
...
|
||||
```
|
||||
|
||||
Der Teasertext aus folgender `config.yml` im `content`-Verzeichnis `de/main/02_Leistungen`
|
||||
|
||||
```yaml
|
||||
This:
|
||||
Data:
|
||||
teaser: Teasertext
|
||||
```
|
||||
|
||||
welcher zum Navigationspunkt im Zielpfad *de/main/leistungen* gehört, ist über folgende Template-Variablen erreichbar:
|
||||
|
||||
```django
|
||||
{{ NavMap.de.SubMap.main.SubMap.leistungen.This.Data.teaser }}
|
||||
|
||||
oder
|
||||
|
||||
{{ NavSlice.0.SubSlice.0.SubSlice.1.This.Data.teaser}}
|
||||
|
||||
oder auch eine Kombination
|
||||
|
||||
{{ NavMap.de.SubMap.main.SubSlice.1.This.Data.teaser }}
|
||||
```
|
||||
|
||||
Natürlich wird diese Variable in der Form so nie verwendet, sondern soll nur den Aufbau der Struktur verdeutlichen. Üblicherweise werden Schleifenblöcke verwendet um die Navigationsbäume auszugeben, wie z.B. eine Liste als Sprachwähler, wenn man annimmt, dass die erste Navigationsebene die Website-Sprache ist:
|
||||
|
||||
```django
|
||||
<ul>
|
||||
{% for lang in NavMap %}
|
||||
<li {% if lang.Active %}class="active"{% endif %}>
|
||||
<a href="{{ lang.GoTo }}">{{ lang.Navname }}</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
```
|
||||
|
||||
Wie im Beispiel zu sehen ist, wird das aktive Navigationselement mit `class="active"` über die Variable `Active` aus der Struktur markiert.
|
||||
@@ -0,0 +1,52 @@
|
||||
Der aktive Navigationspfad ist über eine weitere vorbelegte Variable zu erfahren:
|
||||
|
||||
```django
|
||||
{{ NavActive }}
|
||||
```
|
||||
|
||||
Ähnlich wie `{{ NavSlice }}` oder `{{ ...SubSlice }}` ist dies ein Slice/Array welches als Elemnte Navigationselemente aus oben angegebener Struktur enthält.
|
||||
Im Gegensatz zu `{{ NavSlice }}` besteht die Liste nicht aus Elementen einer Ebene, sonder aus allen aktiven Elemtenten in des aktuellen Pfads.
|
||||
|
||||
Geht man also wieder vom obigen Beispiel aus und der aktive Pfad ist *de/main/leistungen*, so würden folgendes zutreffen:
|
||||
|
||||
```django
|
||||
{{ NavActive.0 }} ist das Navigationselement für "de"
|
||||
{{ NavActive.1 }} ist das Navigationselement für "main"
|
||||
{{ NavActive.2 }} ist das Navigationselement für "Leistungen"
|
||||
```
|
||||
|
||||
Somit lassen sich leicht Pfade anzeigen, bzw. Breadcrumbs in die Website einbinden, wie im folgenden Beispiel:
|
||||
|
||||
```django
|
||||
aktiver Pfad:
|
||||
{% for nav in NavActive %}
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">{{ nav.Navname }}</a>
|
||||
{% endfor %}
|
||||
```
|
||||
|
||||
Ebenso lässt sich bei mehrsprachigen Seite immer die richte Hauptnavigation zur aktuelle Sprache laden:
|
||||
|
||||
```django
|
||||
<h3>Hauptnavigation</h3>
|
||||
<ul>
|
||||
{% for nav in NavActive.0.SubMap.main.SubSlice %}
|
||||
<li {% if nav.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
|
||||
{{ nav.Navname }}
|
||||
</a>
|
||||
{% if nav.SubSlice %}
|
||||
<ul>
|
||||
{% for nav2 in nav.SubSlice %}
|
||||
<li {% if nav2.Active %}class="active"{% endif %}>
|
||||
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
|
||||
{{ nav2.Navname }}
|
||||
</a>
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul
|
||||
{% endif %}
|
||||
</li>
|
||||
{% endfor %}
|
||||
</ul>
|
||||
|
||||
```
|
||||
@@ -0,0 +1,9 @@
|
||||
Über die Variablen
|
||||
|
||||
```django
|
||||
{{ Meta.Title }}
|
||||
{{ Meta.Description }}
|
||||
{{ Meta.Keywords }}
|
||||
```
|
||||
|
||||
stehen die üblichen Meta-Angaben für die Verwendung im `<head>` Tag zur Verfügung.
|
||||
@@ -0,0 +1,9 @@
|
||||
Die Variablen
|
||||
|
||||
```django
|
||||
{{ This.Navname }}
|
||||
{{ This.Data }}
|
||||
{{ Data }}
|
||||
```
|
||||
|
||||
stehen ebenfalls zur Verfügung und spiegeln die Daten aus den Konfig-Dateien `config.yml` und den Kopfdaten der Markdown-Datei wieder.
|
||||
@@ -1,3 +1,9 @@
|
||||
This:
|
||||
Data:
|
||||
teaser: Aus Markdown wird HTML
|
||||
Collections:
|
||||
- Name: doccoll
|
||||
Directory:
|
||||
Path: "."
|
||||
MatchFilename: "^_\\d+(?P<lowdash>_*)(?P<title>.+)\\.md"
|
||||
ReverseOrder: False
|
||||
@@ -1,7 +1,7 @@
|
||||
#This:
|
||||
This:
|
||||
Collections:
|
||||
- Name: blog1st
|
||||
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}'
|
||||
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1'
|
||||
NavTemplate:
|
||||
EntriesAttribute: entries
|
||||
GoTo: '{{ date }}-{{ title }}'
|
||||
@@ -12,7 +12,7 @@
|
||||
Hidden: true # hide from nav, but use this feature for rendering detail sites
|
||||
|
||||
- Name: blog1skip
|
||||
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}'
|
||||
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100'
|
||||
NavTemplate:
|
||||
EntriesAttribute: entries
|
||||
GoTo: '{{ date }}-{{ title }}'
|
||||
|
||||
@@ -303,3 +303,7 @@ code.language-mermaid svg {
|
||||
border: #444 solid 1px;
|
||||
border-radius: 5px;
|
||||
}
|
||||
|
||||
.blogEntry {
|
||||
margin-bottom: 50px;
|
||||
}
|
||||
@@ -31,11 +31,11 @@
|
||||
|
||||
<header id="header">
|
||||
<div class="container">
|
||||
<div class="logo" {% if not Data.slider and not Data.details.slider %}style="max-width: 400px;"{% endif %}>
|
||||
<div class="logo" {% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}style="max-width: 400px;"{% endif %}>
|
||||
<a href="{{ "/"|relative_path }}">
|
||||
<picture>
|
||||
<source media="(max-width: 768px)" srcset="project-files/img/logo_text.png">
|
||||
<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider %}_text{% endif %}.png" alt="mark2web Logo">
|
||||
<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}_text{% endif %}.png" alt="mark2web Logo">
|
||||
</picture>
|
||||
</a>
|
||||
</div>
|
||||
@@ -101,6 +101,7 @@
|
||||
|
||||
<!-- ========== Main Content ========== -->
|
||||
<div class="maincontent">
|
||||
{% block bg %}
|
||||
{% if Data.background %}
|
||||
<img
|
||||
src="{{ Data.background|image_process:"p=fill,w=1440,h=810,q=30,t=/img" }}"
|
||||
@@ -109,6 +110,7 @@
|
||||
{{ Data.background|image_process:"p=fill,w=1920,h=1020,q=30,t=/img" }} 1920w"
|
||||
alt="{{ Meta.Title }}" class="img2bg">
|
||||
{% endif %}
|
||||
{% endblock bg %}
|
||||
<div class="white_section section_padding">
|
||||
<div class="container">
|
||||
{% block part1 %}
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
limit wird für skip in Query gebraucht
|
||||
{% endcomment %}
|
||||
{% for e in NavElement.ColMap.blog1skip.entries %}
|
||||
<div class="blogEntry">
|
||||
<h2>
|
||||
{{ e.title }}
|
||||
<div class="datum">{{ e.date|datum }}</div>
|
||||
@@ -27,5 +28,6 @@
|
||||
{% if e.body %}
|
||||
<a href="{{ e.date|add:"-"|add:e.title|slugify }}" class="btn">mehr lesen »</a>
|
||||
{% endif %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
{% endblock part1 %}
|
||||
|
||||
@@ -8,6 +8,18 @@
|
||||
|
||||
{% if Data.details.slider %}
|
||||
{% block slider %}
|
||||
{% with Data.details.firstimg as img %}
|
||||
{% if img %}
|
||||
<div class="slide">
|
||||
<img
|
||||
src="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=600,q=60" }}"
|
||||
srcset="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=768,h=384,q=60" }} 768w,
|
||||
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=600,q=60" }} 1440w,
|
||||
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1920,h=800,q=60" }} 1920w"
|
||||
alt="{{ img.meta.title }}">
|
||||
</div>
|
||||
{% endif %}
|
||||
{% endwith %}
|
||||
{% for sl in Data.details.slider %}
|
||||
<div class="slide">
|
||||
<img
|
||||
@@ -29,6 +41,19 @@
|
||||
{{ Data.details.teaser|markdown:"s=monokai" }}
|
||||
{% endblock part0 %}
|
||||
|
||||
{% block bg %}
|
||||
{% with Data.details.firstimg as img %}
|
||||
{% if img %}
|
||||
<img
|
||||
src="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=810,q=30,t=/img" }}"
|
||||
srcset="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=768,h=768,q=30,t=/img" }} 768w,
|
||||
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=810,q=30,t=/img" }} 1440w,
|
||||
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1920,h=1020,q=30,t=/img" }} 1920w"
|
||||
alt="{{ Meta.Title }}" class="img2bg">
|
||||
{% endif %}
|
||||
{% endwith %}
|
||||
{% endblock bg %}
|
||||
|
||||
{% block part1 %}
|
||||
{{ Body }}
|
||||
<br>
|
||||
|
||||
Reference in New Issue
Block a user