10 Commits
dev ... jobm

Author SHA1 Message Date
Sebastian Frank
daed37587e improved logging
All checks were successful
continuous-integration/drone/push Build is passing
2019-03-29 15:49:25 +01:00
Sebastian Frank
7695f42e20 started new logger output
All checks were successful
continuous-integration/drone/push Build is passing
2019-03-27 13:52:22 +01:00
Sebastian Frank
5d6d03702e webrequest pkg, web request bar 2019-03-27 12:52:30 +01:00
Sebastian Frank
4cb09fb81f local collections for docu in website
All checks were successful
continuous-integration/drone/push Build is passing
2019-03-25 15:33:53 +01:00
Sebastian Frank
234137f22f cached webrequests 2019-03-25 15:07:02 +01:00
Sebastian Frank
267d1010bb -progress cli param for bars 2019-03-25 14:01:28 +01:00
Sebastian Frank
740fb94556 job manager, jobm pkg 2019-03-25 10:16:33 +01:00
Sebastian Frank
a17926f54b logger pkg 2019-03-25 09:28:58 +01:00
Sebastian Frank
b9c4553577 fixed website blog
All checks were successful
continuous-integration/drone/push Build is passing
2019-03-24 14:51:40 +01:00
Sebastian Frank
745c886cec bump to version 1.2.0
All checks were successful
continuous-integration/drone/push Build is passing
continuous-integration/drone Build is passing
2019-03-22 17:24:41 +01:00
50 changed files with 1159 additions and 885 deletions

12
.gitmodules vendored
View File

@@ -79,3 +79,15 @@
[submodule "vendor/github.com/itchio/go-brotli"]
path = vendor/github.com/itchio/go-brotli
url = https://github.com/itchio/go-brotli
[submodule "vendor/github.com/gosuri/uiprogress"]
path = vendor/github.com/gosuri/uiprogress
url = https://github.com/gosuri/uiprogress
[submodule "vendor/github.com/gosuri/uilive"]
path = vendor/github.com/gosuri/uilive
url = https://github.com/gosuri/uilive
[submodule "vendor/github.com/mattn/go-tty"]
path = vendor/github.com/mattn/go-tty
url = https://github.com/mattn/go-tty
[submodule "vendor/golang.org/x/sys"]
path = vendor/golang.org/x/sys
url = https://go.googlesource.com/sys

View File

@@ -6,7 +6,7 @@
"commands": [
{
"match": "website/.*",
"cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create",
"cmd": "time mark2web -in ${workspaceRoot}/website -out ${workspaceRoot}/html -create -logLevel warning -progress",
"silent": false
}
]

View File

@@ -1 +1 @@
1.2.0-pre
1.2.0

View File

@@ -7,8 +7,9 @@ import (
"path"
"gitbase.de/apairon/mark2web/pkg/filter"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"gitbase.de/apairon/mark2web/pkg/mark2web"
"gitbase.de/apairon/mark2web/pkg/progress"
)
var (
@@ -25,7 +26,8 @@ func main() {
outDir := flag.String("out", "html", "output directory")
createOutDir := flag.Bool("create", false, "create output directory if not existing")
//clearOutDir := flag.Bool("clear", false, "clear output directory before generating website")
logLevel := flag.String("logLevel", "info", "log level: debug, info, warning, error")
logLevel := flag.String("logLevel", "", "log level: debug, info, notice, warning, error")
progressBars := flag.Bool("progress", false, "show progress bars for jobs")
version := flag.Bool("version", false, "print version of this executable")
flag.Parse()
@@ -37,60 +39,66 @@ func main() {
os.Exit(0)
}
level := "info"
if logLevel != nil {
level := "notice"
if progressBars != nil && *progressBars {
progress.Start()
level = "warning" // disable log for progressbars
}
if logLevel != nil && *logLevel != "" {
level = *logLevel
}
helper.ConfigureLogger(level)
logger.SetLogLevel(level)
if inDir == nil || *inDir == "" {
helper.Log.Panic("input directory not specified")
logger.Exit("input directory not specified")
}
iDir := path.Clean(*inDir)
inDir = &iDir
helper.Log.Infof("input directory: %s", *inDir)
logger.I("input directory: %s", *inDir)
if outDir == nil || *outDir == "" {
helper.Log.Panic("output directory not specified")
logger.Exit("output directory not specified")
}
oDir := path.Clean(*outDir)
outDir = &oDir
helper.Log.Infof("output directory: %s", *outDir)
logger.I("output directory: %s", *outDir)
if createOutDir != nil && *createOutDir {
if _, err := os.Stat(*outDir); os.IsNotExist(err) {
helper.Log.Debugf("output directory '%s' does not exist", *outDir)
helper.Log.Debugf("trying to create output directory: %s", *outDir)
logger.D("output directory '%s' does not exist", *outDir)
logger.D("trying to create output directory: %s", *outDir)
err := os.MkdirAll(*outDir, 0755)
if err != nil {
helper.Log.Panic(err)
logger.Log.Panic(err)
}
helper.Log.Noticef("created output directory: %s", *outDir)
logger.I("created output directory: %s", *outDir)
} else {
helper.Log.Noticef("output directory '%s' already exists", *outDir)
logger.I("output directory '%s' already exists", *outDir)
}
}
if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
helper.Log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
logger.Eexit(err, "output directory '%s' does not exist, try -create parameter or create manually", *outDir)
} else {
if fD == nil {
helper.Log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
logger.P("something went wrong, could not get file handle for output dir %s", *outDir)
} else if !fD.IsDir() {
helper.Log.Panicf("output directory '%s' is not a directory", *outDir)
logger.Exit("output directory '%s' is not a directory", *outDir)
}
}
helper.Log.Debug("reading global config...")
logger.D("reading global config...")
configFilename := *inDir + "/config.yml"
err := mark2web.Config.ReadFromFile(configFilename)
if err != nil {
helper.Log.Panicf("could not read file '%s': %s", configFilename, err)
logger.Eexit(err, "could not read file '%s'", configFilename)
}
mark2web.Config.Directories.Input = *inDir
mark2web.Config.Directories.Output = *outDir
helper.Log.Debugf("reading input directory %s", *inDir)
logger.D("reading input directory %s", *inDir)
defaultTemplate := "base.html"
defaultInputFile := "README.md"
@@ -130,4 +138,6 @@ func main() {
mark2web.Run(*inDir, *outDir, defaultPathConfig)
logger.N("done")
}

View File

@@ -5,7 +5,7 @@ import (
"path"
"strings"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"gitbase.de/apairon/mark2web/pkg/mark2web"
"github.com/ddliu/motto"
"github.com/flosch/pongo2"
@@ -15,16 +15,14 @@ import (
// RegisterFilters reads a directory and register filters from files within it
func RegisterFilters(dir string) {
files, err := ioutil.ReadDir(dir)
if err != nil {
helper.Log.Panicf("could not read from template filters dir '%s': %s", dir, err)
}
logger.Eexit(err, "could not read from template filters dir '%s'", dir)
for _, f := range files {
if !f.IsDir() {
switch path.Ext(f.Name()) {
case ".js":
fileBase := strings.TrimSuffix(f.Name(), ".js")
jsFile := dir + "/" + f.Name()
helper.Log.Debugf("trying to register filter from: %s", jsFile)
logger.D("trying to register filter from: %s", jsFile)
/*
jsStr, err := ioutil.ReadFile(jsFile)
if err != nil {
@@ -33,38 +31,30 @@ func RegisterFilters(dir string) {
*/
vm := motto.New()
fn, err := vm.Run(jsFile)
if err != nil {
helper.Log.Panicf("error in javascript vm for '%s': %s", jsFile, err)
}
logger.Eexit(err, "error in javascript vm for '%s'", jsFile)
if !fn.IsFunction() {
helper.Log.Panicf("%s does not contain a function code", jsFile)
logger.Exit("%s does not contain a function code", jsFile)
}
err = pongo2.RegisterFilter(
fileBase,
func(in, param *pongo2.Value) (out *pongo2.Value, erro *pongo2.Error) {
thisObj, _ := vm.Object("({})")
var err error
if mark2web.CurrentContext != nil {
thisObj.Set("context", *mark2web.CurrentContext)
err = thisObj.Set("context", *mark2web.CurrentContext)
}
if err != nil {
helper.Log.Panicf("could not set context as in '%s': %s", jsFile, err)
}
logger.Perr(err, "could not set context in '%s': %s", jsFile)
ret, err := fn.Call(thisObj.Value(), in.Interface(), param.Interface())
if err != nil {
helper.Log.Panicf("error in javascript file '%s' while calling returned function: %s", jsFile, err)
}
logger.Eexit(err, "error in javascript file '%s' while calling returned function", jsFile)
retGo, err := ret.Export()
if err != nil {
helper.Log.Panicf("export error for '%s': %s", jsFile, err)
}
logger.Perr(err, "export error for '%s'", jsFile)
return pongo2.AsValue(retGo), nil
},
)
if err != nil {
helper.Log.Panicf("could not register filter from '%s': %s", jsFile, err)
}
logger.Perr(err, "could not register filter from '%s'", jsFile)
}
}

View File

@@ -5,7 +5,9 @@ import (
"errors"
"fmt"
"image"
"net/http"
"image/gif"
"image/jpeg"
"image/png"
"net/url"
"os"
"path"
@@ -13,7 +15,10 @@ import (
"strings"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/jobm"
"gitbase.de/apairon/mark2web/pkg/logger"
"gitbase.de/apairon/mark2web/pkg/mark2web"
"gitbase.de/apairon/mark2web/pkg/webrequest"
"github.com/disintegration/imaging"
"github.com/flosch/pongo2"
)
@@ -61,20 +66,6 @@ func parseImageParams(str string) (*mark2web.ImagingConfig, error) {
return &p, nil
}
func getImageFromURL(url string) (image.Image, string, error) {
resp, err := http.Get(url)
if err != nil {
return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
}
img, format, err := image.Decode(resp.Body)
if err != nil {
return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
}
return img, format, nil
}
// ImageProcessFilter read the image url and process parameters and saves the resized/processed image
// param: w=WITDH,h=HEIGHT
func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *pongo2.Error) {
@@ -105,27 +96,18 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
p.Quality,
)
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
// remote file
img, p.Format, err = getImageFromURL(imgSource)
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
}
}
// build filename
if p.Filename == "" {
var fBase string
if u, _ := url.Parse(imgSource); u != nil {
fBase = strings.Split(path.Base(u.Path), ".")[0]
fBase = path.Base(u.Path)
}
p.Filename = fmt.Sprintf(
"%s_%x_%s.%s",
"%s_%x_%s",
filePrefix,
md5.Sum([]byte(imgSource)),
fBase,
p.Format,
)
}
} else {
@@ -149,7 +131,7 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
pt := path.Dir(imgTarget)
if _, err := os.Stat(pt); os.IsNotExist(err) {
helper.Log.Infof("create image target dir: %s", pt)
logger.I("create image target dir: %s", pt)
if err := os.MkdirAll(pt, 0755); err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
@@ -165,18 +147,18 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
}
if f, err := os.Stat(imgTarget); err == nil && !f.IsDir() {
helper.Log.Noticef("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
logger.N("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
} else {
mark2web.ThreadStart(func() {
helper.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
jobm.Enqueue(jobm.Job{
Function: func() {
logger.N("processing image from %s to %s", imgSource, imgTarget)
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
// webrequest before finding target filename, because of file format in filename
// remote file
img, p.Format, err = webrequest.GetImage(imgSource)
} else {
img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
if err != nil {
helper.Log.Panicf("filter:image_resize, could not open image '%s': %s", imgSource, err)
}
}
logger.Eexit(err, "filter:image_resize, could not open image '%s'", imgSource)
switch p.Process {
case "resize":
@@ -207,23 +189,54 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
case "bottomright":
anchor = imaging.BottomRight
default:
helper.Log.Panicf("filter:image_resize, unknown anchor a=%s definition", p.Anchor)
logger.Exit("filter:image_resize, unknown anchor a=%s definition", p.Anchor)
}
img = imaging.Fill(img, p.Width, p.Height, anchor, imaging.Lanczos)
default:
helper.Log.Panicf("filter:image_resize, invalid p parameter '%s'", p.Process)
logger.Exit("filter:image_resize, invalid p parameter '%s'", p.Process)
}
if p.Format == "" {
switch strings.ToLower(path.Ext(imgTarget)) {
case ".jpg", ".jpeg", ".gif", ".png":
var encodeOptions = make([]imaging.EncodeOption, 0)
if p.Quality > 0 {
encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
}
err = imaging.Save(img, imgTarget, encodeOptions...)
if err != nil {
helper.Log.Panicf("filter:image_resize, could save image '%s': %s", imgTarget, err)
logger.Eerr(err, "filter:image_resize, could save image '%s'", imgTarget)
default:
logger.Exit("filter:image_resize, invalid filename extension for image: %s", imgTarget)
}
helper.Log.Noticef("finished image: %s", imgTarget)
} else {
out, err := os.Create(imgTarget)
defer out.Close()
logger.Eexit(err, "filter:image_resize, could not create image file '%s'", imgTarget)
switch p.Format {
case "jpeg", "jpg":
var jpegOpt *jpeg.Options
if p.Quality > 0 {
jpegOpt = &jpeg.Options{
Quality: p.Quality,
}
}
err = jpeg.Encode(out, img, jpegOpt)
case "png":
err = png.Encode(out, img)
case "gif":
err = gif.Encode(out, img, nil)
default:
logger.Exit("filter:image_resize, unknown format '%s' for '%s'", p.Format, imgSource)
}
logger.Eexit(err, "filter:image_resize, could not encode image file '%s'", imgTarget)
}
logger.N("finished image: %s", imgTarget)
},
Description: imgSource,
Category: "image process",
})
}
return pongo2.AsValue(mark2web.CurrentTreeNode.ResolveNavPath(p.Filename)), nil

View File

@@ -4,6 +4,7 @@ import (
"os"
"testing"
"gitbase.de/apairon/mark2web/pkg/jobm"
"gitbase.de/apairon/mark2web/pkg/mark2web"
"github.com/flosch/pongo2"
@@ -19,7 +20,7 @@ func TestImageProcessFilter(t *testing.T) {
}
// we want to check files after function calls, so no multithreading
mark2web.SetNumCPU(1)
jobm.SetNumCPU(1)
mark2web.Config.Directories.Input = "../../test/in"
mark2web.Config.Directories.Output = "../../test/out"

View File

@@ -2,25 +2,26 @@ package helper
import (
"os"
"gitbase.de/apairon/mark2web/pkg/logger"
)
// CreateDirectory creates direcory with all missing parents and panic if error
func CreateDirectory(dir string) {
Log.Debugf("trying to create output directory: %s", dir)
logger.D("trying to create output directory: %s", dir)
if dirH, err := os.Stat(dir); os.IsNotExist(err) {
err := os.MkdirAll(dir, 0755)
if err != nil {
Log.Panicf("could not create output directory '%s': %s", dir, err)
}
Log.Noticef("created output directory: %s", dir)
logger.Eexit(err, "could not create output directory '%s'", dir)
logger.I("created output directory: %s", dir)
} else if dirH != nil {
if dirH.IsDir() {
Log.Noticef("output directory '%s' already exists", dir)
logger.I("output directory '%s' already exists", dir)
} else {
Log.Panicf("output directory '%s' is no directory", dir)
logger.Exit("output directory '%s' is no directory", dir)
}
} else {
Log.Panicf("unknown error for output directory '%s': %s", dir, err)
logger.Perr(err, "unknown error for output directory '%s'", dir)
}
}

View File

@@ -1,52 +0,0 @@
package helper
import (
"os"
"github.com/davecgh/go-spew/spew"
"github.com/op/go-logging"
)
// Log is global logger
var Log = logging.MustGetLogger("myLogger")
// ConfigureLogger sets logger backend and level
func ConfigureLogger(level string) {
logBackend := logging.NewLogBackend(os.Stderr, "", 0)
logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
`%{color}%{time:15:04:05.000} %{shortfunc} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
))
logBackendLeveled := logging.AddModuleLevel(logBackendFormatter)
logBackendLevel := logging.INFO
switch level {
case "debug":
logBackendLevel = logging.DEBUG
break
case "info":
logBackendLevel = logging.INFO
break
case "notice":
logBackendLevel = logging.NOTICE
break
case "warning":
logBackendLevel = logging.WARNING
break
case "error":
logBackendLevel = logging.ERROR
break
}
logBackendLeveled.SetLevel(logBackendLevel, "")
logging.SetBackend(logBackendLeveled)
}
func init() {
spew.Config.DisablePointerAddresses = true
spew.Config.DisableCapacities = true
spew.Config.DisableMethods = true
spew.Config.DisablePointerMethods = true
}

16
pkg/helper/string.go Normal file
View File

@@ -0,0 +1,16 @@
package helper
// ShortenStringLeft shortens a string
func ShortenStringLeft(str string, num int) string {
if num <= 4 {
return ""
}
tstr := str
if len(str) > num {
if num > 3 {
num -= 3
}
tstr = "..." + str[len(str)-num:len(str)]
}
return tstr
}

View File

@@ -1,53 +0,0 @@
package helper
import (
"encoding/json"
"io/ioutil"
"net/http"
"strings"
)
// JSONWebRequest will GET a json object/array from a given URL
func JSONWebRequest(url string) interface{} {
Log.Noticef("requesting url via GET %s", url)
resp, err := http.Get(url)
if err != nil {
Log.Panicf("could not get url '%s': %s", url, err)
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
Log.Panicf("could not read body from url '%s': %s", url, err)
}
Log.Debugf("output from url '%s':\n%s", url, string(body))
if resp.StatusCode >= 400 {
Log.Panicf("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
}
contentType := resp.Header.Get("Content-Type")
if strings.Contains(contentType, "json") {
} else {
Log.Panicf("is not json '%s' from url '%s'", contentType, url)
}
jsonMap := make(map[string]interface{})
err = json.Unmarshal(body, &jsonMap)
if err == nil {
return jsonMap
}
jsonArrayMap := make([]map[string]interface{}, 0)
err = json.Unmarshal(body, &jsonArrayMap)
if err == nil {
return jsonArrayMap
}
Log.Panicf("could not read json from '%s': invalid type", url)
return nil
}

61
pkg/jobm/jobmanager.go Normal file
View File

@@ -0,0 +1,61 @@
package jobm
import (
"runtime"
"sync"
"gitbase.de/apairon/mark2web/pkg/progress"
)
var wg sync.WaitGroup
var numCPU = runtime.NumCPU()
// Job is a wrapper to descripe a Job function
type Job struct {
Function func()
Description string
Category string
}
var jobChan = make(chan []Job)
func worker(jobChan <-chan []Job) {
defer wg.Done()
for jobs := range jobChan {
for _, job := range jobs {
progress.DescribeCurrent(job.Category, job.Description)
job.Function()
progress.IncrDone(job.Category)
}
}
}
func init() {
//logger.I("number of CPU core: %d", numCPU)
// one core for main thread
for i := 0; i < numCPU; i++ {
wg.Add(1)
go worker(jobChan)
}
}
// Enqueue enqueues a job to the job queue
func Enqueue(jobs ...Job) {
for _, job := range jobs {
progress.IncrTotal(job.Category)
}
jobChan <- jobs
}
// Wait will wait for all jobs to finish
func Wait() {
close(jobChan)
progress.Stop()
wg.Wait()
}
// SetNumCPU is for testing package without threading
func SetNumCPU(i int) {
numCPU = i
}

138
pkg/logger/logger.go Normal file
View File

@@ -0,0 +1,138 @@
package logger
import (
"os"
"github.com/davecgh/go-spew/spew"
"github.com/op/go-logging"
)
// Log is global logger
var Log = logging.MustGetLogger("myLogger")
var logBackendLeveled logging.LeveledBackend
var Prefix = ""
// SetLogLevel sets log level for global logger (debug, info, notice, warning, error)
func SetLogLevel(level string) {
logBackendLevel := logging.INFO
switch level {
case "debug":
logBackendLevel = logging.DEBUG
break
case "info":
logBackendLevel = logging.INFO
break
case "notice":
logBackendLevel = logging.NOTICE
break
case "warning":
logBackendLevel = logging.WARNING
break
case "error":
logBackendLevel = logging.ERROR
break
}
logBackendLeveled.SetLevel(logBackendLevel, "")
}
// configureLogger sets logger backend and level
func configureLogger() {
logBackend := logging.NewLogBackend(os.Stderr, "", 0)
logBackendFormatter := logging.NewBackendFormatter(logBackend, logging.MustStringFormatter(
`%{color}%{time:2006-01-02 15:04:05.000} ▶ %{level:.4s} %{id:03x}%{color:reset} %{message}`,
))
logBackendLeveled = logging.AddModuleLevel(logBackendFormatter)
logBackendLeveled.SetLevel(logging.NOTICE, "")
logging.SetBackend(logBackendLeveled)
}
func init() {
spew.Config.DisablePointerAddresses = true
spew.Config.DisableCapacities = true
spew.Config.DisableMethods = true
spew.Config.DisablePointerMethods = true
configureLogger()
}
func prefix() string {
return Prefix
}
// D is shorthand for Debugf
func D(format string, args ...interface{}) {
Log.Debugf(prefix()+format, args...)
}
// I is shorthand for Infof
func I(format string, args ...interface{}) {
Log.Infof(prefix()+format, args...)
}
// N is shorthand for Noticef
func N(format string, args ...interface{}) {
Log.Noticef(prefix()+format, args...)
}
// W is shorthand for Warningf
func W(format string, args ...interface{}) {
Log.Warningf(prefix()+format, args...)
}
// E is shorthand for Errorf
func E(format string, args ...interface{}) {
Log.Errorf(prefix()+format, args...)
}
// P is shorthand for Panicf
func P(format string, args ...interface{}) {
Log.Panicf(prefix()+format, args...)
}
// Eerr is shorthand for
// if err != nil {
// Log.Errorf(...)
// }
func Eerr(err error, format string, args ...interface{}) {
if err != nil {
args = append(args, err)
Log.Errorf(prefix()+format+" (Error: %s)", args...)
}
}
// Eexit is shorthand for
// if err != nil {
// Log.Errorf(...)
// os.Exit(1)
// }
func Eexit(err error, format string, args ...interface{}) {
Eerr(err, format, args...)
if err != nil {
os.Exit(1)
}
}
// Exit is shorthand for
// Log.Errorf(...)
// os.Exit(1)
func Exit(format string, args ...interface{}) {
E(format, args...)
os.Exit(1)
}
// Perr is shorthand for
// if err != nil {
// Log.Panicf(...)
// }
func Perr(err error, format string, args ...interface{}) {
if err != nil {
args = append(args, err)
Log.Panicf(prefix()+format+" (Error: %s)", args...)
}
}

View File

@@ -1,12 +1,11 @@
package mark2web
import (
"log"
"path"
"regexp"
"strings"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
cpy "github.com/otiai10/copy"
)
@@ -22,11 +21,10 @@ func ProcessAssets() {
if !strings.HasPrefix(to, "/") {
to = Config.Directories.Output + "/" + to
}
helper.Log.Noticef("copying assets from '%s' to '%s'", from, to)
logger.N("copying assets from '%s' to '%s'", from, to)
err := cpy.Copy(from, to)
if err != nil {
helper.Log.Panicf("could not copy assets from '%s' to '%s': %s", from, to, err)
}
logger.Perr(err, "could not copy assets from '%s' to '%s'", from, to)
if Config.Assets.Compress {
compressFilesInDir(to)
@@ -37,18 +35,17 @@ func ProcessAssets() {
// fixAssetsPath replaces assets path based on current path
func (node *TreeNode) fixAssetsPath(str string) string {
if find := Config.Assets.FixTemplate.Find; find != "" {
helper.Log.Debugf("fixing assets paths for path '%s'", node.CurrentNavPath())
logger.D("fixing assets paths for path '%s'", node.CurrentNavPath())
repl := Config.Assets.FixTemplate.Replace
toPath := Config.Assets.ToPath
bToRoot := node.BackToRootPath()
regex, err := regexp.Compile(find)
if err != nil {
log.Panicf("could not compile regexp '%s' for assets path: %s", find, err)
}
logger.Eexit(err, "could not compile regexp '%s' for assets path", find)
repl = bToRoot + toPath + "/" + repl
repl = path.Clean(repl) + "/"
helper.Log.Debugf("new assets paths: %s", repl)
logger.D("new assets paths: %s", repl)
return regex.ReplaceAllString(str, repl)
}

View File

@@ -6,7 +6,7 @@ import (
"io"
"os"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"github.com/itchio/go-brotli/enc"
)
@@ -15,12 +15,11 @@ var brotliSupported = true
func handleBrotliCompression(filename string, content []byte) {
brFilename := filename + ".br"
helper.Log.Infof("writing to compressed output file: %s", brFilename)
logger.I("writing to compressed output file: %s", brFilename)
f, err := os.Create(brFilename)
if err != nil {
helper.Log.Panicf("could not create file '%s': %s", brFilename, err)
}
logger.Eexit(err, "could not create file '%s'", brFilename)
defer f.Close()
bw := enc.NewBrotliWriter(f, nil)
@@ -29,20 +28,14 @@ func handleBrotliCompression(filename string, content []byte) {
if content != nil {
// content given
_, err = bw.Write(content)
if err != nil {
helper.Log.Panicf("could not write brotli content for '%s': %s", filename, err)
}
logger.Eexit(err, "could not write brotli content for '%s'", filename)
} else {
// read file
r, err := os.Open(filename)
if err != nil {
helper.Log.Panicf("could not open file '%s': %s", filename, err)
}
logger.Eexit(err, "could not open file '%s'", filename)
defer r.Close()
_, err = io.Copy(bw, r)
if err != nil {
helper.Log.Panicf("could not write brotli file for '%s': %s", filename, err)
}
logger.Eexit(err, "could not write brotli file for '%s'", filename)
}
}

View File

@@ -7,6 +7,8 @@ import (
"strings"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"gitbase.de/apairon/mark2web/pkg/webrequest"
"github.com/davecgh/go-spew/spew"
"github.com/flosch/pongo2"
)
@@ -23,11 +25,11 @@ func (node *TreeNode) handleCollections() {
collections := append(node.Config.Collections, node.Config.This.Collections...)
for _, colConfig := range collections {
if colConfig.Name == nil || *colConfig.Name == "" {
helper.Log.Panicf("missing Name in collection config in '%s'", node.InputPath)
logger.Exit("missing Name in collection config in '%s'", node.InputPath)
}
if (colConfig.URL == nil || *colConfig.URL == "") &&
(colConfig.Directory == nil) {
helper.Log.Panicf("missing URL and Directory in collection config in '%s'", node.InputPath)
logger.Exit("missing URL and Directory in collection config in '%s'", node.InputPath)
}
if node.ColMap == nil {
@@ -44,9 +46,7 @@ func (node *TreeNode) handleCollections() {
if colConfig.URL != nil {
url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
if err != nil {
helper.Log.Panicf("invalid template string for Collection Element.URL in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "invalid template string for Collection Element.URL in '%s'", node.InputPath)
errSrcText = "URL " + url
cacheKey = url
@@ -55,8 +55,8 @@ func (node *TreeNode) handleCollections() {
colData = cacheEntry.data
cacheEntry.hit++
} else {
helper.Log.Noticef("reading collection from: %s", errSrcText)
colData = helper.JSONWebRequest(url)
logger.N("reading collection from: %s", errSrcText)
colData = webrequest.GetJSON(url)
colCache[url] = &colCacheEntry{
data: colData,
navnames: make([]string, 0),
@@ -67,20 +67,16 @@ func (node *TreeNode) handleCollections() {
path := node.ResolveInputPath(colConfig.Directory.Path)
errSrcText = "DIR " + path
helper.Log.Noticef("reading collection from: %s", errSrcText)
logger.N("reading collection from: %s", errSrcText)
d, err := ioutil.ReadDir(path)
if err != nil {
helper.Log.Panicf("could not read directory '%s': %s", path, err)
}
logger.Eexit(err, "could not read directory '%s'", path)
mStr := "."
if colConfig.Directory.MatchFilename != "" {
mStr = colConfig.Directory.MatchFilename
}
matcher, err := regexp.Compile(mStr)
if err != nil {
helper.Log.Panicf("could not compile regex for MatchFilename '%s' in '%s': %s", mStr, path, err)
}
logger.Eexit(err, "could not compile regex for MatchFilename '%s' in '%s'", mStr, path)
if colConfig.Directory.ReverseOrder {
for i := len(d)/2 - 1; i >= 0; i-- {
@@ -94,9 +90,8 @@ func (node *TreeNode) handleCollections() {
if !fh.IsDir() && matcher.MatchString(fh.Name()) {
inFile := path + "/" + fh.Name()
md, err := ioutil.ReadFile(inFile)
if err != nil {
helper.Log.Panicf("could not read file '%s': %s", inFile, err)
}
logger.Eexit(err, "could not read file '%s'", inFile)
_, ctx := node.processMarkdownWithHeader(md, inFile)
(*ctx)["FilenameMatch"] = helper.GetRegexpParams(matcher, fh.Name())
fcolData = append(fcolData, *ctx)
@@ -116,41 +111,36 @@ func (node *TreeNode) handleCollections() {
if colDataMap, ok = colData.(map[string]interface{}); ok {
entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
if !ok {
helper.Log.Debug(spew.Sdump(colDataMap))
helper.Log.Panicf("invalid json data in [%s] from '%s' for entries", navT.EntriesAttribute, errSrcText)
logger.D(spew.Sdump(colDataMap))
logger.Exit("invalid json data in [%s] from '%s' for entries", navT.EntriesAttribute, errSrcText)
}
}
} else {
entries, ok = colData.([]interface{})
}
if !ok {
helper.Log.Debug(spew.Sdump(colData))
helper.Log.Panicf("invalid json data from '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", errSrcText)
logger.D(spew.Sdump(colData))
logger.Exit("invalid json data from '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", errSrcText)
}
// build navigation with detail sites
for idx, colEl := range entries {
ctxE := make(pongo2.Context)
err := helper.Merge(&ctxE, ctx)
if err != nil {
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "could not merge context in '%s'", node.InputPath)
var jsonCtx map[string]interface{}
if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
helper.Log.Debug(spew.Sdump(colEl))
helper.Log.Panicf("no json object for entry index %d from '%s'", idx, errSrcText)
logger.D(spew.Sdump(colEl))
logger.Exit("no json object for entry index %d from '%s'", idx, errSrcText)
}
err = helper.Merge(&ctxE, pongo2.Context(jsonCtx))
if err != nil {
helper.Log.Panicf("could not merge context in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "could not merge context in '%s'", node.InputPath)
tpl := ""
if navT.Template != "" {
tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Template in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "invalid template string for NavTemplate.Template in '%s'", node.InputPath)
}
if tpl == "" {
tpl = *node.Config.Template
@@ -159,48 +149,41 @@ func (node *TreeNode) handleCollections() {
dataKey := ""
if navT.DataKey != "" {
dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.DataKey in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "invalid template string for NavTemplate.DataKey in '%s'", node.InputPath)
}
goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.GoTo in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "invalid template string for NavTemplate.GoTo in '%s'", node.InputPath)
goTo = strings.Trim(goTo, "/")
goTo = path.Clean(goTo)
if strings.Contains(goTo, "..") {
helper.Log.Panicf("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
logger.Exit("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
}
if goTo == "." {
helper.Log.Panicf("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
logger.Exit("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
}
if goTo == "" {
helper.Log.Panicf("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
logger.Exit("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
}
navname := ""
if navT.Navname != "" {
navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Navname in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "invalid template string for NavTemplate.Navname in '%s'", node.InputPath)
}
body := ""
if navT.Body != "" {
body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
if err != nil {
helper.Log.Panicf("invalid template string for NavTemplate.Body in '%s': %s", node.InputPath, err)
}
logger.Eexit(err, "invalid template string for NavTemplate.Body in '%s'", node.InputPath)
}
if l := len(colCache[cacheKey].navnames); colCache[cacheKey].hit > 1 &&
l > 0 &&
navname == colCache[cacheKey].navnames[l-1] {
// navname before used same url, so recursion loop
helper.Log.Panicf("collection request loop detected for in '%s' for : %s", node.InputPath, errSrcText)
logger.Exit("collection request loop detected for in '%s' for : %s", node.InputPath, errSrcText)
}
colCache[cacheKey].navnames = append(colCache[cacheKey].navnames, navname)

View File

@@ -7,11 +7,14 @@ import (
"os"
"path"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/jobm"
"gitbase.de/apairon/mark2web/pkg/logger"
)
func handleCompression(filename string, content []byte) {
ThreadStart(func() {
jobm.Enqueue(jobm.Job{
Function: func() {
if _, ok := Config.Compress.Extensions[path.Ext(filename)]; ok {
if Config.Compress.Brotli {
@@ -21,54 +24,45 @@ func handleCompression(filename string, content []byte) {
if Config.Compress.GZIP {
gzFilename := filename + ".gz"
helper.Log.Infof("writing to compressed output file: %s", gzFilename)
logger.I("writing to compressed output file: %s", gzFilename)
f, err := os.Create(gzFilename)
if err != nil {
helper.Log.Panicf("could not create file '%s': %s", gzFilename, err)
}
logger.Eexit(err, "could not create file '%s'", gzFilename)
defer f.Close()
zw, err := gzip.NewWriterLevel(f, gzip.BestCompression)
if err != nil {
helper.Log.Panicf("could not initialize gzip writer for '%s': %s", filename, err)
}
logger.Eexit(err, "could not initialize gzip writer for '%s'", filename)
defer zw.Close()
if content != nil {
// content given
_, err = zw.Write(content)
if err != nil {
helper.Log.Panicf("could not write gziped content for '%s': %s", filename, err)
}
logger.Eexit(err, "could not write gziped content for '%s'", filename)
} else {
// read file
r, err := os.Open(filename)
if err != nil {
helper.Log.Panicf("could not open file '%s': %s", filename, err)
}
logger.Eexit(err, "could not open file '%s'", filename)
defer r.Close()
_, err = io.Copy(zw, r)
if err != nil {
helper.Log.Panicf("could not gzip file '%s': %s", filename, err)
}
logger.Eexit(err, "could not gzip file '%s'", filename)
}
}
}
},
Description: filename,
Category: "compress",
})
}
func compressFilesInDir(dir string) {
helper.Log.Noticef("compressing configured files in: %s", dir)
logger.N("compressing configured files in: %s", dir)
var _processDir func(string)
_processDir = func(d string) {
entries, err := ioutil.ReadDir(d)
if err != nil {
helper.Log.Panicf("could not read dir '%s': %s", d, err)
}
logger.Eexit(err, "could not read dir '%s'", d)
for _, entry := range entries {
if entry.IsDir() {

View File

@@ -7,7 +7,10 @@ import (
"regexp"
"strings"
"gitbase.de/apairon/mark2web/pkg/progress"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"github.com/davecgh/go-spew/spew"
"github.com/flosch/pongo2"
cpy "github.com/otiai10/copy"
@@ -16,6 +19,9 @@ import (
// ReadContentDir walks through content directory and builds the tree of configurations
func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string, conf *PathConfig) {
progress.IncrTotal("content dir")
progress.DescribeCurrent("content dir", "found "+inBase)
if node.root == nil {
// first node is root
node.root = node
@@ -23,9 +29,7 @@ func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string,
node.fillConfig(inBase, outBase, dir, conf)
files, err := ioutil.ReadDir(node.InputPath)
if err != nil {
helper.Log.Panic(err)
}
logger.Eexit(err, "could not read dir '%s'", node.InputPath)
// first only files
for _, f := range files {
@@ -33,14 +37,14 @@ func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string,
if !f.IsDir() && f.Name() != "config.yml" {
switch path.Ext(f.Name()) {
case ".md":
helper.Log.Debugf(".MD %s", p)
logger.D(".MD %s", p)
if node.InputFiles == nil {
node.InputFiles = make([]string, 0)
}
node.InputFiles = append(node.InputFiles, f.Name())
break
default:
helper.Log.Debugf("FIL %s", p)
logger.D("FIL %s", p)
if node.OtherFiles == nil {
node.OtherFiles = make([]string, 0)
}
@@ -53,7 +57,7 @@ func (node *TreeNode) ReadContentDir(inBase string, outBase string, dir string,
for _, f := range files {
p := node.InputPath + "/" + f.Name()
if f.IsDir() {
helper.Log.Debugf("DIR %s", p)
logger.D("DIR %s", p)
newTree := new(TreeNode)
newTree.root = node.root
if node.Sub == nil {
@@ -75,18 +79,16 @@ func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*Pa
// replace tabs
yamlData = bytes.Replace(yamlData, []byte("\t"), []byte(" "), -1)
helper.Log.Debugf("found yaml header in '%s', merging config", errorRef)
logger.D("found yaml header in '%s', merging config", errorRef)
err := yaml.Unmarshal(yamlData, newConfig)
if err != nil {
helper.Log.Panicf("could not parse YAML header from '%s': %s", errorRef, err)
}
logger.Eexit(err, "could not parse YAML header from '%s'", errorRef)
helper.Log.Debug("merging config with upper config")
logger.D("merging config with upper config")
oldThis := newConfig.This
helper.Merge(newConfig, node.Config)
newConfig.This = oldThis
helper.Log.Debug(spew.Sdump(newConfig))
logger.D(spew.Sdump(newConfig))
md = headerRegex.ReplaceAll(md, []byte(""))
} else {
@@ -154,6 +156,8 @@ func (node *TreeNode) processMarkdownWithHeader(md []byte, errorRef string) (*Pa
// ProcessContent walks recursivly through the input paths and processes all files for output
func (node *TreeNode) ProcessContent() {
progress.DescribeCurrent("content dir", "processing "+node.InputPath)
helper.CreateDirectory(node.OutputPath)
if node.root != node {
@@ -183,27 +187,24 @@ func (node *TreeNode) ProcessContent() {
}
if ignoreRegex != nil && *ignoreRegex != "" {
regex, err := regexp.Compile(*ignoreRegex)
if err != nil {
helper.Log.Panicf("could not compile filename.ignore regexp '%s' for file '%s': %s", *ignoreRegex, inFile, err)
}
logger.Eexit(err, "could not compile filename.ignore regexp '%s' for file '%s'", *ignoreRegex, inFile)
ignoreFile = regex.MatchString(file)
}
}
if ignoreFile {
helper.Log.Infof("ignoring file '%s', because of filename.ignore", inFile)
logger.I("ignoring file '%s', because of filename.ignore", inFile)
} else {
var input []byte
if file != "" {
helper.Log.Debugf("reading file: %s", inFile)
logger.D("reading file: %s", inFile)
var err error
input, err = ioutil.ReadFile(inFile)
if err != nil {
helper.Log.Panicf("could not read '%s':%s", inFile, err)
}
helper.Log.Infof("processing input file '%s'", inFile)
logger.Eexit(err, "could not read '%s'", inFile)
logger.I("processing input file '%s'", inFile)
} else {
// use input string if available and input filename == ""
var inputString *string
@@ -211,7 +212,7 @@ func (node *TreeNode) ProcessContent() {
inputString = i.InputString
}
if inputString != nil {
helper.Log.Debugf("using input string instead of file")
logger.D("using input string instead of file")
input = []byte(*inputString)
}
}
@@ -243,9 +244,8 @@ func (node *TreeNode) ProcessContent() {
} else {
if stripRegex != nil && *stripRegex != "" {
regex, err := regexp.Compile(*stripRegex)
if err != nil {
helper.Log.Panicf("could not compile filename.strip regexp '%s' for file '%s': %s", *stripRegex, inFile, err)
}
logger.Eexit(err, "could not compile filename.strip regexp '%s' for file '%s'", *stripRegex, inFile)
outputFilename = regex.ReplaceAllString(outputFilename, "$1")
}
if outputExt != nil && *outputExt != "" {
@@ -254,21 +254,17 @@ func (node *TreeNode) ProcessContent() {
}
outFile := node.OutputPath + "/" + outputFilename
helper.Log.Debugf("using '%s' as output file", outFile)
helper.Log.Debugf("rendering template '%s' for '%s'", *newConfig.Template, outFile)
logger.D("using '%s' as output file", outFile)
logger.D("rendering template '%s' for '%s'", *newConfig.Template, outFile)
templateFilename := *newConfig.Template
result, err := renderTemplate(*newConfig.Template, node, newConfig, ctx)
if err != nil {
helper.Log.Panicf("could not execute template '%s' for input file '%s': %s", templateFilename, inFile, err)
}
logger.Eexit(err, "could not execute template '%s' for input file '%s': %s", templateFilename, inFile)
result = node.fixAssetsPath(result)
helper.Log.Noticef("writing to output file: %s", outFile)
logger.N("writing to output file: %s", outFile)
err = ioutil.WriteFile(outFile, []byte(result), 0644)
if err != nil {
helper.Log.Panicf("could not write to output file '%s': %s", outFile, err)
}
logger.Eexit(err, "could not write to output file '%s'", outFile)
handleCompression(outFile, []byte(result))
@@ -282,16 +278,16 @@ func (node *TreeNode) ProcessContent() {
case "copy":
from := node.InputPath + "/" + file
to := node.OutputPath + "/" + file
helper.Log.Noticef("copying file from '%s' to '%s'", from, to)
logger.N("copying file from '%s' to '%s'", from, to)
err := cpy.Copy(from, to)
if err != nil {
helper.Log.Panicf("could not copy file from '%s' to '%s': %s", from, to, err)
}
logger.Eexit(err, "could not copy file from '%s' to '%s': %s", from, to)
handleCompression(to, nil)
}
}
progress.IncrDone("content dir")
i := 0
// sub can dynamically increase, so no for range
for i < len(node.Sub) {

View File

@@ -9,6 +9,8 @@ import (
"time"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"gitbase.de/apairon/mark2web/pkg/progress"
"github.com/davecgh/go-spew/spew"
"github.com/extemporalgenome/slug"
"github.com/flosch/pongo2"
@@ -41,36 +43,33 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
inPath += "/" + subDir
}
helper.Log.Infof("reading input directory: %s", inPath)
logger.I("reading input directory: %s", inPath)
node.InputPath = inPath
// read config
newConfig := new(PathConfig)
helper.Log.Debug("looking for config.yml ...")
logger.D("looking for config.yml ...")
configFile := inPath + "/config.yml"
if _, err := os.Stat(configFile); os.IsNotExist(err) {
helper.Log.Debug("no config.yml found in this directory, using upper configs")
logger.D("no config.yml found in this directory, using upper configs")
helper.Merge(newConfig, conf)
// remove this
newConfig.This = ThisPathConfig{}
} else {
helper.Log.Debug("reading config...")
logger.D("reading config...")
data, err := ioutil.ReadFile(configFile)
if err != nil {
helper.Log.Panicf("could not read file '%s': %s", configFile, err)
}
err = yaml.Unmarshal(data, newConfig)
if err != nil {
helper.Log.Panicf("could not parse YAML file '%s': %s", configFile, err)
}
logger.Eexit(err, "could not read file '%s'", configFile)
helper.Log.Debug("merging config with upper config")
err = yaml.Unmarshal(data, newConfig)
logger.Eexit(err, "could not parse YAML file '%s'", configFile)
logger.D("merging config with upper config")
oldThis := newConfig.This
helper.Merge(newConfig, conf)
newConfig.This = oldThis
helper.Log.Debug(spew.Sdump(newConfig))
logger.D(spew.Sdump(newConfig))
}
node.Config = newConfig
@@ -83,7 +82,7 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
}
if regexStr != nil && *regexStr != "" {
if regex, err := regexp.Compile(*regexStr); err != nil {
helper.Log.Panicf("error compiling path.strip regex '%s' from '%s': %s", *regexStr, inBase+"/"+subDir, err)
logger.Eexit(err, "error compiling path.strip regex '%s' from '%s'", *regexStr, inBase+"/"+subDir)
} else {
stripedDir = regex.ReplaceAllString(stripedDir, "$1")
}
@@ -98,7 +97,7 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
outPath := outBase + "/" + stripedDir
outPath = path.Clean(outPath)
helper.Log.Infof("calculated output directory: %s", outPath)
logger.I("calculated output directory: %s", outPath)
node.OutputPath = outPath
// handle collections
@@ -106,6 +105,9 @@ func (node *TreeNode) fillConfig(inBase, outBase, subDir string, conf *PathConfi
}
func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx interface{}, dataMapKey string, body string, hideInNav bool) {
progress.IncrTotal("content dir")
progress.DescribeCurrent("content dir", "subdir "+node.InputPath+"/"+subDir)
newNode := new(TreeNode)
newNode.root = node.root
@@ -128,9 +130,8 @@ func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx
mergedConfig := new(PathConfig)
err := helper.Merge(mergedConfig, node.Config)
if err != nil {
helper.Log.Panicf("merge of path config failed: %s", err)
}
logger.Eexit(err, "merge of path config failed")
// dont merge Data[DataKey]
if dataMapKey != "" {
mergedConfig.Data[dataMapKey] = nil
@@ -138,9 +139,7 @@ func (node *TreeNode) addSubNode(tplFilename, subDir string, navname string, ctx
mergedConfig.Data = make(helper.MapString)
}
err = helper.Merge(mergedConfig, newPathConfig)
if err != nil {
helper.Log.Panicf("merge of path config failed: %s", err)
}
logger.Eexit(err, "merge of path config failed")
newNode.fillConfig(
node.InputPath,

View File

@@ -1,14 +1,14 @@
package mark2web
import (
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/webrequest"
"github.com/flosch/pongo2"
)
// RequestFn will make a web request and returns map[string]interface form pongo2
func RequestFn(url *pongo2.Value, args ...*pongo2.Value) *pongo2.Value {
u := url.String()
return pongo2.AsValue(helper.JSONWebRequest(u))
return pongo2.AsValue(webrequest.GetJSON(u))
}
// RenderFn renders a pongo2 template with additional context

View File

@@ -5,20 +5,18 @@ import (
"regexp"
"strings"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
)
func htaccessRedirect(outDir, goTo string) {
switch Config.Webserver.Type {
case "apache":
htaccessFile := outDir + "/.htaccess"
helper.Log.Noticef("writing '%s' with redirect to: %s", htaccessFile, goTo)
logger.N("writing '%s' with redirect to: %s", htaccessFile, goTo)
err := ioutil.WriteFile(htaccessFile, []byte(`RewriteEngine on
RewriteRule ^$ %{REQUEST_URI}`+goTo+`/ [R,L]
`), 0644)
if err != nil {
helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
}
logger.Eexit(err, "could not write '%s'", htaccessFile)
}
}
@@ -90,11 +88,9 @@ RemoveLanguage .br
if configStr != "" {
htaccessFile := Config.Directories.Output + "/.htaccess"
helper.Log.Noticef("writing webserver config to: %s", htaccessFile)
logger.N("writing webserver config to: %s", htaccessFile)
err := ioutil.WriteFile(htaccessFile, []byte(configStr), 0644)
if err != nil {
helper.Log.Panicf("could not write '%s': %s", htaccessFile, err)
}
logger.Eexit(err, "could not write '%s'", htaccessFile)
}
}
}

View File

@@ -6,6 +6,7 @@ import (
"strings"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
)
// NavElement is one element with ist attributes and subs
@@ -41,11 +42,10 @@ func buildNavigationRecursive(tree *TreeNode, curNavMap *map[string]*NavElement,
}
if ignNav != nil && *ignNav != "" {
regex, err := regexp.Compile(*ignNav)
if err != nil {
helper.Log.Panicf("could not compile IngoreForNav regexp '%s' in '%s': %s", *ignNav, el.InputPath, err)
}
logger.Eexit(err, "could not compile IngoreForNav regexp '%s' in '%s'", *ignNav, el.InputPath)
if regex.MatchString(path.Base(el.InputPath)) {
helper.Log.Debugf("ignoring input directory '%s' in navigation", el.InputPath)
logger.D("ignoring input directory '%s' in navigation", el.InputPath)
continue
}
}

View File

@@ -1,5 +1,7 @@
package mark2web
import "gitbase.de/apairon/mark2web/pkg/jobm"
// Run will do a complete run of mark2web
func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
SetTemplateDir(inDir + "/templates")
@@ -12,5 +14,5 @@ func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
tree.WriteWebserverConfig()
Wait()
jobm.Wait()
}

View File

@@ -1,60 +0,0 @@
package mark2web
import (
"runtime"
"sync"
"gitbase.de/apairon/mark2web/pkg/helper"
)
var wg sync.WaitGroup
var numCPU = runtime.NumCPU()
var curNumThreads = 1 // main thread is 1
func init() {
helper.Log.Infof("number of CPU core: %d", numCPU)
}
// Wait will wait for all our internal go threads
func Wait() {
wg.Wait()
}
// ThreadSetup adds 1 to wait group
func ThreadSetup() {
curNumThreads++
wg.Add(1)
}
// ThreadDone removes 1 from wait group
func ThreadDone() {
curNumThreads--
wg.Done()
}
// ThreadStart will start a thread an manages the wait group
func ThreadStart(f func(), forceNewThread ...bool) {
force := false
if len(forceNewThread) > 0 && forceNewThread[0] {
force = true
}
if numCPU > curNumThreads || force {
// only new thread if empty CPU core available or forced
threadF := func() {
f()
ThreadDone()
}
ThreadSetup()
go threadF()
} else {
helper.Log.Debugf("no more CPU core (%d used), staying in main thread", curNumThreads)
f()
}
}
// SetNumCPU is for testing package without threading
func SetNumCPU(i int) {
numCPU = i
}

96
pkg/progress/bar.go Normal file
View File

@@ -0,0 +1,96 @@
package progress
import (
"fmt"
"time"
"gitbase.de/apairon/mark2web/pkg/helper"
"gitbase.de/apairon/mark2web/pkg/logger"
"github.com/gosuri/uiprogress"
"github.com/mattn/go-tty"
)
type bar struct {
Bar *uiprogress.Bar
Description string
}
var bars = make(map[string]*bar)
var initialized = false
var terminalWidth = 80
// OverallTotal is number of total jobs
var OverallTotal = 0
// OverallDone is number of done jobs
var OverallDone = 0
func init() {
updateLoggerPrefix()
}
// Start initializes the bar drawing
func Start() {
if t, err := tty.Open(); err == nil && t != nil {
terminalWidth, _, _ = t.Size()
t.Close()
}
uiprogress.Start() // start rendering
initialized = true
}
func updateLoggerPrefix() {
logger.Prefix = fmt.Sprintf("%3d/%3d: ", OverallDone, OverallTotal)
}
// IncrTotal increases the total jobs for the bar
func IncrTotal(barname string) {
OverallTotal++
updateLoggerPrefix()
if initialized {
_bar := bars[barname]
if _bar == nil {
_bar = new(bar)
_bar.Bar = uiprogress.AddBar(1)
_bar.Bar.Width = 25
_bar.Bar.PrependFunc(func(b *uiprogress.Bar) string {
return fmt.Sprintf("%15s: %3d/%3d", helper.ShortenStringLeft(barname, 15), b.Current(), b.Total)
})
_bar.Bar.AppendFunc(func(b *uiprogress.Bar) string {
return fmt.Sprintf("%s", helper.ShortenStringLeft(_bar.Description, terminalWidth-80))
})
bars[barname] = _bar
} else {
_bar.Bar.Total++
}
}
}
// IncrDone increases to done jobs counter
func IncrDone(barname string) {
OverallDone++
updateLoggerPrefix()
if initialized {
bars[barname].Bar.Incr()
bars[barname].Description = ""
}
}
// DescribeCurrent describes the current job
func DescribeCurrent(barname, description string) {
if initialized {
bars[barname].Description = description
}
}
// Stop stops the bar drawing
func Stop() {
if initialized {
time.Sleep(time.Millisecond * 200)
uiprogress.Stop()
}
}

109
pkg/webrequest/request.go Normal file
View File

@@ -0,0 +1,109 @@
package webrequest
import (
"encoding/json"
"fmt"
"image"
"io/ioutil"
"net/http"
"strings"
"gitbase.de/apairon/mark2web/pkg/progress"
"gitbase.de/apairon/mark2web/pkg/logger"
)
type wrImageEntry struct {
img image.Image
format string
}
type wrJSONEntry struct {
data interface{}
}
var wrImageCache = make(map[string]*wrImageEntry)
var wrJSONCache = make(map[string]*wrJSONEntry)
// Get will fetch an url and returns reponse
func Get(url string, opts interface{}) (resp *http.Response, err error) {
logger.N("requesting url via GET %s", url)
progress.IncrTotal("web request")
progress.DescribeCurrent("web request", url)
resp, err = http.Get(url)
progress.IncrDone("web request")
return resp, err
}
// GetJSON will GET a json object/array from a given URL
func GetJSON(url string) interface{} {
cached := wrJSONCache[url]
if cached == nil {
resp, err := Get(url, nil)
logger.Eexit(err, "could not get url '%s'", url)
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
logger.Eexit(err, "could not read body from url '%s'", url)
logger.D("output from url '%s':\n%s", url, string(body))
if resp.StatusCode >= 400 {
logger.Exit("bad status '%d - %s' from url '%s'", resp.StatusCode, resp.Status, url)
}
contentType := resp.Header.Get("Content-Type")
if strings.Contains(contentType, "json") {
} else {
logger.Exit("is not json '%s' from url '%s'", contentType, url)
}
cached = new(wrJSONEntry)
jsonMap := make(map[string]interface{})
err = json.Unmarshal(body, &jsonMap)
if err == nil {
cached.data = jsonMap
} else {
jsonArrayMap := make([]map[string]interface{}, 0)
err = json.Unmarshal(body, &jsonArrayMap)
if err == nil {
cached.data = jsonArrayMap
} else {
logger.Exit("could not read json from '%s': invalid type", url)
}
}
wrJSONCache[url] = cached
}
return cached.data
}
// GetImage gets an image from an url
func GetImage(url string) (image.Image, string, error) {
cached := wrImageCache[url]
if cached == nil {
resp, err := Get(url, nil)
if err != nil {
return nil, "", fmt.Errorf("could not get url '%s': %s", url, err)
}
img, format, err := image.Decode(resp.Body)
if err != nil {
return nil, "", fmt.Errorf("could read body from url '%s': %s", url, err)
}
cached = &wrImageEntry{
img: img,
format: format,
}
wrImageCache[url] = cached
}
return cached.img, cached.format, nil
}

1
vendor/github.com/gosuri/uilive generated vendored Submodule

1
vendor/github.com/gosuri/uiprogress generated vendored Submodule

1
vendor/github.com/mattn/go-tty generated vendored Submodule

1
vendor/golang.org/x/sys generated vendored Submodule

Submodule vendor/golang.org/x/sys added at f49334f85d

View File

@@ -1,4 +1,6 @@
---
Template: base_doc.html
Data:
background: /img/folder.jpg
@@ -9,9 +11,7 @@ Data:
---
# Benutzung
## Ordnerstruktur
# Ordnerstruktur
Das Ausgangsmaterial für die zu generierende Website wird in folgender Ordnerstruktur organisiert:
@@ -58,25 +58,3 @@ FIL config.yml (globale Konfiguration, enthält andere Anweisungen als individue
```
In der Minimal-Variante sind nur die Ordner `content` und `templates` nötig.
---
### `content`
- enthält die Markdown-Dateien und Konfigurationen für die Navigationsstruktur und Einzelseiten
- voranestellte Nummer mit Unterstrich wie z.B. `01_` dienen nur der Sortierung und gehen nicht in den eigentlichen Navigationspfad mit ein
- zur Bildung des Navigationspfades werden die Verzeichnisnamen in Kleinschreibung konvertiert
- Navigationsnamen für die Website werden aus dem Pfad gebildet, wobei `_`(Unterstriche) in Leerzeichen umgewandelt werden
- Navigationsnamen können durch die `config.yml` überschrieben werden
#### Medien und Downloads
- Mediendateien werden neben den Inhalten in `content` abgelegt und müssen dementsprechend relativ verlinkt werden
### `assets`
- Bilder/CSS/JS die in Templates oder mehrfach in den Content-Seiten benötigt werden liegen in `assets`
### `templates`
- Template-Dateien für die Generierung der Website liegen hier

View File

@@ -0,0 +1,11 @@
---
Data:
Version: "ab v1.0"
---
- enthält die Markdown-Dateien und Konfigurationen für die Navigationsstruktur und Einzelseiten
- voranestellte Nummer mit Unterstrich wie z.B. `01_` dienen nur der Sortierung und gehen nicht in den eigentlichen Navigationspfad mit ein
- zur Bildung des Navigationspfades werden die Verzeichnisnamen in Kleinschreibung konvertiert
- Navigationsnamen für die Website werden aus dem Pfad gebildet, wobei `_`(Unterstriche) in Leerzeichen umgewandelt werden
- Navigationsnamen können durch die `config.yml` überschrieben werden

View File

@@ -0,0 +1 @@
- Mediendateien werden neben den Inhalten in `content` abgelegt und müssen dementsprechend relativ verlinkt werden

View File

@@ -0,0 +1,7 @@
---
Data:
Version: "ab v1.0"
---
- Bilder/CSS/JS die in Templates oder mehrfach in den Content-Seiten benötigt werden liegen in `assets`

View File

@@ -0,0 +1,7 @@
---
Data:
Version: "ab v1.0"
---
- Template-Dateien für die Generierung der Website liegen hier

View File

@@ -1,3 +1,9 @@
This:
Data:
teaser: Wie werden die Inhalte und Templates organisiert?
Collections:
- Name: doccoll
Directory:
Path: "."
MatchFilename: "^_\\d+(?P<lowdash>_*)(?P<title>.+)\\.md"
ReverseOrder: False

View File

@@ -1,4 +1,6 @@
---
Template: base_doc.html
Data:
background: /img/design.jpg
@@ -15,361 +17,3 @@ Templates werden über das pongo2-Paket gerendert. Dieses nutzt die Template-Spr
Sämtliche Template-Dateien sind im Ordner `templates` zu speichern.
Die Endung kann frei gewählt werden. Für diese Dokumentation und auch als Grundlage für Beispiele wurde `.html` gewählt, da somit auch das Syntax-Highlighting gegeben ist.
## grober Überblick
Nachfolgend ist ein Beispiel eines Templates:
```django
<html>
<meta>
<meta charset="UTF-8">
<title>{{ Meta.Title }}</title>
<meta name="description" content="{{ Meta.Description }}" />
<meta name="keywords" content="{{ Meta.Keywords }}" />
<link rel="stylesheet" type="text/css" href="../assets/css/main.css">
</meta>
<body>
{% block header %}
<header>
<div class="langSelect">
{% for nav in NavSlice %}
<a href="{{ nav.GoTo }}" {% if nav.Active %}class="active"{% endif %}>
<img src="../assets/img/{{ nav.Navname }}.png" alt="{{ nav.Navname }}" style="height: 20px;">
</a>
{% endfor %}
</div>
<div id="logoDiv"><img src="../assets/img/logo.png"></div>
<div id="mainNavDiv" class="nav">
<table border="0" style="width: 100%">
<tr>
<td>
<div><b>main Level 1 ...</b></div>
<ul>
{% for nav in NavActive.0.SubMap.main.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
{% if nav.SubSlice %}
<ul>
{% for nav2 in nav.SubSlice %}
<li {% if nav2.Active %}class="active"{% endif %}>
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
{{ nav2.Navname }}
</a>
</li>
{% endfor %}
</ul
{% endif %}
</li>
{% endfor %}
</ul>
</td>
<td>
<div><b>main/service Level 2</b></div>
<ul>
{% for nav in NavActive.2.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
</li>
{% endfor %}
</ul>
</td>
</tr>
</table>
</div>
</header>
{% endblock %}
{% block breadcrumb %}
<div id="breadcrumb">
{% for nav in NavActive %}
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
{% endfor %}
</div>
{% endblock %}
{% block content %}
<div id="content">
{{ Body }}
</div>
{% endblock %}
{% block footer %}
<footer class="nav">
<div><b>service Level 1</b></div>
<ul>
{% for nav in NavActive.0.SubMap.service.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}</a>
</li>
{% endfor %}
</ul>
</footer>
{% endblock %}
</body>
</html>
```
Wie im Beispiel zu sehen ist, werden einfache **Variables** über:
```django
{{ Variable }}
```
eingebunden. Variablen können außerdem speziell weiterverarbeitet werden. Dies geschieht mit sogenannten Filtern oder Filterfunktionen. Die Syntax dafür ist folgendermaßen:
```django
{{ Variable|Filter }}
```
Blockanweisungen dagegen verwenden zum Beispiel folgende Platzhalter:
```django
{% if Variable %}
...
{% endif %}
```
Eine Liste der in Django möglichen Anweisungen finden Sie unter [Django builtins](https://docs.djangoproject.com/en/2.1/ref/templates/builtins/).
---
## mark2web Variablen
Der mark2web-Generator liefert für die Template-Verarbeitung Variablen für die Navigation und den Inhalt.
### Website-Inhalt
Das rohe HTML, welches aus einer Markdown-Datei generiert wird steht über folgende Variablen zur Verfügung.
```django
{{ Body }} = komplettes HTML aus der Markdown-Datei
{{ BodyParts.0 }} = erster HTML-Block
{{ BodyParts.1 }} = zweiter HTML-Block
usw.
```
Ist die Markdown-Datei durch `---` auf einer Zeile (nach den Kopfdaten) geteilt, stehen die Einzelteile im Slice/Array `{{ BodyParts }}` zur Verfügung.
Aus folgender Markdown-Datei `README.md` in einem `content`-Unterverzeichnis:
```markdown
# Titel 1
Text 1
---
## Titel 2
Text 2
```
wird für `{{ Body }}` folgendes HTML:
```html
<h1>Titel 1</h1>
<p>Text 1</p>
<hr>
<h2>Titel 2</h2>
<p>Text 2</p>
```
`BodyParts` erklärt sich an folgendem Template:
```django
<table>
<tr>
{% for part in BodyParts %}
<td>
{{ part }}
</td>
{% endfor %}
</tr>
</table>
```
Aus dem Template wird nach dem Rendern mit obiger Markdown-Datei also folgendes HTML:
```html
<table>
<tr>
<td>
<h1>Titel 1</h1>
<p>Text 1</p>
</td>
<td>
<h2>Titel 2</h2>
<p>Text 2</p>
</td>
</tr>
</table>
```
Die Einrückungen im HTML wurden für die bessere Lesbarkeit angepasst. Wie zu sehen ist, wird `---` in `{{ Body }}` laut Markdown-Syntax zu `<br>`. In `{{ BodyParts.N }}` ist es jedoch nicht enthalten, da es hier nur zur Trennung des Dokuments dient.
### Navigation
Jedes Navigationselement steht intern in folgender go-Struktur zur Verfügung:
```go
type navElement struct {
Navname string
GoTo string
Active bool
Data interface{}
This ThisPathConfig
SubMap *map[string]*navElement
SubSlice *[]*navElement
}
```
Diese erste Navigationsebene wird mit seinen Unternavigationspunkten zum einen auf die Variable `{{ NavMap }}` in Form einer Map (assoziatives Array) mit dem umgeformten Namen (wie im Zielverzeichnis) abgebildet.
Außerdem steht die erste Navigationsebene als Liste, bzw. Slice (Array) über die Variable `{{ NavSlice }}` zur verfügung.
Wird z.B. folgende Navigation als Zielverzeichnis-Struktur angenommen:
```plain
de
main
home
leistungen
referenzen
service
impressum
en
main
home
...
```
Der Teasertext aus folgender `config.yml` im `content`-Verzeichnis `de/main/02_Leistungen`
```yaml
This:
Data:
teaser: Teasertext
```
welcher zum Navigationspunkt im Zielpfad *de/main/leistungen* gehört, ist über folgende Template-Variablen erreichbar:
```django
{{ NavMap.de.SubMap.main.SubMap.leistungen.This.Data.teaser }}
oder
{{ NavSlice.0.SubSlice.0.SubSlice.1.This.Data.teaser}}
oder auch eine Kombination
{{ NavMap.de.SubMap.main.SubSlice.1.This.Data.teaser }}
```
Natürlich wird diese Variable in der Form so nie verwendet, sondern soll nur den Aufbau der Struktur verdeutlichen. Üblicherweise werden Schleifenblöcke verwendet um die Navigationsbäume auszugeben, wie z.B. eine Liste als Sprachwähler, wenn man annimmt, dass die erste Navigationsebene die Website-Sprache ist:
```django
<ul>
{% for lang in NavMap %}
<li {% if lang.Active %}class="active"{% endif %}>
<a href="{{ lang.GoTo }}">{{ lang.Navname }}</a>
</li>
{% endfor %}
</ul>
```
Wie im Beispiel zu sehen ist, wird das aktive Navigationselement mit `class="active"` über die Variable `Active` aus der Struktur markiert.
#### aktiver Navigationspfad
Der aktive Navigationspfad ist über eine weitere vorbelegte Variable zu erfahren:
```django
{{ NavActive }}
```
Ähnlich wie `{{ NavSlice }}` oder `{{ ...SubSlice }}` ist dies ein Slice/Array welches als Elemnte Navigationselemente aus oben angegebener Struktur enthält.
Im Gegensatz zu `{{ NavSlice }}` besteht die Liste nicht aus Elementen einer Ebene, sonder aus allen aktiven Elemtenten in des aktuellen Pfads.
Geht man also wieder vom obigen Beispiel aus und der aktive Pfad ist *de/main/leistungen*, so würden folgendes zutreffen:
```django
{{ NavActive.0 }} ist das Navigationselement für "de"
{{ NavActive.1 }} ist das Navigationselement für "main"
{{ NavActive.2 }} ist das Navigationselement für "Leistungen"
```
Somit lassen sich leicht Pfade anzeigen, bzw. Breadcrumbs in die Website einbinden, wie im folgenden Beispiel:
```django
aktiver Pfad:
{% for nav in NavActive %}
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">{{ nav.Navname }}</a>
{% endfor %}
```
Ebenso lässt sich bei mehrsprachigen Seite immer die richte Hauptnavigation zur aktuelle Sprache laden:
```django
<h3>Hauptnavigation</h3>
<ul>
{% for nav in NavActive.0.SubMap.main.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
{% if nav.SubSlice %}
<ul>
{% for nav2 in nav.SubSlice %}
<li {% if nav2.Active %}class="active"{% endif %}>
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
{{ nav2.Navname }}
</a>
</li>
{% endfor %}
</ul
{% endif %}
</li>
{% endfor %}
</ul>
```
### Meta-Angaben
Über die Variablen
```django
{{ Meta.Title }}
{{ Meta.Description }}
{{ Meta.Keywords }}
```
stehen die üblichen Meta-Angaben für die Verwendung im `<head>` Tag zur Verfügung.
### weitere Daten
Die Variablen
```django
{{ This.Navname }}
{{ This.Data }}
{{ Data }}
```
stehen ebenfalls zur Verfügung und spiegeln die Daten aus den Konfig-Dateien `config.yml` und den Kopfdaten der Markdown-Datei wieder.

View File

@@ -0,0 +1,125 @@
Nachfolgend ist ein Beispiel eines Templates:
```django
<html>
<meta>
<meta charset="UTF-8">
<title>{{ Meta.Title }}</title>
<meta name="description" content="{{ Meta.Description }}" />
<meta name="keywords" content="{{ Meta.Keywords }}" />
<link rel="stylesheet" type="text/css" href="../assets/css/main.css">
</meta>
<body>
{% block header %}
<header>
<div class="langSelect">
{% for nav in NavSlice %}
<a href="{{ nav.GoTo }}" {% if nav.Active %}class="active"{% endif %}>
<img src="../assets/img/{{ nav.Navname }}.png" alt="{{ nav.Navname }}" style="height: 20px;">
</a>
{% endfor %}
</div>
<div id="logoDiv"><img src="../assets/img/logo.png"></div>
<div id="mainNavDiv" class="nav">
<table border="0" style="width: 100%">
<tr>
<td>
<div><b>main Level 1 ...</b></div>
<ul>
{% for nav in NavActive.0.SubMap.main.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
{% if nav.SubSlice %}
<ul>
{% for nav2 in nav.SubSlice %}
<li {% if nav2.Active %}class="active"{% endif %}>
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
{{ nav2.Navname }}
</a>
</li>
{% endfor %}
</ul
{% endif %}
</li>
{% endfor %}
</ul>
</td>
<td>
<div><b>main/service Level 2</b></div>
<ul>
{% for nav in NavActive.2.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
</li>
{% endfor %}
</ul>
</td>
</tr>
</table>
</div>
</header>
{% endblock %}
{% block breadcrumb %}
<div id="breadcrumb">
{% for nav in NavActive %}
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
{% endfor %}
</div>
{% endblock %}
{% block content %}
<div id="content">
{{ Body }}
</div>
{% endblock %}
{% block footer %}
<footer class="nav">
<div><b>service Level 1</b></div>
<ul>
{% for nav in NavActive.0.SubMap.service.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}</a>
</li>
{% endfor %}
</ul>
</footer>
{% endblock %}
</body>
</html>
```
Wie im Beispiel zu sehen ist, werden einfache **Variables** über:
```django
{{ Variable }}
```
eingebunden. Variablen können außerdem speziell weiterverarbeitet werden. Dies geschieht mit sogenannten Filtern oder Filterfunktionen. Die Syntax dafür ist folgendermaßen:
```django
{{ Variable|Filter }}
```
Blockanweisungen dagegen verwenden zum Beispiel folgende Platzhalter:
```django
{% if Variable %}
...
{% endif %}
```
Eine Liste der in Django möglichen Anweisungen finden Sie unter [Django builtins](https://docs.djangoproject.com/en/2.1/ref/templates/builtins/).

View File

@@ -0,0 +1 @@
Der mark2web-Generator liefert für die Template-Verarbeitung Variablen für die Navigation und den Inhalt.

View File

@@ -0,0 +1,67 @@
Das rohe HTML, welches aus einer Markdown-Datei generiert wird steht über folgende Variablen zur Verfügung.
```django
{{ Body }} = komplettes HTML aus der Markdown-Datei
{{ BodyParts.0 }} = erster HTML-Block
{{ BodyParts.1 }} = zweiter HTML-Block
usw.
```
Ist die Markdown-Datei durch `---` auf einer Zeile (nach den Kopfdaten) geteilt, stehen die Einzelteile im Slice/Array `{{ BodyParts }}` zur Verfügung.
Aus folgender Markdown-Datei `README.md` in einem `content`-Unterverzeichnis:
```markdown
# Titel 1
Text 1
---
## Titel 2
Text 2
```
wird für `{{ Body }}` folgendes HTML:
```html
<h1>Titel 1</h1>
<p>Text 1</p>
<hr>
<h2>Titel 2</h2>
<p>Text 2</p>
```
`BodyParts` erklärt sich an folgendem Template:
```django
<table>
<tr>
{% for part in BodyParts %}
<td>
{{ part }}
</td>
{% endfor %}
</tr>
</table>
```
Aus dem Template wird nach dem Rendern mit obiger Markdown-Datei also folgendes HTML:
```html
<table>
<tr>
<td>
<h1>Titel 1</h1>
<p>Text 1</p>
</td>
<td>
<h2>Titel 2</h2>
<p>Text 2</p>
</td>
</tr>
</table>
```
Die Einrückungen im HTML wurden für die bessere Lesbarkeit angepasst. Wie zu sehen ist, wird `---` in `{{ Body }}` laut Markdown-Syntax zu `<br>`. In `{{ BodyParts.N }}` ist es jedoch nicht enthalten, da es hier nur zur Trennung des Dokuments dient.

View File

@@ -0,0 +1,72 @@
Jedes Navigationselement steht intern in folgender go-Struktur zur Verfügung:
```go
type navElement struct {
Navname string
GoTo string
Active bool
Data interface{}
This ThisPathConfig
SubMap *map[string]*navElement
SubSlice *[]*navElement
}
```
Diese erste Navigationsebene wird mit seinen Unternavigationspunkten zum einen auf die Variable `{{ NavMap }}` in Form einer Map (assoziatives Array) mit dem umgeformten Namen (wie im Zielverzeichnis) abgebildet.
Außerdem steht die erste Navigationsebene als Liste, bzw. Slice (Array) über die Variable `{{ NavSlice }}` zur verfügung.
Wird z.B. folgende Navigation als Zielverzeichnis-Struktur angenommen:
```plain
de
main
home
leistungen
referenzen
service
impressum
en
main
home
...
```
Der Teasertext aus folgender `config.yml` im `content`-Verzeichnis `de/main/02_Leistungen`
```yaml
This:
Data:
teaser: Teasertext
```
welcher zum Navigationspunkt im Zielpfad *de/main/leistungen* gehört, ist über folgende Template-Variablen erreichbar:
```django
{{ NavMap.de.SubMap.main.SubMap.leistungen.This.Data.teaser }}
oder
{{ NavSlice.0.SubSlice.0.SubSlice.1.This.Data.teaser}}
oder auch eine Kombination
{{ NavMap.de.SubMap.main.SubSlice.1.This.Data.teaser }}
```
Natürlich wird diese Variable in der Form so nie verwendet, sondern soll nur den Aufbau der Struktur verdeutlichen. Üblicherweise werden Schleifenblöcke verwendet um die Navigationsbäume auszugeben, wie z.B. eine Liste als Sprachwähler, wenn man annimmt, dass die erste Navigationsebene die Website-Sprache ist:
```django
<ul>
{% for lang in NavMap %}
<li {% if lang.Active %}class="active"{% endif %}>
<a href="{{ lang.GoTo }}">{{ lang.Navname }}</a>
</li>
{% endfor %}
</ul>
```
Wie im Beispiel zu sehen ist, wird das aktive Navigationselement mit `class="active"` über die Variable `Active` aus der Struktur markiert.

View File

@@ -0,0 +1,52 @@
Der aktive Navigationspfad ist über eine weitere vorbelegte Variable zu erfahren:
```django
{{ NavActive }}
```
Ähnlich wie `{{ NavSlice }}` oder `{{ ...SubSlice }}` ist dies ein Slice/Array welches als Elemnte Navigationselemente aus oben angegebener Struktur enthält.
Im Gegensatz zu `{{ NavSlice }}` besteht die Liste nicht aus Elementen einer Ebene, sonder aus allen aktiven Elemtenten in des aktuellen Pfads.
Geht man also wieder vom obigen Beispiel aus und der aktive Pfad ist *de/main/leistungen*, so würden folgendes zutreffen:
```django
{{ NavActive.0 }} ist das Navigationselement für "de"
{{ NavActive.1 }} ist das Navigationselement für "main"
{{ NavActive.2 }} ist das Navigationselement für "Leistungen"
```
Somit lassen sich leicht Pfade anzeigen, bzw. Breadcrumbs in die Website einbinden, wie im folgenden Beispiel:
```django
aktiver Pfad:
{% for nav in NavActive %}
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">{{ nav.Navname }}</a>
{% endfor %}
```
Ebenso lässt sich bei mehrsprachigen Seite immer die richte Hauptnavigation zur aktuelle Sprache laden:
```django
<h3>Hauptnavigation</h3>
<ul>
{% for nav in NavActive.0.SubMap.main.SubSlice %}
<li {% if nav.Active %}class="active"{% endif %}>
<a href="{{ nav.GoTo }}" title="{{ nav.This.Data.hoverText }}">
{{ nav.Navname }}
</a>
{% if nav.SubSlice %}
<ul>
{% for nav2 in nav.SubSlice %}
<li {% if nav2.Active %}class="active"{% endif %}>
<a href="{{ nav2.GoTo }}" title="{{ nav2.This.Data.hoverText }}">
{{ nav2.Navname }}
</a>
</li>
{% endfor %}
</ul
{% endif %}
</li>
{% endfor %}
</ul>
```

View File

@@ -0,0 +1,9 @@
Über die Variablen
```django
{{ Meta.Title }}
{{ Meta.Description }}
{{ Meta.Keywords }}
```
stehen die üblichen Meta-Angaben für die Verwendung im `<head>` Tag zur Verfügung.

View File

@@ -0,0 +1,9 @@
Die Variablen
```django
{{ This.Navname }}
{{ This.Data }}
{{ Data }}
```
stehen ebenfalls zur Verfügung und spiegeln die Daten aus den Konfig-Dateien `config.yml` und den Kopfdaten der Markdown-Datei wieder.

View File

@@ -1,3 +1,9 @@
This:
Data:
teaser: Aus Markdown wird HTML
Collections:
- Name: doccoll
Directory:
Path: "."
MatchFilename: "^_\\d+(?P<lowdash>_*)(?P<title>.+)\\.md"
ReverseOrder: False

View File

@@ -1,7 +1,7 @@
#This:
This:
Collections:
- Name: blog1st
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}'
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=0&limit=1'
NavTemplate:
EntriesAttribute: entries
GoTo: '{{ date }}-{{ title }}'
@@ -12,7 +12,7 @@
Hidden: true # hide from nav, but use this feature for rendering detail sites
- Name: blog1skip
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100{% if Data.details._id %}&filter[link._id]={{ Data.details._id }}{% else %}&filter[link][$exists]=0{% endif %}'
URL: 'https://mark2web.basiscms.de/api/collections/get/mark2webBlog?token={{ Data.token }}&filter[published]=true&sort[date]=-1&skip=1&limit=100'
NavTemplate:
EntriesAttribute: entries
GoTo: '{{ date }}-{{ title }}'

View File

@@ -303,3 +303,7 @@ code.language-mermaid svg {
border: #444 solid 1px;
border-radius: 5px;
}
.blogEntry {
margin-bottom: 50px;
}

View File

@@ -31,11 +31,11 @@
<header id="header">
<div class="container">
<div class="logo" {% if not Data.slider and not Data.details.slider %}style="max-width: 400px;"{% endif %}>
<div class="logo" {% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}style="max-width: 400px;"{% endif %}>
<a href="{{ "/"|relative_path }}">
<picture>
<source media="(max-width: 768px)" srcset="project-files/img/logo_text.png">
<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider %}_text{% endif %}.png" alt="mark2web Logo">
<img src="project-files/img/logo{% if not Data.slider and not Data.details.slider and not Data.details.firstimg %}_text{% endif %}.png" alt="mark2web Logo">
</picture>
</a>
</div>
@@ -101,6 +101,7 @@
<!-- ========== Main Content ========== -->
<div class="maincontent">
{% block bg %}
{% if Data.background %}
<img
src="{{ Data.background|image_process:"p=fill,w=1440,h=810,q=30,t=/img" }}"
@@ -109,6 +110,7 @@
{{ Data.background|image_process:"p=fill,w=1920,h=1020,q=30,t=/img" }} 1920w"
alt="{{ Meta.Title }}" class="img2bg">
{% endif %}
{% endblock bg %}
<div class="white_section section_padding">
<div class="container">
{% block part1 %}

View File

@@ -19,6 +19,7 @@
limit wird für skip in Query gebraucht
{% endcomment %}
{% for e in NavElement.ColMap.blog1skip.entries %}
<div class="blogEntry">
<h2>
{{ e.title }}
<div class="datum">{{ e.date|datum }}</div>
@@ -27,5 +28,6 @@
{% if e.body %}
<a href="{{ e.date|add:"-"|add:e.title|slugify }}" class="btn">mehr lesen &raquo;</a>
{% endif %}
</div>
{% endfor %}
{% endblock part1 %}

View File

@@ -8,6 +8,18 @@
{% if Data.details.slider %}
{% block slider %}
{% with Data.details.firstimg as img %}
{% if img %}
<div class="slide">
<img
src="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=600,q=60" }}"
srcset="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=768,h=384,q=60" }} 768w,
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=600,q=60" }} 1440w,
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1920,h=800,q=60" }} 1920w"
alt="{{ img.meta.title }}">
</div>
{% endif %}
{% endwith %}
{% for sl in Data.details.slider %}
<div class="slide">
<img
@@ -29,6 +41,19 @@
{{ Data.details.teaser|markdown:"s=monokai" }}
{% endblock part0 %}
{% block bg %}
{% with Data.details.firstimg as img %}
{% if img %}
<img
src="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=810,q=30,t=/img" }}"
srcset="{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=768,h=768,q=30,t=/img" }} 768w,
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1440,h=810,q=30,t=/img" }} 1440w,
{{ "https://mark2web.basiscms.de"|add:img.path|image_process:"p=fill,w=1920,h=1020,q=30,t=/img" }} 1920w"
alt="{{ Meta.Title }}" class="img2bg">
{% endif %}
{% endwith %}
{% endblock bg %}
{% block part1 %}
{{ Body }}
<br>