apairon/mark2web
1
1
Fork 0
Code Issues 8 Pull Requests Releases 5 Activity

multithreaded image processing
Some checks failed
continuous-integration/drone/push Build is failing
Details

Browse Source
This commit is contained in:
Sebastian Frank 2019-03-19 12:46:32 +01:00
parent ada333a0e1
commit d652afd633
Signed by: apairon
GPG Key ID: 7270D06DDA7FE8C3
8 changed files with 132 additions and 77 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
cmd/mark2web/main.go
View File

@ -20,8 +20,6 @@ var (
BuildTime = "UNKNOWN"
)
var log = helper.Log
func main() {
inDir := flag.String("in", "./", "input directory")
outDir := flag.String("out", "html", "output directory")
@ -46,53 +44,53 @@ func main() {
helper.ConfigureLogger(level)
if inDir == nil || *inDir == "" {
log.Panic("input directory not specified")
helper.Log.Panic("input directory not specified")
}
iDir := path.Clean(*inDir)
inDir = &iDir
log.Infof("input directory: %s", *inDir)
helper.Log.Infof("input directory: %s", *inDir)
if outDir == nil || *outDir == "" {
log.Panic("output directory not specified")
helper.Log.Panic("output directory not specified")
}
oDir := path.Clean(*outDir)
outDir = &oDir
log.Infof("output directory: %s", *outDir)
helper.Log.Infof("output directory: %s", *outDir)
if createOutDir != nil && *createOutDir {
if _, err := os.Stat(*outDir); os.IsNotExist(err) {
log.Debugf("output directory '%s' does not exist", *outDir)
log.Debugf("trying to create output directory: %s", *outDir)
helper.Log.Debugf("output directory '%s' does not exist", *outDir)
helper.Log.Debugf("trying to create output directory: %s", *outDir)
err := os.MkdirAll(*outDir, 0755)
if err != nil {
log.Panic(err)
helper.Log.Panic(err)
}
log.Noticef("created output directory: %s", *outDir)
helper.Log.Noticef("created output directory: %s", *outDir)
} else {
log.Noticef("output directory '%s' already exists", *outDir)
helper.Log.Noticef("output directory '%s' already exists", *outDir)
}
}
if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
helper.Log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
} else {
if fD == nil {
log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
helper.Log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
} else if !fD.IsDir() {
log.Panicf("output directory '%s' is not a directory", *outDir)
helper.Log.Panicf("output directory '%s' is not a directory", *outDir)
}
}
log.Debug("reading global config...")
helper.Log.Debug("reading global config...")
configFilename := *inDir + "/config.yml"
err := mark2web.Config.ReadFromFile(configFilename)
if err != nil {
log.Panicf("could not read file '%s': %s", configFilename, err)
helper.Log.Panicf("could not read file '%s': %s", configFilename, err)
}
mark2web.Config.Directories.Input = *inDir
mark2web.Config.Directories.Output = *outDir
log.Debugf("reading input directory %s", *inDir)
helper.Log.Debugf("reading input directory %s", *inDir)
defaultTemplate := "base.html"
defaultInputFile := "README.md"

111
pkg/filter/image_process.go
View File

@ -167,73 +167,64 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
if f, err := os.Stat(imgTarget); err == nil && !f.IsDir() {
helper.Log.Noticef("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
} else {
helper.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
// webrequest before finding target filename, because of file format in filename
} else {
img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
mark2web.ThreadStart(func() {
helper.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
// webrequest before finding target filename, because of file format in filename
} else {
img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
if err != nil {
helper.Log.Panicf("filter:image_resize, could not open image '%s': %s", imgSource, err)
}
}
}
switch p.Process {
case "resize":
img = imaging.Resize(img, p.Width, p.Height, imaging.Lanczos)
case "fit":
img = imaging.Fit(img, p.Width, p.Height, imaging.Lanczos)
case "fill":
var anchor imaging.Anchor
switch strings.ToLower(p.Anchor) {
case "":
fallthrough
case "center":
anchor = imaging.Center
case "topleft":
anchor = imaging.TopLeft
case "top":
anchor = imaging.Top
case "topright":
anchor = imaging.TopRight
case "left":
anchor = imaging.Left
case "right":
anchor = imaging.Right
case "bottomleft":
anchor = imaging.BottomLeft
case "bottom":
anchor = imaging.Bottom
case "bottomright":
anchor = imaging.BottomRight
switch p.Process {
case "resize":
img = imaging.Resize(img, p.Width, p.Height, imaging.Lanczos)
case "fit":
img = imaging.Fit(img, p.Width, p.Height, imaging.Lanczos)
case "fill":
var anchor imaging.Anchor
switch strings.ToLower(p.Anchor) {
case "":
fallthrough
case "center":
anchor = imaging.Center
case "topleft":
anchor = imaging.TopLeft
case "top":
anchor = imaging.Top
case "topright":
anchor = imaging.TopRight
case "left":
anchor = imaging.Left
case "right":
anchor = imaging.Right
case "bottomleft":
anchor = imaging.BottomLeft
case "bottom":
anchor = imaging.Bottom
case "bottomright":
anchor = imaging.BottomRight
default:
helper.Log.Panicf("filter:image_resize, unknown anchor a=%s definition", p.Anchor)
}
img = imaging.Fill(img, p.Width, p.Height, anchor, imaging.Lanczos)
default:
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("unknown anchor a=%s definition", p.Anchor),
}
helper.Log.Panicf("filter:image_resize, invalid p parameter '%s'", p.Process)
}
img = imaging.Fill(img, p.Width, p.Height, anchor, imaging.Lanczos)
default:
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("invalid p parameter '%s'", p.Process),
}
}
var encodeOptions = make([]imaging.EncodeOption, 0)
if p.Quality > 0 {
encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
}
err = imaging.Save(img, imgTarget, encodeOptions...)
if err != nil {
return nil, &pongo2.Error{
Sender: "filter:image_resize",
OrigError: fmt.Errorf("could save image '%s': %s", imgTarget, err),
var encodeOptions = make([]imaging.EncodeOption, 0)
if p.Quality > 0 {
encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
}
}
err = imaging.Save(img, imgTarget, encodeOptions...)
if err != nil {
helper.Log.Panicf("filter:image_resize, could save image '%s': %s", imgTarget, err)
}
helper.Log.Noticef("finished image: %s", imgTarget)
})
}
return pongo2.AsValue(mark2web.ResolveNavPath(p.Filename)), nil
}

1
pkg/helper/markdown.go
View File

@ -7,6 +7,7 @@ import (
"gopkg.in/russross/blackfriday.v2"
)
// RenderMarkdown renders input to html with chroma syntax highlighting if wanted
func RenderMarkdown(input []byte, chromaRenderer bool, chromaStyle string) []byte {
var options []blackfriday.Option

1
pkg/helper/webrequest.go
View File

@ -7,6 +7,7 @@ import (
"strings"
)
// JSONWebRequest will GET a json object/array from a given URL
func JSONWebRequest(url string) interface{} {
Log.Noticef("requesting url via GET %s", url)

2
pkg/mark2web/config_global.go
View File

@ -32,8 +32,10 @@ type GlobalConfig struct {
}
}
// Config is global config
var Config = new(GlobalConfig)
// ReadFromFile reads yaml config from file
func (c *GlobalConfig) ReadFromFile(filename string) error {
data, err := ioutil.ReadFile(filename)
if err != nil {

4
pkg/mark2web/context.go
View File

@ -15,9 +15,13 @@ import (
"gopkg.in/yaml.v2"
)
// CurrentContext is current pongo2 template context
var CurrentContext *pongo2.Context
// CurrentTreeNode is current node we are on while processing template
var CurrentTreeNode *TreeNode
// NewContext returns prefilled context with some functions and variables
func NewContext() pongo2.Context {
ctx := pongo2.Context{
"fnRequest": RequestFn,

3
pkg/mark2web/run.go
View File

@ -1,5 +1,6 @@
package mark2web
// Run will do a complete run of mark2web
func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
SetTemplateDir(inDir + "/templates")
@ -8,4 +9,6 @@ func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
tree.ProcessContent()
ProcessAssets()
Wait()
}

55
pkg/mark2web/wait.go Normal file
View File

@ -0,0 +1,55 @@
package mark2web
import (
"runtime"
"sync"
"gitbase.de/apairon/mark2web/pkg/helper"
)
var wg sync.WaitGroup
var numCPU = runtime.NumCPU()
var curNumThreads = 1 // main thread is 1
func init() {
helper.Log.Infof("number of CPU core: %d", numCPU)
}
// Wait will wait for all our internal go threads
func Wait() {
wg.Wait()
}
// ThreadSetup adds 1 to wait group
func ThreadSetup() {
curNumThreads++
wg.Add(1)
}
// ThreadDone removes 1 from wait group
func ThreadDone() {
curNumThreads--
wg.Done()
}
// ThreadStart will start a thread an manages the wait group
func ThreadStart(f func(), forceNewThread ...bool) {
force := false
if len(forceNewThread) > 0 && forceNewThread[0] {
force = true
}
if numCPU > curNumThreads || force {
// only new thread if empty CPU core available or forced
threadF := func() {
f()
ThreadDone()
}
ThreadSetup()
go threadF()
} else {
helper.Log.Debugf("no more CPU core (%d used), staying in main thread", curNumThreads)
f()
}
}