multithreaded image processing

cmd/mark2web/main.go

@@ -20,8 +20,6 @@ var (
 	BuildTime = "UNKNOWN"
 )
 
-var log = helper.Log
-
 func main() {
 	inDir := flag.String("in", "./", "input directory")
 	outDir := flag.String("out", "html", "output directory")
@@ -46,53 +44,53 @@ func main() {
 	helper.ConfigureLogger(level)
 
 	if inDir == nil || *inDir == "" {
-		log.Panic("input directory not specified")
+		helper.Log.Panic("input directory not specified")
 	}
 	iDir := path.Clean(*inDir)
 	inDir = &iDir
-	log.Infof("input directory: %s", *inDir)
+	helper.Log.Infof("input directory: %s", *inDir)
 
 	if outDir == nil || *outDir == "" {
-		log.Panic("output directory not specified")
+		helper.Log.Panic("output directory not specified")
 	}
 	oDir := path.Clean(*outDir)
 	outDir = &oDir
-	log.Infof("output directory: %s", *outDir)
+	helper.Log.Infof("output directory: %s", *outDir)
 
 	if createOutDir != nil && *createOutDir {
 		if _, err := os.Stat(*outDir); os.IsNotExist(err) {
-			log.Debugf("output directory '%s' does not exist", *outDir)
+			helper.Log.Debugf("output directory '%s' does not exist", *outDir)
-			log.Debugf("trying to create output directory: %s", *outDir)
+			helper.Log.Debugf("trying to create output directory: %s", *outDir)
 			err := os.MkdirAll(*outDir, 0755)
 			if err != nil {
-				log.Panic(err)
+				helper.Log.Panic(err)
 			}
-			log.Noticef("created output directory: %s", *outDir)
+			helper.Log.Noticef("created output directory: %s", *outDir)
 		} else {
-			log.Noticef("output directory '%s' already exists", *outDir)
+			helper.Log.Noticef("output directory '%s' already exists", *outDir)
 		}
 	}
 
 	if fD, err := os.Stat(*outDir); os.IsNotExist(err) {
-		log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
+		helper.Log.Panicf("output directory '%s' does not exist, try -create parameter or create manually", *outDir)
 	} else {
 		if fD == nil {
-			log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
+			helper.Log.Panicf("something went wrong, could not get file handle for output dir %s", *outDir)
 		} else if !fD.IsDir() {
-			log.Panicf("output directory '%s' is not a directory", *outDir)
+			helper.Log.Panicf("output directory '%s' is not a directory", *outDir)
 		}
 	}
 
-	log.Debug("reading global config...")
+	helper.Log.Debug("reading global config...")
 	configFilename := *inDir + "/config.yml"
 	err := mark2web.Config.ReadFromFile(configFilename)
 	if err != nil {
-		log.Panicf("could not read file '%s': %s", configFilename, err)
+		helper.Log.Panicf("could not read file '%s': %s", configFilename, err)
 	}
 	mark2web.Config.Directories.Input = *inDir
 	mark2web.Config.Directories.Output = *outDir
 
-	log.Debugf("reading input directory %s", *inDir)
+	helper.Log.Debugf("reading input directory %s", *inDir)
 
 	defaultTemplate := "base.html"
 	defaultInputFile := "README.md"

pkg/filter/image_process.go

@@ -167,16 +167,14 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 	if f, err := os.Stat(imgTarget); err == nil && !f.IsDir() {
 		helper.Log.Noticef("skipped processing image from %s to %s, file already exists", imgSource, imgTarget)
 	} else {
+		mark2web.ThreadStart(func() {
 			helper.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
 			if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
 				// webrequest before finding target filename, because of file format in filename
 			} else {
 				img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
 				if err != nil {
-				return nil, &pongo2.Error{
+					helper.Log.Panicf("filter:image_resize, could not open image '%s': %s", imgSource, err)
-					Sender:    "filter:image_resize",
-					OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
-				}
 				}
 			}
 
@@ -209,17 +207,11 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 				case "bottomright":
 					anchor = imaging.BottomRight
 				default:
-				return nil, &pongo2.Error{
+					helper.Log.Panicf("filter:image_resize, unknown anchor a=%s definition", p.Anchor)
-					Sender:    "filter:image_resize",
-					OrigError: fmt.Errorf("unknown anchor a=%s definition", p.Anchor),
-				}
 				}
 				img = imaging.Fill(img, p.Width, p.Height, anchor, imaging.Lanczos)
 			default:
-			return nil, &pongo2.Error{
+				helper.Log.Panicf("filter:image_resize, invalid p parameter '%s'", p.Process)
-				Sender:    "filter:image_resize",
-				OrigError: fmt.Errorf("invalid p parameter '%s'", p.Process),
-			}
 			}
 
 			var encodeOptions = make([]imaging.EncodeOption, 0)
@@ -229,11 +221,10 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 
 			err = imaging.Save(img, imgTarget, encodeOptions...)
 			if err != nil {
-			return nil, &pongo2.Error{
+				helper.Log.Panicf("filter:image_resize, could save image '%s': %s", imgTarget, err)
-				Sender:    "filter:image_resize",
-				OrigError: fmt.Errorf("could save image '%s': %s", imgTarget, err),
-			}
 			}
+			helper.Log.Noticef("finished image: %s", imgTarget)
+		})
 	}
 	return pongo2.AsValue(mark2web.ResolveNavPath(p.Filename)), nil
 }

pkg/helper/markdown.go

@@ -7,6 +7,7 @@ import (
 	"gopkg.in/russross/blackfriday.v2"
 )
 
+// RenderMarkdown renders input to html with chroma syntax highlighting if wanted
 func RenderMarkdown(input []byte, chromaRenderer bool, chromaStyle string) []byte {
 	var options []blackfriday.Option
 

pkg/helper/webrequest.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 )
 
+// JSONWebRequest will GET a json object/array from a given URL
 func JSONWebRequest(url string) interface{} {
 	Log.Noticef("requesting url via GET %s", url)
 

pkg/mark2web/config_global.go

@@ -32,8 +32,10 @@ type GlobalConfig struct {
 	}
 }
 
+// Config is global config
 var Config = new(GlobalConfig)
 
+// ReadFromFile reads yaml config from file
 func (c *GlobalConfig) ReadFromFile(filename string) error {
 	data, err := ioutil.ReadFile(filename)
 	if err != nil {

pkg/mark2web/context.go

@@ -15,9 +15,13 @@ import (
 	"gopkg.in/yaml.v2"
 )
 
+// CurrentContext is current pongo2 template context
 var CurrentContext *pongo2.Context
+
+// CurrentTreeNode is current node we are on while processing template
 var CurrentTreeNode *TreeNode
 
+// NewContext returns prefilled context with some functions and variables
 func NewContext() pongo2.Context {
 	ctx := pongo2.Context{
 		"fnRequest": RequestFn,

pkg/mark2web/run.go

@@ -1,5 +1,6 @@
 package mark2web
 
+// Run will do a complete run of mark2web
 func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
 	SetTemplateDir(inDir + "/templates")
 
@@ -8,4 +9,6 @@ func Run(inDir, outDir string, defaultPathConfig *PathConfig) {
 	tree.ProcessContent()
 
 	ProcessAssets()
+
+	Wait()
 }

pkg/mark2web/wait.go

@@ -0,0 +1,55 @@
+package mark2web
+
+import (
+	"runtime"
+	"sync"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+)
+
+var wg sync.WaitGroup
+var numCPU = runtime.NumCPU()
+var curNumThreads = 1 // main thread is 1
+
+func init() {
+	helper.Log.Infof("number of CPU core: %d", numCPU)
+}
+
+// Wait will wait for all our internal go threads
+func Wait() {
+	wg.Wait()
+}
+
+// ThreadSetup adds 1 to wait group
+func ThreadSetup() {
+	curNumThreads++
+	wg.Add(1)
+}
+
+// ThreadDone removes 1 from wait group
+func ThreadDone() {
+	curNumThreads--
+	wg.Done()
+}
+
+// ThreadStart will start a thread an manages the wait group
+func ThreadStart(f func(), forceNewThread ...bool) {
+	force := false
+	if len(forceNewThread) > 0 && forceNewThread[0] {
+		force = true
+	}
+
+	if numCPU > curNumThreads || force {
+		// only new thread if empty CPU core available or forced
+		threadF := func() {
+			f()
+			ThreadDone()
+		}
+
+		ThreadSetup()
+		go threadF()
+	} else {
+		helper.Log.Debugf("no more CPU core (%d used), staying in main thread", curNumThreads)
+		f()
+	}
+}