-progress cli param for bars

.gitmodules

@@ -79,3 +79,15 @@
 [submodule "vendor/github.com/itchio/go-brotli"]
 	path = vendor/github.com/itchio/go-brotli
 	url = https://github.com/itchio/go-brotli
+[submodule "vendor/github.com/gosuri/uiprogress"]
+	path = vendor/github.com/gosuri/uiprogress
+	url = https://github.com/gosuri/uiprogress
+[submodule "vendor/github.com/gosuri/uilive"]
+	path = vendor/github.com/gosuri/uilive
+	url = https://github.com/gosuri/uilive
+[submodule "vendor/github.com/mattn/go-tty"]
+	path = vendor/github.com/mattn/go-tty
+	url = https://github.com/mattn/go-tty
+[submodule "vendor/golang.org/x/sys"]
+	path = vendor/golang.org/x/sys
+	url = https://go.googlesource.com/sys

cmd/mark2web/main.go

@@ -9,6 +9,7 @@ import (
 	"gitbase.de/apairon/mark2web/pkg/filter"
 	"gitbase.de/apairon/mark2web/pkg/logger"
 	"gitbase.de/apairon/mark2web/pkg/mark2web"
+	"gitbase.de/apairon/mark2web/pkg/progress"
 )
 
 var (
@@ -26,6 +27,7 @@ func main() {
 	createOutDir := flag.Bool("create", false, "create output directory if not existing")
 	//clearOutDir := flag.Bool("clear", false, "clear output directory before generating website")
 	logLevel := flag.String("logLevel", "notice", "log level: debug, info, notice, warning, error")
+	progressBars := flag.Bool("progress", false, "show progress bars for jobs")
 	version := flag.Bool("version", false, "print version of this executable")
 
 	flag.Parse()
@@ -43,6 +45,10 @@ func main() {
 	}
 	logger.SetLogLevel(level)
 
+	if progressBars != nil && *progressBars {
+		progress.Init()
+	}
+
 	if inDir == nil || *inDir == "" {
 		logger.Log.Panic("input directory not specified")
 	}

pkg/filter/image_process.go

@@ -5,6 +5,9 @@ import (
 	"errors"
 	"fmt"
 	"image"
+	"image/gif"
+	"image/jpeg"
+	"image/png"
 	"net/http"
 	"net/url"
 	"os"
@@ -107,27 +110,18 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 		p.Quality,
 	)
 	if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
-		// remote file
-		img, p.Format, err = getImageFromURL(imgSource)
-		if err != nil {
-			return nil, &pongo2.Error{
-				Sender:    "filter:image_resize",
-				OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err),
-			}
-		}
 		// build filename
 		if p.Filename == "" {
 			var fBase string
 			if u, _ := url.Parse(imgSource); u != nil {
-				fBase = strings.Split(path.Base(u.Path), ".")[0]
+				fBase = path.Base(u.Path)
 			}
 
 			p.Filename = fmt.Sprintf(
-				"%s_%x_%s.%s",
+				"%s_%x_%s",
 				filePrefix,
 				md5.Sum([]byte(imgSource)),
 				fBase,
-				p.Format,
 			)
 		}
 	} else {
@@ -173,13 +167,12 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 			Function: func() {
 				logger.Log.Noticef("processing image from %s to %s", imgSource, imgTarget)
 				if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") {
-					// webrequest before finding target filename, because of file format in filename
+					// remote file
+					img, p.Format, err = getImageFromURL(imgSource)
 				} else {
 					img, err = imaging.Open(imgSource, imaging.AutoOrientation(true))
-					if err != nil {
-						logger.Log.Panicf("filter:image_resize, could not open image '%s': %s", imgSource, err)
-					}
 				}
+				logger.Eexit(err, "filter:image_resize, could not open image '%s'", imgSource)
 
 				switch p.Process {
 				case "resize":
@@ -217,18 +210,47 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, *
 					logger.Log.Panicf("filter:image_resize, invalid p parameter '%s'", p.Process)
 				}
 
-				var encodeOptions = make([]imaging.EncodeOption, 0)
-				if p.Quality > 0 {
-					encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
+				if p.Format == "" {
+					switch strings.ToLower(path.Ext(imgTarget)) {
+					case ".jpg", ".jpeg", ".gif", ".png":
+						var encodeOptions = make([]imaging.EncodeOption, 0)
+						if p.Quality > 0 {
+							encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality))
+						}
+
+						err = imaging.Save(img, imgTarget, encodeOptions...)
+						logger.Eerr(err, "filter:image_resize, could save image '%s'", imgTarget)
+					default:
+						logger.E("filter:image_resize, invalid filename extension for image: %s", imgTarget)
+						os.Exit(1)
+					}
+				} else {
+					out, err := os.Create(imgTarget)
+					defer out.Close()
+
+					logger.Eerr(err, "filter:image_resize, could not create image file '%s'", imgTarget)
+					switch p.Format {
+					case "jpeg", "jpg":
+						var jpegOpt *jpeg.Options
+						if p.Quality > 0 {
+							jpegOpt = &jpeg.Options{
+								Quality: p.Quality,
+							}
+						}
+						err = jpeg.Encode(out, img, jpegOpt)
+					case "png":
+						err = png.Encode(out, img)
+					case "gif":
+						err = gif.Encode(out, img, nil)
+					default:
+						logger.E("filter:image_resize, unknown format '%s' for '%s'", p.Format, imgSource)
+					}
+					logger.Eerr(err, "filter:image_resize, could not encode image file '%s'", imgTarget)
 				}
 
-				err = imaging.Save(img, imgTarget, encodeOptions...)
-				if err != nil {
-					logger.Log.Panicf("filter:image_resize, could save image '%s': %s", imgTarget, err)
-				}
 				logger.Log.Noticef("finished image: %s", imgTarget)
 			},
-			Description: "process image " + imgSource,
+			Description: imgSource,
 			Category:    "image process",
 		})
 	}

pkg/filter/image_process_test.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"testing"
 
+	"gitbase.de/apairon/mark2web/pkg/jobm"
 	"gitbase.de/apairon/mark2web/pkg/mark2web"
 
 	"github.com/flosch/pongo2"
@@ -19,7 +20,7 @@ func TestImageProcessFilter(t *testing.T) {
 		}
 
 		// we want to check files after function calls, so no multithreading
-		mark2web.SetNumCPU(1)
+		jobm.SetNumCPU(1)
 
 		mark2web.Config.Directories.Input = "../../test/in"
 		mark2web.Config.Directories.Output = "../../test/out"

pkg/helper/string.go

@@ -0,0 +1,13 @@
+package helper
+
+// ShortenStringLeft shortens a string
+func ShortenStringLeft(str string, num int) string {
+	tstr := str
+	if len(str) > num {
+		if num > 3 {
+			num -= 3
+		}
+		tstr = "..." + str[len(str)-num:len(str)]
+	}
+	return tstr
+}

pkg/jobm/jobmanager.go

@@ -3,8 +3,9 @@ package jobm
 import (
 	"runtime"
 	"sync"
+	"time"
 
-	"gitbase.de/apairon/mark2web/pkg/logger"
+	"gitbase.de/apairon/mark2web/pkg/progress"
 )
 
 var wg sync.WaitGroup
@@ -17,37 +18,40 @@ type Job struct {
 	Category    string
 }
 
-var jobChan = make(chan Job)
+var jobChan = make(chan []Job)
 
-func worker(jobChan <-chan Job) {
+func worker(jobChan <-chan []Job) {
 	defer wg.Done()
 
-	for job := range jobChan {
-		job.Function()
+	for jobs := range jobChan {
+		for _, job := range jobs {
+			progress.DescribeCurrent(job.Category, job.Description)
+			job.Function()
+			progress.IncrDone(job.Category)
+		}
 	}
 }
 
 func init() {
-	logger.Log.Infof("number of CPU core: %d", numCPU)
+	//logger.Log.Infof("number of CPU core: %d", numCPU)
 	// one core for main thread
-	for i := 0; i < (numCPU - 1); i++ {
+	for i := 0; i < numCPU; i++ {
 		wg.Add(1)
 		go worker(jobChan)
 	}
 }
 
 // Enqueue enqueues a job to the job queue
-func Enqueue(job Job) {
-	if numCPU <= 1 {
-		// no threading
-		job.Function()
-		return
+func Enqueue(jobs ...Job) {
+	for _, job := range jobs {
+		progress.IncrTotal(job.Category)
 	}
-	jobChan <- job
+	jobChan <- jobs
 }
 
 // Wait will wait for all jobs to finish
 func Wait() {
+	time.Sleep(time.Millisecond * 300)
 	close(jobChan)
 	wg.Wait()
 }

pkg/mark2web/compress.go

@@ -61,7 +61,7 @@ func handleCompression(filename string, content []byte) {
 				}
 			}
 		},
-		Description: "compress " + filename,
+		Description: filename,
 		Category:    "compress",
 	})
 }

pkg/progress/bar.go

@@ -0,0 +1,66 @@
+package progress
+
+import (
+	"fmt"
+
+	"gitbase.de/apairon/mark2web/pkg/helper"
+	"github.com/gosuri/uiprogress"
+	"github.com/mattn/go-tty"
+)
+
+type bar struct {
+	Bar         *uiprogress.Bar
+	Description string
+}
+
+var bars = make(map[string]*bar)
+var initialized = false
+var terminalWidth = 80
+
+// Init initializes the bar drawing
+func Init() {
+	if t, err := tty.Open(); err == nil && t != nil {
+		terminalWidth, _, _ = t.Size()
+		t.Close()
+	}
+	uiprogress.Start() // start rendering
+	initialized = true
+}
+
+// IncrTotal increases the total jobs for the bar
+func IncrTotal(barname string) {
+	if initialized {
+		_bar := bars[barname]
+		if _bar == nil {
+			_bar = new(bar)
+			_bar.Bar = uiprogress.AddBar(1)
+			_bar.Bar.Width = 25
+
+			_bar.Bar.PrependFunc(func(b *uiprogress.Bar) string {
+				return fmt.Sprintf("%15s: %3d/%3d", helper.ShortenStringLeft(barname, 15), b.Current(), b.Total)
+			})
+			_bar.Bar.AppendFunc(func(b *uiprogress.Bar) string {
+				return fmt.Sprintf("%s", helper.ShortenStringLeft(_bar.Description, terminalWidth-80))
+			})
+
+			bars[barname] = _bar
+		} else {
+			_bar.Bar.Total++
+		}
+	}
+}
+
+// IncrDone increases to done jobs counter
+func IncrDone(barname string) {
+	if initialized {
+		bars[barname].Bar.Incr()
+		bars[barname].Description = ""
+	}
+}
+
+// DescribeCurrent describes the current job
+func DescribeCurrent(barname, description string) {
+	if initialized {
+		bars[barname].Description = description
+	}
+}

vendor/github.com/gosuri/uilive

@@ -0,0 +1 @@
+Subproject commit ac356e6e42cd31fcef8e6aec13ae9ed6fe87713e

vendor/github.com/gosuri/uiprogress

@@ -0,0 +1 @@
+Subproject commit d0567a9d84a1c40dd7568115ea66f4887bf57b33

vendor/github.com/mattn/go-tty

@@ -0,0 +1 @@
+Subproject commit 5518497423d1a71429a4bea2a4536ef5939c3c27

vendor/golang.org/x/sys

@@ -0,0 +1 @@
+Subproject commit f49334f85ddcf0f08d7fb6dd7363e9e6d6b777eb