From 267d1010bb03aa86f7aefc3b7418df05ea60425f Mon Sep 17 00:00:00 2001 From: Sebastian Frank Date: Mon, 25 Mar 2019 14:01:28 +0100 Subject: [PATCH] -progress cli param for bars --- .gitmodules | 12 +++++ cmd/mark2web/main.go | 6 +++ pkg/filter/image_process.go | 68 +++++++++++++++++++---------- pkg/filter/image_process_test.go | 3 +- pkg/helper/string.go | 13 ++++++ pkg/jobm/jobmanager.go | 30 +++++++------ pkg/mark2web/compress.go | 2 +- pkg/progress/bar.go | 66 ++++++++++++++++++++++++++++ vendor/github.com/gosuri/uilive | 1 + vendor/github.com/gosuri/uiprogress | 1 + vendor/github.com/mattn/go-tty | 1 + vendor/golang.org/x/sys | 1 + 12 files changed, 166 insertions(+), 38 deletions(-) create mode 100644 pkg/helper/string.go create mode 100644 pkg/progress/bar.go create mode 160000 vendor/github.com/gosuri/uilive create mode 160000 vendor/github.com/gosuri/uiprogress create mode 160000 vendor/github.com/mattn/go-tty create mode 160000 vendor/golang.org/x/sys diff --git a/.gitmodules b/.gitmodules index e159477..1f68c4f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -79,3 +79,15 @@ [submodule "vendor/github.com/itchio/go-brotli"] path = vendor/github.com/itchio/go-brotli url = https://github.com/itchio/go-brotli +[submodule "vendor/github.com/gosuri/uiprogress"] + path = vendor/github.com/gosuri/uiprogress + url = https://github.com/gosuri/uiprogress +[submodule "vendor/github.com/gosuri/uilive"] + path = vendor/github.com/gosuri/uilive + url = https://github.com/gosuri/uilive +[submodule "vendor/github.com/mattn/go-tty"] + path = vendor/github.com/mattn/go-tty + url = https://github.com/mattn/go-tty +[submodule "vendor/golang.org/x/sys"] + path = vendor/golang.org/x/sys + url = https://go.googlesource.com/sys diff --git a/cmd/mark2web/main.go b/cmd/mark2web/main.go index b498575..cf2fba1 100644 --- a/cmd/mark2web/main.go +++ b/cmd/mark2web/main.go @@ -9,6 +9,7 @@ import ( "gitbase.de/apairon/mark2web/pkg/filter" "gitbase.de/apairon/mark2web/pkg/logger" "gitbase.de/apairon/mark2web/pkg/mark2web" + "gitbase.de/apairon/mark2web/pkg/progress" ) var ( @@ -26,6 +27,7 @@ func main() { createOutDir := flag.Bool("create", false, "create output directory if not existing") //clearOutDir := flag.Bool("clear", false, "clear output directory before generating website") logLevel := flag.String("logLevel", "notice", "log level: debug, info, notice, warning, error") + progressBars := flag.Bool("progress", false, "show progress bars for jobs") version := flag.Bool("version", false, "print version of this executable") flag.Parse() @@ -43,6 +45,10 @@ func main() { } logger.SetLogLevel(level) + if progressBars != nil && *progressBars { + progress.Init() + } + if inDir == nil || *inDir == "" { logger.Log.Panic("input directory not specified") } diff --git a/pkg/filter/image_process.go b/pkg/filter/image_process.go index a80d0a9..d466f39 100644 --- a/pkg/filter/image_process.go +++ b/pkg/filter/image_process.go @@ -5,6 +5,9 @@ import ( "errors" "fmt" "image" + "image/gif" + "image/jpeg" + "image/png" "net/http" "net/url" "os" @@ -107,27 +110,18 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, * p.Quality, ) if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") { - // remote file - img, p.Format, err = getImageFromURL(imgSource) - if err != nil { - return nil, &pongo2.Error{ - Sender: "filter:image_resize", - OrigError: fmt.Errorf("could not open image '%s': %s", imgSource, err), - } - } // build filename if p.Filename == "" { var fBase string if u, _ := url.Parse(imgSource); u != nil { - fBase = strings.Split(path.Base(u.Path), ".")[0] + fBase = path.Base(u.Path) } p.Filename = fmt.Sprintf( - "%s_%x_%s.%s", + "%s_%x_%s", filePrefix, md5.Sum([]byte(imgSource)), fBase, - p.Format, ) } } else { @@ -173,13 +167,12 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, * Function: func() { logger.Log.Noticef("processing image from %s to %s", imgSource, imgTarget) if strings.HasPrefix(imgSource, "http://") || strings.HasPrefix(imgSource, "https://") { - // webrequest before finding target filename, because of file format in filename + // remote file + img, p.Format, err = getImageFromURL(imgSource) } else { img, err = imaging.Open(imgSource, imaging.AutoOrientation(true)) - if err != nil { - logger.Log.Panicf("filter:image_resize, could not open image '%s': %s", imgSource, err) - } } + logger.Eexit(err, "filter:image_resize, could not open image '%s'", imgSource) switch p.Process { case "resize": @@ -217,18 +210,47 @@ func ImageProcessFilter(in *pongo2.Value, param *pongo2.Value) (*pongo2.Value, * logger.Log.Panicf("filter:image_resize, invalid p parameter '%s'", p.Process) } - var encodeOptions = make([]imaging.EncodeOption, 0) - if p.Quality > 0 { - encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality)) + if p.Format == "" { + switch strings.ToLower(path.Ext(imgTarget)) { + case ".jpg", ".jpeg", ".gif", ".png": + var encodeOptions = make([]imaging.EncodeOption, 0) + if p.Quality > 0 { + encodeOptions = append(encodeOptions, imaging.JPEGQuality(p.Quality)) + } + + err = imaging.Save(img, imgTarget, encodeOptions...) + logger.Eerr(err, "filter:image_resize, could save image '%s'", imgTarget) + default: + logger.E("filter:image_resize, invalid filename extension for image: %s", imgTarget) + os.Exit(1) + } + } else { + out, err := os.Create(imgTarget) + defer out.Close() + + logger.Eerr(err, "filter:image_resize, could not create image file '%s'", imgTarget) + switch p.Format { + case "jpeg", "jpg": + var jpegOpt *jpeg.Options + if p.Quality > 0 { + jpegOpt = &jpeg.Options{ + Quality: p.Quality, + } + } + err = jpeg.Encode(out, img, jpegOpt) + case "png": + err = png.Encode(out, img) + case "gif": + err = gif.Encode(out, img, nil) + default: + logger.E("filter:image_resize, unknown format '%s' for '%s'", p.Format, imgSource) + } + logger.Eerr(err, "filter:image_resize, could not encode image file '%s'", imgTarget) } - err = imaging.Save(img, imgTarget, encodeOptions...) - if err != nil { - logger.Log.Panicf("filter:image_resize, could save image '%s': %s", imgTarget, err) - } logger.Log.Noticef("finished image: %s", imgTarget) }, - Description: "process image " + imgSource, + Description: imgSource, Category: "image process", }) } diff --git a/pkg/filter/image_process_test.go b/pkg/filter/image_process_test.go index 1608a25..633a128 100644 --- a/pkg/filter/image_process_test.go +++ b/pkg/filter/image_process_test.go @@ -4,6 +4,7 @@ import ( "os" "testing" + "gitbase.de/apairon/mark2web/pkg/jobm" "gitbase.de/apairon/mark2web/pkg/mark2web" "github.com/flosch/pongo2" @@ -19,7 +20,7 @@ func TestImageProcessFilter(t *testing.T) { } // we want to check files after function calls, so no multithreading - mark2web.SetNumCPU(1) + jobm.SetNumCPU(1) mark2web.Config.Directories.Input = "../../test/in" mark2web.Config.Directories.Output = "../../test/out" diff --git a/pkg/helper/string.go b/pkg/helper/string.go new file mode 100644 index 0000000..a51be5a --- /dev/null +++ b/pkg/helper/string.go @@ -0,0 +1,13 @@ +package helper + +// ShortenStringLeft shortens a string +func ShortenStringLeft(str string, num int) string { + tstr := str + if len(str) > num { + if num > 3 { + num -= 3 + } + tstr = "..." + str[len(str)-num:len(str)] + } + return tstr +} diff --git a/pkg/jobm/jobmanager.go b/pkg/jobm/jobmanager.go index 6c62c4e..de1fd73 100644 --- a/pkg/jobm/jobmanager.go +++ b/pkg/jobm/jobmanager.go @@ -3,8 +3,9 @@ package jobm import ( "runtime" "sync" + "time" - "gitbase.de/apairon/mark2web/pkg/logger" + "gitbase.de/apairon/mark2web/pkg/progress" ) var wg sync.WaitGroup @@ -17,37 +18,40 @@ type Job struct { Category string } -var jobChan = make(chan Job) +var jobChan = make(chan []Job) -func worker(jobChan <-chan Job) { +func worker(jobChan <-chan []Job) { defer wg.Done() - for job := range jobChan { - job.Function() + for jobs := range jobChan { + for _, job := range jobs { + progress.DescribeCurrent(job.Category, job.Description) + job.Function() + progress.IncrDone(job.Category) + } } } func init() { - logger.Log.Infof("number of CPU core: %d", numCPU) + //logger.Log.Infof("number of CPU core: %d", numCPU) // one core for main thread - for i := 0; i < (numCPU - 1); i++ { + for i := 0; i < numCPU; i++ { wg.Add(1) go worker(jobChan) } } // Enqueue enqueues a job to the job queue -func Enqueue(job Job) { - if numCPU <= 1 { - // no threading - job.Function() - return +func Enqueue(jobs ...Job) { + for _, job := range jobs { + progress.IncrTotal(job.Category) } - jobChan <- job + jobChan <- jobs } // Wait will wait for all jobs to finish func Wait() { + time.Sleep(time.Millisecond * 300) close(jobChan) wg.Wait() } diff --git a/pkg/mark2web/compress.go b/pkg/mark2web/compress.go index 4b2451a..f764885 100644 --- a/pkg/mark2web/compress.go +++ b/pkg/mark2web/compress.go @@ -61,7 +61,7 @@ func handleCompression(filename string, content []byte) { } } }, - Description: "compress " + filename, + Description: filename, Category: "compress", }) } diff --git a/pkg/progress/bar.go b/pkg/progress/bar.go new file mode 100644 index 0000000..c57fe58 --- /dev/null +++ b/pkg/progress/bar.go @@ -0,0 +1,66 @@ +package progress + +import ( + "fmt" + + "gitbase.de/apairon/mark2web/pkg/helper" + "github.com/gosuri/uiprogress" + "github.com/mattn/go-tty" +) + +type bar struct { + Bar *uiprogress.Bar + Description string +} + +var bars = make(map[string]*bar) +var initialized = false +var terminalWidth = 80 + +// Init initializes the bar drawing +func Init() { + if t, err := tty.Open(); err == nil && t != nil { + terminalWidth, _, _ = t.Size() + t.Close() + } + uiprogress.Start() // start rendering + initialized = true +} + +// IncrTotal increases the total jobs for the bar +func IncrTotal(barname string) { + if initialized { + _bar := bars[barname] + if _bar == nil { + _bar = new(bar) + _bar.Bar = uiprogress.AddBar(1) + _bar.Bar.Width = 25 + + _bar.Bar.PrependFunc(func(b *uiprogress.Bar) string { + return fmt.Sprintf("%15s: %3d/%3d", helper.ShortenStringLeft(barname, 15), b.Current(), b.Total) + }) + _bar.Bar.AppendFunc(func(b *uiprogress.Bar) string { + return fmt.Sprintf("%s", helper.ShortenStringLeft(_bar.Description, terminalWidth-80)) + }) + + bars[barname] = _bar + } else { + _bar.Bar.Total++ + } + } +} + +// IncrDone increases to done jobs counter +func IncrDone(barname string) { + if initialized { + bars[barname].Bar.Incr() + bars[barname].Description = "" + } +} + +// DescribeCurrent describes the current job +func DescribeCurrent(barname, description string) { + if initialized { + bars[barname].Description = description + } +} diff --git a/vendor/github.com/gosuri/uilive b/vendor/github.com/gosuri/uilive new file mode 160000 index 0000000..ac356e6 --- /dev/null +++ b/vendor/github.com/gosuri/uilive @@ -0,0 +1 @@ +Subproject commit ac356e6e42cd31fcef8e6aec13ae9ed6fe87713e diff --git a/vendor/github.com/gosuri/uiprogress b/vendor/github.com/gosuri/uiprogress new file mode 160000 index 0000000..d0567a9 --- /dev/null +++ b/vendor/github.com/gosuri/uiprogress @@ -0,0 +1 @@ +Subproject commit d0567a9d84a1c40dd7568115ea66f4887bf57b33 diff --git a/vendor/github.com/mattn/go-tty b/vendor/github.com/mattn/go-tty new file mode 160000 index 0000000..5518497 --- /dev/null +++ b/vendor/github.com/mattn/go-tty @@ -0,0 +1 @@ +Subproject commit 5518497423d1a71429a4bea2a4536ef5939c3c27 diff --git a/vendor/golang.org/x/sys b/vendor/golang.org/x/sys new file mode 160000 index 0000000..f49334f --- /dev/null +++ b/vendor/golang.org/x/sys @@ -0,0 +1 @@ +Subproject commit f49334f85ddcf0f08d7fb6dd7363e9e6d6b777eb