mark2web/pkg/mark2web/collection.go

package mark2web

import (
	"io/ioutil"
	"path"
	"regexp"
	"strings"

	"gitbase.de/apairon/mark2web/pkg/helper"
	"gitbase.de/apairon/mark2web/pkg/logger"
	"gitbase.de/apairon/mark2web/pkg/webrequest"
	"github.com/davecgh/go-spew/spew"
	"github.com/flosch/pongo2/v4"
)

type colCacheEntry struct {
	data     interface{}
	hit      int
	navnames []string
}

var colCache = make(map[string]*colCacheEntry)

func (node *TreeNode) handleCollections() {
	collections := append(node.Config.Collections, node.Config.This.Collections...)
	for _, colConfig := range collections {
		if colConfig.Name == nil || *colConfig.Name == "" {
			logger.Exit("missing Name in collection config in '%s'", node.InputPath)
		}
		if (colConfig.URL == nil || *colConfig.URL == "") &&
			(colConfig.Directory == nil) {
			logger.Exit("missing URL and Directory in collection config in '%s'", node.InputPath)
		}

		if node.ColMap == nil {
			node.ColMap = make(helper.MapString)
		}
		ctx := NewContext()
		ctx["This"] = node.Config.This
		ctx["Data"] = node.Config.Data

		var colData interface{}

		errSrcText := ""
		cacheKey := ""

		if colConfig.URL != nil {
			url, err := pongo2.RenderTemplateString(*colConfig.URL, ctx)
			logger.Eexit(err, "invalid template string for Collection Element.URL in '%s'", node.InputPath)

			errSrcText = "URL " + url
			cacheKey = url

			if cacheEntry, ok := colCache[url]; ok {
				colData = cacheEntry.data
				cacheEntry.hit++
			} else {
				logger.N("reading collection from: %s", errSrcText)
				colData = webrequest.GetJSON(url)
				colCache[url] = &colCacheEntry{
					data:     colData,
					navnames: make([]string, 0),
				}
			}

		} else {
			path := node.ResolveInputPath(colConfig.Directory.Path)
			errSrcText = "DIR " + path

			logger.N("reading collection from: %s", errSrcText)
			d, err := ioutil.ReadDir(path)
			logger.Eexit(err, "could not read directory '%s'", path)

			mStr := "."
			if colConfig.Directory.MatchFilename != "" {
				mStr = colConfig.Directory.MatchFilename
			}
			matcher, err := regexp.Compile(mStr)
			logger.Eexit(err, "could not compile regex for MatchFilename '%s' in '%s'", mStr, path)

			if colConfig.Directory.ReverseOrder {
				for i := len(d)/2 - 1; i >= 0; i-- {
					opp := len(d) - 1 - i
					d[i], d[opp] = d[opp], d[i]
				}
			}

			fcolData := make([]pongo2.Context, 0)
			for _, fh := range d {
				if !fh.IsDir() && matcher.MatchString(fh.Name()) {
					inFile := path + "/" + fh.Name()
					md, err := ioutil.ReadFile(inFile)
					logger.Eexit(err, "could not read file '%s'", inFile)

					_, ctx := node.processMarkdownWithHeader(md, inFile)
					(*ctx)["FilenameMatch"] = helper.GetRegexpParams(matcher, fh.Name())
					fcolData = append(fcolData, *ctx)
				}
			}

			colData = fcolData
		}

		node.ColMap[*colConfig.Name] = colData

		if navT := colConfig.NavTemplate; navT != nil {
			var entries []interface{}
			var ok bool
			if navT.EntriesAttribute != "" {
				var colDataMap map[string]interface{}
				if colDataMap, ok = colData.(map[string]interface{}); ok {
					entries, ok = colDataMap[navT.EntriesAttribute].([]interface{})
					if !ok {
						logger.D(spew.Sdump(colDataMap))
						logger.Exit("invalid json data in [%s] from '%s' for entries", navT.EntriesAttribute, errSrcText)
					}
				}
			} else {
				entries, ok = colData.([]interface{})
			}
			if !ok {
				logger.D(spew.Sdump(colData))
				logger.Exit("invalid json data from '%s', need array of objects for entries or object with configured NavTemplate.EntriesAttribute", errSrcText)
			}

			// build navigation with detail sites
			for idx, colEl := range entries {
				ctxE := make(pongo2.Context)
				err := helper.Merge(&ctxE, ctx)
				logger.Eexit(err, "could not merge context in '%s'", node.InputPath)

				var jsonCtx map[string]interface{}
				if jsonCtx, ok = colEl.(map[string]interface{}); !ok {
					logger.D(spew.Sdump(colEl))
					logger.Exit("no json object for entry index %d from '%s'", idx, errSrcText)
				}
				err = helper.Merge(&ctxE, pongo2.Context(jsonCtx))
				logger.Eexit(err, "could not merge context in '%s'", node.InputPath)

				tpl := ""
				if navT.Template != "" {
					tpl, err = pongo2.RenderTemplateString(navT.Template, ctxE)
					logger.Eexit(err, "invalid template string for NavTemplate.Template in '%s'", node.InputPath)
				}
				if tpl == "" {
					tpl = *node.Config.Template
				}

				dataKey := ""
				if navT.DataKey != "" {
					dataKey, err = pongo2.RenderTemplateString(navT.DataKey, ctxE)
					logger.Eexit(err, "invalid template string for NavTemplate.DataKey in '%s'", node.InputPath)
				}

				goTo, err := pongo2.RenderTemplateString(navT.GoTo, ctxE)
				logger.Eexit(err, "invalid template string for NavTemplate.GoTo in '%s'", node.InputPath)

				goTo = strings.Trim(goTo, "/")
				goTo = path.Clean(goTo)

				if strings.Contains(goTo, "..") {
					logger.Exit("going back via .. in NavTemplate.GoTo forbidden in collection config in '%s': %s", node.InputPath, goTo)
				}
				if goTo == "." {
					logger.Exit("invalid config '.' for NavTemplate.GoTo in collection config in '%s'", node.InputPath)
				}
				if goTo == "" {
					logger.Exit("missing NavTemplate.GoTo in collection config in '%s'", node.InputPath)
				}

				navname := ""
				if navT.Navname != "" {
					navname, err = pongo2.RenderTemplateString(navT.Navname, ctxE)
					logger.Eexit(err, "invalid template string for NavTemplate.Navname in '%s'", node.InputPath)
				}
				body := ""
				if navT.Body != "" {
					body, err = pongo2.RenderTemplateString(navT.Body, ctxE)
					logger.Eexit(err, "invalid template string for NavTemplate.Body in '%s'", node.InputPath)
				}

				if l := len(colCache[cacheKey].navnames); colCache[cacheKey].hit > 1 &&
					l > 0 &&
					navname == colCache[cacheKey].navnames[l-1] {
					// navname before used same url, so recursion loop
					logger.Exit("collection request loop detected for in '%s' for : %s", node.InputPath, errSrcText)
				}

				colCache[cacheKey].navnames = append(colCache[cacheKey].navnames, navname)

				node.addSubNode(tpl, goTo, navname, colEl, dataKey, body, navT.Hidden)
			}
		}
	}

}