ssr via fetch

.env

@@ -5,4 +5,4 @@ UID=100
 GID=101
 RELEASE_ORG_SLUG=webmakers-gmbh
 RELEASE_PROJECT_SLUG=tibi_starter
-#START_SCRIPT=:ssr
+START_SCRIPT=:ssr

.vscode/settings.json

@@ -18,11 +18,14 @@
         {
             "match": "/api/.*(\\.ya?ml|js|env)$",
             "isAsync": false,
-            "cmd": "docker compose -p tibi-docs restart tibiserver",
+            "cmd": "cd ${currentWorkspace} && docker compose -f docker-compose-local.yml restart tibiserver",
             "event": "onFileChange"
         }
     ],
     "i18n-ally.localesPaths": ["frontend/locales"],
     "i18n-ally.sourceLanguage": "de",
-    "i18n-ally.keystyle": "nested"
+    "i18n-ally.keystyle": "nested",
+    "[svelte]": {
+        "editor.defaultFormatter": "svelte.svelte-vscode"
+    }
 }

api/collections/ssr.yml

@@ -18,28 +18,22 @@ meta:
           columns:
               - id
               - insertTime
-              - path
+              - source: path
+                filter: true
 
 permissions:
     public:
         methods:
-            get: false
-            post: false
-            put: false
-            delete: false
-    user:
-        methods:
-            get: false
-            post: false
-            put: false
-            delete: false
-    "token:${SSR_TOKEN}":
-        methods:
-            # only via url=
             get: true
             post: true
             put: false
             delete: false
+    user:
+        methods:
+            get: true
+            post: false
+            put: false
+            delete: true
 
 hooks:
     get:
@@ -51,7 +45,6 @@ hooks:
             type: javascript
             file: hooks/ssr/post_bind.js
 
-# we only need hooks
 fields:
     - name: path
       type: string

api/hooks/config.js

@@ -1,8 +1,9 @@
-const apiSsrBaseURL = "http://localhost:8080/api/v1/_/allkids_erfurt"
+const apiSsrBaseURL =
+    "http://localhost:" + context.config.server().api.port + "/api/v1/_/" + context.api().namespace + "/"
 
 module.exports = {
     apiSsrBaseURL,
-    ssrValidatePath: function (path) {
+    ssrValidatePath: function (/** @type {string} */ path) {
         // validate if path ssr rendering is ok, -1 = NOTFOUND, 0 = NO SSR, 1 = SSR
         // pe. use context.readCollection("product", {filter: {path: path}}) ... to validate dynamic urls
 
@@ -11,7 +12,6 @@ module.exports = {
 
         // // all other sites are in db
         //path = path?.replace(/^\//, "")
-        console.log("PATH:", path)
         const resp = context.db.find("content", {
             filter: {
                 $and: [{ path }],
@@ -19,7 +19,6 @@ module.exports = {
 
             selector: { _id: 1 },
         })
-        console.log("RESP:", resp?.length)
         if (resp && resp.length) {
             return 1
         }

api/hooks/lib/ssr-server.js

@@ -1,4 +1,4 @@
-const { apiSsrBaseURL, ssrPublishCheckCollections } = require("../config")
+const { apiSsrBaseURL } = require("../config")
 
 /**
  * api request via server, cache result in context.ssrCache
@@ -6,21 +6,24 @@ const { apiSsrBaseURL, ssrPublishCheckCollections } = require("../config")
  *
  * @param {string} cacheKey
  * @param {string} endpoint
+ * @param {string} query
  * @param {ApiOptions} options
- * @returns {ApiResult}
+ * @returns {ApiResult<any>}
  */
 function ssrRequest(cacheKey, endpoint, query, options) {
     let url = endpoint + (query ? "?" + query : "")
 
     // console.log("############ FETCHING ", apiSsrBaseURL + url)
 
-    const response = context.http.fetch(apiSsrBaseURL + "/" + url, {
+    const response = context.http.fetch(apiSsrBaseURL + url, {
         method: options.method,
         headers: options.headers,
     })
 
+    // console.log(JSON.stringify(response.headers, null, 2))
+
     const json = response.body.json()
-    const count = parseInt(response.headers["x-results-count"] || "0")
+    const count = parseInt(response.headers["X-Results-Count"] || "0")
 
     // json is go data structure and incompatible with js, so we need to convert it
     const r = { data: JSON.parse(JSON.stringify(json)), count: count }

api/hooks/lib/ssr.js

@@ -3,6 +3,7 @@ const { apiClientBaseURL } = require("../config-client")
 /**
  * convert object to string
  * @param {any} obj object
+ * @returns {string} string
  */
 function obj2str(obj) {
     if (Array.isArray(obj)) {
@@ -32,70 +33,6 @@ function obj2str(obj) {
     if (obj) return obj
 }
 
-// fetch polyfill
-// [MIT License](LICENSE.md) © [Jason Miller](https://jasonformat.com/)
-const _f = function (
-    /** @type {string | URL} */ url,
-    /** @type {{ method?: any; credentials?: any; headers?: any; body?: any; }} */ options
-) {
-    if (typeof XMLHttpRequest === "undefined") {
-        return Promise.resolve(null)
-    }
-
-    options = options || {}
-    return new Promise((resolve, reject) => {
-        const request = new XMLHttpRequest()
-        const keys = []
-        const all = []
-        const headers = {}
-
-        const response = () => ({
-            ok: ((request.status / 100) | 0) == 2, // 200-299
-            statusText: request.statusText,
-            status: request.status,
-            url: request.responseURL,
-            text: () => Promise.resolve(request.responseText),
-            json: () => Promise.resolve(request.responseText).then(JSON.parse),
-            blob: () => Promise.resolve(new Blob([request.response])),
-            clone: response,
-            headers: {
-                // @ts-ignore
-                keys: () => keys,
-                // @ts-ignore
-                entries: () => all,
-                get: (n) => headers[n.toLowerCase()],
-                has: (n) => n.toLowerCase() in headers,
-            },
-        })
-
-        request.open(options.method || "get", url, true)
-
-        request.onload = () => {
-            request
-                .getAllResponseHeaders()
-                // @ts-ignore
-                .replace(/^(.*?):[^\S\n]*([\s\S]*?)$/gm, (m, key, value) => {
-                    keys.push((key = key.toLowerCase()))
-                    all.push([key, value])
-                    headers[key] = headers[key] ? `${headers[key]},${value}` : value
-                })
-            resolve(response())
-        }
-
-        request.onerror = reject
-
-        request.withCredentials = options.credentials == "include"
-
-        for (const i in options.headers) {
-            request.setRequestHeader(i, options.headers[i])
-        }
-
-        request.send(options.body || null)
-    })
-}
-
-const _fetch = typeof fetch === "undefined" ? (typeof window === "undefined" ? _f : window.fetch || _f) : fetch
-
 /**
  * api request via client or server
  * server function ssrRequest is called via context.ssrRequest, binded in ssr hook
@@ -103,27 +40,29 @@ const _fetch = typeof fetch === "undefined" ? (typeof window === "undefined" ? _
  * @param {string} endpoint
  * @param {ApiOptions} options
  * @param {any} body
+ * @param {import("../../../frontend/src/sentry")} sentry
+ * @param {typeof fetch} _fetch
  * @returns {Promise<ApiResult<any>>}
  */
-function apiRequest(endpoint, options, body) {
+function apiRequest(endpoint, options, body, sentry, _fetch) {
     // TODO cache only for GET
 
     // first check cache if on client
     const cacheKey = obj2str({ endpoint: endpoint, options: options })
-    options.method = options?.method || "GET"
+    console.log("SSR CHECK", cacheKey)
+
+    let method = options?.method || "GET"
 
     // @ts-ignore
-    if (typeof window !== "undefined" && window.__SSR_CACHE__ && options?.method === "GET") {
+    if (typeof window !== "undefined" && window.__SSR_CACHE__ && method === "GET") {
         // @ts-ignore
         const cache = window.__SSR_CACHE__[cacheKey]
-        console.log("SSR:", cacheKey, cache)
+        console.log("SSR HIT:", cacheKey, cache)
         if (cache) {
             return Promise.resolve(cache)
         }
     }
 
-    let method = options?.method || "GET"
-
     let query = "&count=1"
     if (options?.filter) query += "&filter=" + encodeURIComponent(JSON.stringify(options.filter))
     if (options?.sort) query += "&sort=" + options.sort + "&sort=_id"
@@ -138,6 +77,7 @@ function apiRequest(endpoint, options, body) {
         })
     }
 
+    /** @type {{[key: string]: string}} */
     let headers = {
         "Content-Type": "application/json",
     }
@@ -153,8 +93,19 @@ function apiRequest(endpoint, options, body) {
         return d
     } else {
         // client
-        let url = endpoint + (query ? "?" + query : "")
-        console.log("URL:", url)
+        let url = apiClientBaseURL + endpoint + (query ? "?" + query : "")
+
+        const span = sentry.currentTransaction()?.startChild({
+            op: "fetch",
+            description: method + " " + url,
+            data: Object.assign({}, options, { url }),
+        })
+        const trace_id = span?.toTraceparent()
+        if (trace_id) {
+            headers["sentry-trace"] = trace_id
+        }
+
+        /** @type {{[key: string]: any}} */
         const requestOptions = {
             method,
             mode: "cors",
@@ -165,7 +116,7 @@ function apiRequest(endpoint, options, body) {
             requestOptions.body = JSON.stringify(body)
         }
 
-        return _fetch(apiClientBaseURL + url, requestOptions).then((response) => {
+        const response = _fetch(url, requestOptions).then((response) => {
             return response?.json().then((json) => {
                 if (response?.status < 200 || response?.status >= 400) {
                     return Promise.reject({ response, data: json })
@@ -173,6 +124,11 @@ function apiRequest(endpoint, options, body) {
                 return Promise.resolve({ data: json || null, count: response.headers?.get("x-results-count") || 0 })
             })
         })
+
+        span?.end()
+
+        // @ts-ignore
+        return response
     }
 }
 

api/hooks/lib/utils.js

@@ -41,10 +41,12 @@ var { LIGHTHOUSE_TOKEN } = require("../config")
 
 /**
  * clear SSR cache
+ * @returns {number}
  */
 function clearSSRCache() {
     var info = context.db.deleteMany("ssr", {})
     context.response.header("X-SSR-Cleared", info.removed)
+    return info.removed
 }
 
 function calculateAverageDynamically(dbObjs) {

api/hooks/ssr/get_read.js

@@ -1,29 +1,38 @@
-// TODO: add query string functionality to cache
+const utils = require("../lib/utils")
+
+const { release } = require("../config-client")
 
 const { ssrValidatePath } = require("../config")
-const { log } = require("../lib/utils")
-const { ssrRequest } = require("../lib/ssr-server.js")
-
+const { ssrRequest } = require("../lib/ssr-server")
 ;(function () {
-    /** @type {HookResponse} */
-    let response = null
+    var request = context.request()
 
-    const request = context.request()
-    let url = request.query("url")
+    var trackingCall = request.header("x-ssr-skip")
+    if (trackingCall) {
+        // skip tracking
+        throw {
+            status: parseInt(trackingCall),
+            html: "",
+            log: false,
+        }
+    }
+
+    let url = request.query("url") || request.header("x-ssr-url") || "/"
     const noCache = request.query("noCache")
 
-    // add sentry trace id to head
     const trace_id = context.debug.sentryTraceId()
+    /**
+     * @param {string} content
+     */
     function addSentryTrace(content) {
         return content.replace("</head>", '<meta name="sentry-trace" content="' + trace_id + '" /></head>')
     }
-    context.response.header("sentry-trace", trace_id)
 
-    if (url) {
-        // comment will be printed to html later
-        let comment = ""
-        /** @type {Date} */ // @ts-ignore
-        context.ssrCacheValidUntil = null
+    context.response.header("sentry-trace", trace_id)
+    const auth = context.user.auth()
+    if (auth && auth.role == 0) {
+    } else if (url) {
+        var comment = ""
 
         url = url.split("?")[0]
         comment += "url: " + url
@@ -31,19 +40,19 @@ const { ssrRequest } = require("../lib/ssr-server.js")
         if (url && url.length > 1) {
             url = url.replace(/\/$/, "")
         }
-        if (url == "/noindex" || !url) {
+        if (url == "/index" || !url) {
             url = "/" // see .htaccess
         }
 
-        // check if url is in cache
-        /** @type {Ssr[]} */ // @ts-ignore
-        const cache =
+        function useCache(/** @type {string} */ _url) {
+            var cache =
                 !noCache &&
                 context.db.find("ssr", {
                     filter: {
-                    path: url,
+                        path: _url,
                     },
                 })
+
             if (cache && cache.length) {
                 // use cache
                 context.response.header("X-SSR-Cache", "true")
@@ -53,18 +62,33 @@ const { ssrRequest } = require("../lib/ssr-server.js")
                     html: addSentryTrace(cache[0].content),
                 }
             }
+        }
 
         // validate url
-        let status = 200
+        var status = 200
 
-        let pNorender = false
-        let pNotfound = false
+        var pNorender = false
+        var pNotfound = false
 
         const pR = ssrValidatePath(url)
-        if (pR < 0) {
+        if (pR === -1) {
             pNotfound = true
+            comment += ", notFound"
         } else if (!pR) {
             pNorender = true
+            comment += ", noRender"
+        } else if (typeof pR === "string") {
+            url = pR
+            comment += ", cache url: " + url
+        }
+
+        if (noCache) {
+            comment += ", noCache"
+        }
+
+        if (!pNorender && !pNotfound) {
+            // check if we have a cache
+            useCache(url)
         }
 
         let head = ""
@@ -73,7 +97,7 @@ const { ssrRequest } = require("../lib/ssr-server.js")
 
         comment += ", path: " + url
 
-        let cacheIt = false
+        var cacheIt = false
         if (pNorender) {
             html = "<!-- NO SSR RENDERING -->"
         } else if (pNotfound) {
@@ -85,20 +109,17 @@ const { ssrRequest } = require("../lib/ssr-server.js")
             // @ts-ignore
             context.ssrRequest = ssrRequest
 
-            // try rendering, if error output plain html
             try {
-                // include App.svelte and render it
+                // if error, output plain html without prerendering
                 // @ts-ignore
-
-                // console.log("####### RENDERING ", url)
                 const app = require("../lib/app.server")
+
                 const rendered = app.default.render({
                     url: url,
                 })
                 head = rendered.head
                 html = rendered.html
 
-                // add ssrCache to head, cache is built in ssr.js/apiRequest
                 head +=
                     "\n\n" +
                     "<script>window.__SSR_CACHE__ = " +
@@ -109,50 +130,49 @@ const { ssrRequest } = require("../lib/ssr-server.js")
                 // status from webapp
                 // @ts-ignore
                 if (context.is404) {
-                    // console.log("########## 404")
                     status = 404
                 } else {
                     cacheIt = true
                 }
-            } catch (e) {
-                // save error for later insert into html
-                log(e.message)
-                log(e.stack)
+            } catch (/** @type {any} */ e) {
+                utils.log(e.message)
+                utils.log(e.stack)
                 error = "error: " + e.message + "\n\n" + e.stack
+
+                // utils.log(e)
+                // for (var property in e) {
+                //     utils.log(property + ": " + e[property])
+                // }
+                // error = JSON.stringify(e)
             }
         }
 
-        // read html template and replace placeholders
-        let tpl = context.fs.readFile("templates/spa.html")
+        var tpl = context.fs.readFile("templates/spa.html")
         tpl = tpl.replace("<!--HEAD-->", head)
         tpl = tpl.replace("<!--HTML-->", html)
         tpl = tpl.replace("<!--SSR.ERROR-->", error ? "<!--" + error + "-->" : "")
         tpl = tpl.replace("<!--SSR.COMMENT-->", comment ? "<!--" + comment + "-->" : "")
 
-        // save cache if adviced
         if (cacheIt && !noCache) {
+            // save cache
             context.db.create("ssr", {
-                // context.debug.dump("ssr", {
                 path: url,
                 content: tpl,
             })
         }
 
-        // return html
         throw {
             status: status,
             log: false,
             html: addSentryTrace(tpl),
         }
     } else {
-        // only admins are allowed to get without url parameter
-        const auth = context.user.auth()
-        if (!auth || auth.role !== 0) {
+        // only admins are allowed
         throw {
             status: 403,
             message: "invalid auth",
             auth: auth,
-            }
+            release: release,
         }
     }
 })()

api/hooks/ssr/post_bind.js

@@ -1,16 +1,25 @@
-const utils = require("../lib/utils")
+const { release } = require("../config-client")
+var utils = require("../lib/utils")
 
 ;(function () {
     if (context.request().query("clear")) {
-        utils.clearSSRCache()
-        throw {
+        console.log("CLEARING SSR CACHE")
+        var removed = utils.clearSSRCache()
+        var stats = {
             status: 200,
-            message: "cache cleared",
+            message: "ok",
+            removed: removed,
+            release: release,
         }
+
+        utils.log(stats)
+
+        throw stats
     }
 
     throw {
         status: 500,
         message: "ssr is only a dummy collection",
+        release: release,
     }
 })()

babel.config.server.json

@@ -0,0 +1,5 @@
+{
+    "sourceMaps": "inline",
+    "inputSourceMap": true,
+    "plugins": [["@babel/plugin-transform-async-to-generator"]]
+}

esbuild.config.server.js

@@ -5,6 +5,9 @@ config.options.sourcemap = "inline"
 config.options.minify = false
 config.options.platform = "node"
 config.options.format = "cjs"
+// es2015 will transform async/await to generators, but not working with svelte
+// so we need babel to transform async/await to promises
+// config.options.target = "es2015"
 config.options.entryPoints = ["./frontend/src/ssr.ts"]
 config.options.outfile = __dirname + "/_temp/app.server.js"
 config.options.plugins = [
@@ -16,6 +19,11 @@ config.options.plugins = [
             dev: (process.argv?.length > 2 ? process.argv[2] : "build") !== "build",
         },
         preprocess: svelteConfig.preprocess,
+        filterWarnings: (warning) => {
+            // filter out a11y
+            if (warning.code.match(/^a11y/)) return false
+            return true
+        },
     }),
     config.resolvePlugin,
 ]

frontend/src/api.ts

@@ -1,12 +1,77 @@
 import { apiRequest } from "../../api/hooks/lib/ssr"
+import * as sentry from "./sentry"
+
+// fetch polyfill
+// [MIT License](LICENSE.md) © [Jason Miller](https://jasonformat.com/)
+const _f = function (url: string, options?: { [key: string]: any }) {
+    if (typeof XMLHttpRequest === "undefined") {
+        return Promise.resolve(null)
+    }
+
+    options = options || {}
+    return new Promise((resolve, reject) => {
+        const request = new XMLHttpRequest()
+        const keys: string[] = []
+        const all = []
+        const headers = {}
+
+        const response = () => ({
+            ok: ((request.status / 100) | 0) == 2, // 200-299
+            statusText: request.statusText,
+            status: request.status,
+            url: request.responseURL,
+            text: () => Promise.resolve(request.responseText),
+            json: () => Promise.resolve(request.responseText).then(JSON.parse),
+            blob: () => Promise.resolve(new Blob([request.response])),
+            clone: response,
+            headers: {
+                // @ts-ignore
+                keys: () => keys,
+                // @ts-ignore
+                entries: () => all,
+                get: (n) => headers[n.toLowerCase()],
+                has: (n) => n.toLowerCase() in headers,
+            },
+        })
+
+        request.open(options.method || "get", url, true)
+
+        request.onload = () => {
+            request
+                .getAllResponseHeaders()
+                // @ts-ignore
+                .replace(/^(.*?):[^\S\n]*([\s\S]*?)$/gm, (m, key, value) => {
+                    keys.push((key = key.toLowerCase()))
+                    all.push([key, value])
+                    headers[key] = headers[key] ? `${headers[key]},${value}` : value
+                })
+            resolve(response())
+        }
+
+        request.onerror = reject
+
+        request.withCredentials = options.credentials == "include"
+
+        for (const i in options.headers) {
+            request.setRequestHeader(i, options.headers[i])
+        }
+
+        request.send(options.body || null)
+    })
+}
+
+// fetch must be declared after sentry import to get the hijacked fetch
+// @ts-ignore
+export const _fetch: typeof fetch =
+    typeof fetch === "undefined" ? (typeof window === "undefined" ? _f : window.fetch || _f) : fetch
 
 export const api = async <T>(
     endpoint: string,
     options?: ApiOptions,
     body?: any
 ): Promise<{ data: T; count: number } | any> => {
-    let data = await apiRequest(endpoint, options, body)
+    let data = await apiRequest(endpoint, options, body, sentry, _fetch)
     // @ts-ignore
-    console.log(data, "data")
+    // console.log(data, "data")
     return data
 }

frontend/src/lib/components/CookieSet.svelte

@@ -10,6 +10,7 @@
         unten: "flex-end",
     }
 
+    if (typeof window !== "undefined") {
         window.addEventListener("ccAccept", (e) => {
             if ((e as CustomEvent).detail[1] == cookieName) contentShown = true
         })
@@ -35,6 +36,7 @@
 
         // Verwendung
         if (checkCookie(cookieName)) contentShown = true
+    }
 </script>
 
 {#if contentShown}

frontend/src/lib/components/Header.svelte

@@ -4,7 +4,7 @@
 
     let opened = -1
     let openMenu = false
-    $: {
+    $: if (typeof document !== "undefined") {
         if (openMenu) document.body.style.overflow = "hidden"
         else document.body.style.overflow = "auto"
     }

frontend/src/lib/components/pagebuilder/form/Datepicker.svelte

@@ -90,6 +90,7 @@
 <!-- svelte-ignore a11y-click-events-have-key-events -->
 <!-- svelte-ignore a11y-no-static-element-interactions -->
 <div class="container">
+    {#if typeof window !== "undefined"}
         <div
             class="datepicker"
             on:click|preventDefault|stopPropagation
@@ -108,6 +109,7 @@
                 locale="de-DE"
             />
         </div>
+    {/if}
 </div>
 
 <style lang="less">

frontend/src/routes/Page.svelte

@@ -2,14 +2,11 @@
     import { mediaLibrary, pages } from "../lib/store"
     import Pagebuilder from "../lib/components/pagebuilder/Pagebuilder.svelte"
     import { apiBaseURL, baseURL } from "../config"
-    import { onMount } from "svelte"
     import NotFound from "./NotFound.svelte"
     export let path: string = "/"
     export let isHomepage = true
     let page: Content = $pages[path]
     $: page = $pages[path]
-    let personPage = false
-    //test
 </script>
 
 <svelte:head>

package.json

@@ -14,7 +14,7 @@
         "build": "node scripts/esbuild-wrapper.js build",
         "build:admin": "node scripts/esbuild-wrapper.js build esbuild.config.admin.js",
         "build:legacy": "node scripts/esbuild-wrapper.js build esbuild.config.legacy.js && babel _temp/index.js -o _temp/index.babeled.js && esbuild _temp/index.babeled.js --outfile=frontend/dist/index.es5.js --target=es5 --bundle --minify --sourcemap",
-        "build:server": "node scripts/esbuild-wrapper.js build esbuild.config.server.js && babel _temp/app.server.js -o _temp/app.server.babeled.js && esbuild _temp/app.server.babeled.js --outfile=api/hooks/lib/app.server.js --target=es5 --bundle --sourcemap --platform=node",
+        "build:server": "node scripts/esbuild-wrapper.js build esbuild.config.server.js && babel --config-file ./babel.config.server.json _temp/app.server.js -o _temp/app.server.babeled.js && esbuild _temp/app.server.babeled.js --outfile=api/hooks/lib/app.server.js --bundle --sourcemap --platform=node",
         "build:test": "node scripts/esbuild-wrapper.js build esbuild.config.test.js && babel --config-file ./babel.config.test.json _temp/hook.test.js -o _temp/hook.test.babeled.js && esbuild _temp/hook.test.babeled.js --outfile=api/hooks/lib/hook.test.js --target=es5 --bundle --sourcemap --platform=node",
         "upload:sourcemaps": "scripts/upload-sourcemaps.sh"
     },

types/global.d.ts

@@ -2,7 +2,7 @@ interface Ssr {
     id?: string
     path: string
     content: string
-    validUntil: any // go Time
+    // validUntil: any // go Time
 }
 
 interface ApiOptions {