ssr via fetch
Some checks failed
deploy to production / deploy (push) Failing after 52s

This commit is contained in:
Sebastian Frank 2024-04-10 08:26:40 +00:00
parent 6f1ab474fc
commit 1ebd1424e5
Signed by: apairon
SSH Key Fingerprint: SHA256:lYVOnGlR42QHj7wuqfFgGw8cKbfyZUpzeRDGVBBAHQU
18 changed files with 275 additions and 211 deletions

2
.env
View File

@ -5,4 +5,4 @@ UID=100
GID=101
RELEASE_ORG_SLUG=webmakers-gmbh
RELEASE_PROJECT_SLUG=tibi_starter
#START_SCRIPT=:ssr
START_SCRIPT=:ssr

View File

@ -18,11 +18,14 @@
{
"match": "/api/.*(\\.ya?ml|js|env)$",
"isAsync": false,
"cmd": "docker compose -p tibi-docs restart tibiserver",
"cmd": "cd ${currentWorkspace} && docker compose -f docker-compose-local.yml restart tibiserver",
"event": "onFileChange"
}
],
"i18n-ally.localesPaths": ["frontend/locales"],
"i18n-ally.sourceLanguage": "de",
"i18n-ally.keystyle": "nested"
"i18n-ally.keystyle": "nested",
"[svelte]": {
"editor.defaultFormatter": "svelte.svelte-vscode"
}
}

View File

@ -18,28 +18,22 @@ meta:
columns:
- id
- insertTime
- path
- source: path
filter: true
permissions:
public:
methods:
get: false
post: false
put: false
delete: false
user:
methods:
get: false
post: false
put: false
delete: false
"token:${SSR_TOKEN}":
methods:
# only via url=
get: true
post: true
put: false
delete: false
user:
methods:
get: true
post: false
put: false
delete: true
hooks:
get:
@ -51,7 +45,6 @@ hooks:
type: javascript
file: hooks/ssr/post_bind.js
# we only need hooks
fields:
- name: path
type: string

View File

@ -1,8 +1,9 @@
const apiSsrBaseURL = "http://localhost:8080/api/v1/_/allkids_erfurt"
const apiSsrBaseURL =
"http://localhost:" + context.config.server().api.port + "/api/v1/_/" + context.api().namespace + "/"
module.exports = {
apiSsrBaseURL,
ssrValidatePath: function (path) {
ssrValidatePath: function (/** @type {string} */ path) {
// validate if path ssr rendering is ok, -1 = NOTFOUND, 0 = NO SSR, 1 = SSR
// pe. use context.readCollection("product", {filter: {path: path}}) ... to validate dynamic urls
@ -11,7 +12,6 @@ module.exports = {
// // all other sites are in db
//path = path?.replace(/^\//, "")
console.log("PATH:", path)
const resp = context.db.find("content", {
filter: {
$and: [{ path }],
@ -19,7 +19,6 @@ module.exports = {
selector: { _id: 1 },
})
console.log("RESP:", resp?.length)
if (resp && resp.length) {
return 1
}

View File

@ -1,4 +1,4 @@
const { apiSsrBaseURL, ssrPublishCheckCollections } = require("../config")
const { apiSsrBaseURL } = require("../config")
/**
* api request via server, cache result in context.ssrCache
@ -6,21 +6,24 @@ const { apiSsrBaseURL, ssrPublishCheckCollections } = require("../config")
*
* @param {string} cacheKey
* @param {string} endpoint
* @param {string} query
* @param {ApiOptions} options
* @returns {ApiResult}
* @returns {ApiResult<any>}
*/
function ssrRequest(cacheKey, endpoint, query, options) {
let url = endpoint + (query ? "?" + query : "")
// console.log("############ FETCHING ", apiSsrBaseURL + url)
const response = context.http.fetch(apiSsrBaseURL + "/" + url, {
const response = context.http.fetch(apiSsrBaseURL + url, {
method: options.method,
headers: options.headers,
})
// console.log(JSON.stringify(response.headers, null, 2))
const json = response.body.json()
const count = parseInt(response.headers["x-results-count"] || "0")
const count = parseInt(response.headers["X-Results-Count"] || "0")
// json is go data structure and incompatible with js, so we need to convert it
const r = { data: JSON.parse(JSON.stringify(json)), count: count }

View File

@ -3,6 +3,7 @@ const { apiClientBaseURL } = require("../config-client")
/**
* convert object to string
* @param {any} obj object
* @returns {string} string
*/
function obj2str(obj) {
if (Array.isArray(obj)) {
@ -32,70 +33,6 @@ function obj2str(obj) {
if (obj) return obj
}
// fetch polyfill
// [MIT License](LICENSE.md) © [Jason Miller](https://jasonformat.com/)
const _f = function (
/** @type {string | URL} */ url,
/** @type {{ method?: any; credentials?: any; headers?: any; body?: any; }} */ options
) {
if (typeof XMLHttpRequest === "undefined") {
return Promise.resolve(null)
}
options = options || {}
return new Promise((resolve, reject) => {
const request = new XMLHttpRequest()
const keys = []
const all = []
const headers = {}
const response = () => ({
ok: ((request.status / 100) | 0) == 2, // 200-299
statusText: request.statusText,
status: request.status,
url: request.responseURL,
text: () => Promise.resolve(request.responseText),
json: () => Promise.resolve(request.responseText).then(JSON.parse),
blob: () => Promise.resolve(new Blob([request.response])),
clone: response,
headers: {
// @ts-ignore
keys: () => keys,
// @ts-ignore
entries: () => all,
get: (n) => headers[n.toLowerCase()],
has: (n) => n.toLowerCase() in headers,
},
})
request.open(options.method || "get", url, true)
request.onload = () => {
request
.getAllResponseHeaders()
// @ts-ignore
.replace(/^(.*?):[^\S\n]*([\s\S]*?)$/gm, (m, key, value) => {
keys.push((key = key.toLowerCase()))
all.push([key, value])
headers[key] = headers[key] ? `${headers[key]},${value}` : value
})
resolve(response())
}
request.onerror = reject
request.withCredentials = options.credentials == "include"
for (const i in options.headers) {
request.setRequestHeader(i, options.headers[i])
}
request.send(options.body || null)
})
}
const _fetch = typeof fetch === "undefined" ? (typeof window === "undefined" ? _f : window.fetch || _f) : fetch
/**
* api request via client or server
* server function ssrRequest is called via context.ssrRequest, binded in ssr hook
@ -103,27 +40,29 @@ const _fetch = typeof fetch === "undefined" ? (typeof window === "undefined" ? _
* @param {string} endpoint
* @param {ApiOptions} options
* @param {any} body
* @param {import("../../../frontend/src/sentry")} sentry
* @param {typeof fetch} _fetch
* @returns {Promise<ApiResult<any>>}
*/
function apiRequest(endpoint, options, body) {
function apiRequest(endpoint, options, body, sentry, _fetch) {
// TODO cache only for GET
// first check cache if on client
const cacheKey = obj2str({ endpoint: endpoint, options: options })
options.method = options?.method || "GET"
console.log("SSR CHECK", cacheKey)
let method = options?.method || "GET"
// @ts-ignore
if (typeof window !== "undefined" && window.__SSR_CACHE__ && options?.method === "GET") {
if (typeof window !== "undefined" && window.__SSR_CACHE__ && method === "GET") {
// @ts-ignore
const cache = window.__SSR_CACHE__[cacheKey]
console.log("SSR:", cacheKey, cache)
console.log("SSR HIT:", cacheKey, cache)
if (cache) {
return Promise.resolve(cache)
}
}
let method = options?.method || "GET"
let query = "&count=1"
if (options?.filter) query += "&filter=" + encodeURIComponent(JSON.stringify(options.filter))
if (options?.sort) query += "&sort=" + options.sort + "&sort=_id"
@ -138,6 +77,7 @@ function apiRequest(endpoint, options, body) {
})
}
/** @type {{[key: string]: string}} */
let headers = {
"Content-Type": "application/json",
}
@ -153,8 +93,19 @@ function apiRequest(endpoint, options, body) {
return d
} else {
// client
let url = endpoint + (query ? "?" + query : "")
console.log("URL:", url)
let url = apiClientBaseURL + endpoint + (query ? "?" + query : "")
const span = sentry.currentTransaction()?.startChild({
op: "fetch",
description: method + " " + url,
data: Object.assign({}, options, { url }),
})
const trace_id = span?.toTraceparent()
if (trace_id) {
headers["sentry-trace"] = trace_id
}
/** @type {{[key: string]: any}} */
const requestOptions = {
method,
mode: "cors",
@ -165,7 +116,7 @@ function apiRequest(endpoint, options, body) {
requestOptions.body = JSON.stringify(body)
}
return _fetch(apiClientBaseURL + url, requestOptions).then((response) => {
const response = _fetch(url, requestOptions).then((response) => {
return response?.json().then((json) => {
if (response?.status < 200 || response?.status >= 400) {
return Promise.reject({ response, data: json })
@ -173,6 +124,11 @@ function apiRequest(endpoint, options, body) {
return Promise.resolve({ data: json || null, count: response.headers?.get("x-results-count") || 0 })
})
})
span?.end()
// @ts-ignore
return response
}
}

View File

@ -41,10 +41,12 @@ var { LIGHTHOUSE_TOKEN } = require("../config")
/**
* clear SSR cache
* @returns {number}
*/
function clearSSRCache() {
var info = context.db.deleteMany("ssr", {})
context.response.header("X-SSR-Cleared", info.removed)
return info.removed
}
function calculateAverageDynamically(dbObjs) {

View File

@ -1,29 +1,38 @@
// TODO: add query string functionality to cache
const utils = require("../lib/utils")
const { release } = require("../config-client")
const { ssrValidatePath } = require("../config")
const { log } = require("../lib/utils")
const { ssrRequest } = require("../lib/ssr-server.js")
const { ssrRequest } = require("../lib/ssr-server")
;(function () {
/** @type {HookResponse} */
let response = null
var request = context.request()
const request = context.request()
let url = request.query("url")
var trackingCall = request.header("x-ssr-skip")
if (trackingCall) {
// skip tracking
throw {
status: parseInt(trackingCall),
html: "",
log: false,
}
}
let url = request.query("url") || request.header("x-ssr-url") || "/"
const noCache = request.query("noCache")
// add sentry trace id to head
const trace_id = context.debug.sentryTraceId()
/**
* @param {string} content
*/
function addSentryTrace(content) {
return content.replace("</head>", '<meta name="sentry-trace" content="' + trace_id + '" /></head>')
}
context.response.header("sentry-trace", trace_id)
if (url) {
// comment will be printed to html later
let comment = ""
/** @type {Date} */ // @ts-ignore
context.ssrCacheValidUntil = null
context.response.header("sentry-trace", trace_id)
const auth = context.user.auth()
if (auth && auth.role == 0) {
} else if (url) {
var comment = ""
url = url.split("?")[0]
comment += "url: " + url
@ -31,40 +40,55 @@ const { ssrRequest } = require("../lib/ssr-server.js")
if (url && url.length > 1) {
url = url.replace(/\/$/, "")
}
if (url == "/noindex" || !url) {
if (url == "/index" || !url) {
url = "/" // see .htaccess
}
// check if url is in cache
/** @type {Ssr[]} */ // @ts-ignore
const cache =
!noCache &&
context.db.find("ssr", {
filter: {
path: url,
},
})
if (cache && cache.length) {
// use cache
context.response.header("X-SSR-Cache", "true")
throw {
status: 200,
log: false,
html: addSentryTrace(cache[0].content),
function useCache(/** @type {string} */ _url) {
var cache =
!noCache &&
context.db.find("ssr", {
filter: {
path: _url,
},
})
if (cache && cache.length) {
// use cache
context.response.header("X-SSR-Cache", "true")
throw {
status: 200,
log: false,
html: addSentryTrace(cache[0].content),
}
}
}
// validate url
let status = 200
var status = 200
let pNorender = false
let pNotfound = false
var pNorender = false
var pNotfound = false
const pR = ssrValidatePath(url)
if (pR < 0) {
if (pR === -1) {
pNotfound = true
comment += ", notFound"
} else if (!pR) {
pNorender = true
comment += ", noRender"
} else if (typeof pR === "string") {
url = pR
comment += ", cache url: " + url
}
if (noCache) {
comment += ", noCache"
}
if (!pNorender && !pNotfound) {
// check if we have a cache
useCache(url)
}
let head = ""
@ -73,7 +97,7 @@ const { ssrRequest } = require("../lib/ssr-server.js")
comment += ", path: " + url
let cacheIt = false
var cacheIt = false
if (pNorender) {
html = "<!-- NO SSR RENDERING -->"
} else if (pNotfound) {
@ -85,20 +109,17 @@ const { ssrRequest } = require("../lib/ssr-server.js")
// @ts-ignore
context.ssrRequest = ssrRequest
// try rendering, if error output plain html
try {
// include App.svelte and render it
// if error, output plain html without prerendering
// @ts-ignore
// console.log("####### RENDERING ", url)
const app = require("../lib/app.server")
const rendered = app.default.render({
url: url,
})
head = rendered.head
html = rendered.html
// add ssrCache to head, cache is built in ssr.js/apiRequest
head +=
"\n\n" +
"<script>window.__SSR_CACHE__ = " +
@ -109,50 +130,49 @@ const { ssrRequest } = require("../lib/ssr-server.js")
// status from webapp
// @ts-ignore
if (context.is404) {
// console.log("########## 404")
status = 404
} else {
cacheIt = true
}
} catch (e) {
// save error for later insert into html
log(e.message)
log(e.stack)
} catch (/** @type {any} */ e) {
utils.log(e.message)
utils.log(e.stack)
error = "error: " + e.message + "\n\n" + e.stack
// utils.log(e)
// for (var property in e) {
// utils.log(property + ": " + e[property])
// }
// error = JSON.stringify(e)
}
}
// read html template and replace placeholders
let tpl = context.fs.readFile("templates/spa.html")
var tpl = context.fs.readFile("templates/spa.html")
tpl = tpl.replace("<!--HEAD-->", head)
tpl = tpl.replace("<!--HTML-->", html)
tpl = tpl.replace("<!--SSR.ERROR-->", error ? "<!--" + error + "-->" : "")
tpl = tpl.replace("<!--SSR.COMMENT-->", comment ? "<!--" + comment + "-->" : "")
// save cache if adviced
if (cacheIt && !noCache) {
// save cache
context.db.create("ssr", {
// context.debug.dump("ssr", {
path: url,
content: tpl,
})
}
// return html
throw {
status: status,
log: false,
html: addSentryTrace(tpl),
}
} else {
// only admins are allowed to get without url parameter
const auth = context.user.auth()
if (!auth || auth.role !== 0) {
throw {
status: 403,
message: "invalid auth",
auth: auth,
}
// only admins are allowed
throw {
status: 403,
message: "invalid auth",
auth: auth,
release: release,
}
}
})()

View File

@ -1,16 +1,25 @@
const utils = require("../lib/utils")
const { release } = require("../config-client")
var utils = require("../lib/utils")
;(function () {
if (context.request().query("clear")) {
utils.clearSSRCache()
throw {
console.log("CLEARING SSR CACHE")
var removed = utils.clearSSRCache()
var stats = {
status: 200,
message: "cache cleared",
message: "ok",
removed: removed,
release: release,
}
utils.log(stats)
throw stats
}
throw {
status: 500,
message: "ssr is only a dummy collection",
release: release,
}
})()

5
babel.config.server.json Normal file
View File

@ -0,0 +1,5 @@
{
"sourceMaps": "inline",
"inputSourceMap": true,
"plugins": [["@babel/plugin-transform-async-to-generator"]]
}

View File

@ -5,6 +5,9 @@ config.options.sourcemap = "inline"
config.options.minify = false
config.options.platform = "node"
config.options.format = "cjs"
// es2015 will transform async/await to generators, but not working with svelte
// so we need babel to transform async/await to promises
// config.options.target = "es2015"
config.options.entryPoints = ["./frontend/src/ssr.ts"]
config.options.outfile = __dirname + "/_temp/app.server.js"
config.options.plugins = [
@ -16,6 +19,11 @@ config.options.plugins = [
dev: (process.argv?.length > 2 ? process.argv[2] : "build") !== "build",
},
preprocess: svelteConfig.preprocess,
filterWarnings: (warning) => {
// filter out a11y
if (warning.code.match(/^a11y/)) return false
return true
},
}),
config.resolvePlugin,
]

View File

@ -1,12 +1,77 @@
import { apiRequest } from "../../api/hooks/lib/ssr"
import * as sentry from "./sentry"
// fetch polyfill
// [MIT License](LICENSE.md) © [Jason Miller](https://jasonformat.com/)
const _f = function (url: string, options?: { [key: string]: any }) {
if (typeof XMLHttpRequest === "undefined") {
return Promise.resolve(null)
}
options = options || {}
return new Promise((resolve, reject) => {
const request = new XMLHttpRequest()
const keys: string[] = []
const all = []
const headers = {}
const response = () => ({
ok: ((request.status / 100) | 0) == 2, // 200-299
statusText: request.statusText,
status: request.status,
url: request.responseURL,
text: () => Promise.resolve(request.responseText),
json: () => Promise.resolve(request.responseText).then(JSON.parse),
blob: () => Promise.resolve(new Blob([request.response])),
clone: response,
headers: {
// @ts-ignore
keys: () => keys,
// @ts-ignore
entries: () => all,
get: (n) => headers[n.toLowerCase()],
has: (n) => n.toLowerCase() in headers,
},
})
request.open(options.method || "get", url, true)
request.onload = () => {
request
.getAllResponseHeaders()
// @ts-ignore
.replace(/^(.*?):[^\S\n]*([\s\S]*?)$/gm, (m, key, value) => {
keys.push((key = key.toLowerCase()))
all.push([key, value])
headers[key] = headers[key] ? `${headers[key]},${value}` : value
})
resolve(response())
}
request.onerror = reject
request.withCredentials = options.credentials == "include"
for (const i in options.headers) {
request.setRequestHeader(i, options.headers[i])
}
request.send(options.body || null)
})
}
// fetch must be declared after sentry import to get the hijacked fetch
// @ts-ignore
export const _fetch: typeof fetch =
typeof fetch === "undefined" ? (typeof window === "undefined" ? _f : window.fetch || _f) : fetch
export const api = async <T>(
endpoint: string,
options?: ApiOptions,
body?: any
): Promise<{ data: T; count: number } | any> => {
let data = await apiRequest(endpoint, options, body)
let data = await apiRequest(endpoint, options, body, sentry, _fetch)
// @ts-ignore
console.log(data, "data")
// console.log(data, "data")
return data
}

View File

@ -10,31 +10,33 @@
unten: "flex-end",
}
window.addEventListener("ccAccept", (e) => {
if ((e as CustomEvent).detail[1] == cookieName) contentShown = true
})
//isCookieSet isnt really precise
function checkCookie(cookieName: string) {
// Get all cookies
var allCookies = decodeURIComponent(document.cookie)
// Split into individual cookies
var cookies = allCookies.split(";")
var ccTagCookies: string[] = []
cookies.forEach((e) => {
e.includes("ccTags") ? (ccTagCookies = e.split(",")) : void 0
if (typeof window !== "undefined") {
window.addEventListener("ccAccept", (e) => {
if ((e as CustomEvent).detail[1] == cookieName) contentShown = true
})
for (var i = 0; i < ccTagCookies.length; i++) {
var c = ccTagCookies[i]
// Trim whitespace
while (c.charAt(0) == " ") c = c.substring(1)
// If the cookie's name matches the given name
if (c == cookieName) return true
//isCookieSet isnt really precise
function checkCookie(cookieName: string) {
// Get all cookies
var allCookies = decodeURIComponent(document.cookie)
// Split into individual cookies
var cookies = allCookies.split(";")
var ccTagCookies: string[] = []
cookies.forEach((e) => {
e.includes("ccTags") ? (ccTagCookies = e.split(",")) : void 0
})
for (var i = 0; i < ccTagCookies.length; i++) {
var c = ccTagCookies[i]
// Trim whitespace
while (c.charAt(0) == " ") c = c.substring(1)
// If the cookie's name matches the given name
if (c == cookieName) return true
}
return false
}
return false
}
// Verwendung
if (checkCookie(cookieName)) contentShown = true
// Verwendung
if (checkCookie(cookieName)) contentShown = true
}
</script>
{#if contentShown}

View File

@ -4,7 +4,7 @@
let opened = -1
let openMenu = false
$: {
$: if (typeof document !== "undefined") {
if (openMenu) document.body.style.overflow = "hidden"
else document.body.style.overflow = "auto"
}

View File

@ -90,24 +90,26 @@
<!-- svelte-ignore a11y-click-events-have-key-events -->
<!-- svelte-ignore a11y-no-static-element-interactions -->
<div class="container">
<div
class="datepicker"
on:click|preventDefault|stopPropagation
on:keydown|stopPropagation
on:submit|stopPropagation|preventDefault
>
<CalendarView
weekStart="{1}"
bind:vaue="{value}"
on:change="{(e) => {
value = e.detail
}}"
min="{today}"
max="{oneYearFromNow}"
blackout="{blackoutDates}"
locale="de-DE"
/>
</div>
{#if typeof window !== "undefined"}
<div
class="datepicker"
on:click|preventDefault|stopPropagation
on:keydown|stopPropagation
on:submit|stopPropagation|preventDefault
>
<CalendarView
weekStart="{1}"
bind:vaue="{value}"
on:change="{(e) => {
value = e.detail
}}"
min="{today}"
max="{oneYearFromNow}"
blackout="{blackoutDates}"
locale="de-DE"
/>
</div>
{/if}
</div>
<style lang="less">

View File

@ -2,14 +2,11 @@
import { mediaLibrary, pages } from "../lib/store"
import Pagebuilder from "../lib/components/pagebuilder/Pagebuilder.svelte"
import { apiBaseURL, baseURL } from "../config"
import { onMount } from "svelte"
import NotFound from "./NotFound.svelte"
export let path: string = "/"
export let isHomepage = true
let page: Content = $pages[path]
$: page = $pages[path]
let personPage = false
//test
</script>
<svelte:head>

View File

@ -14,7 +14,7 @@
"build": "node scripts/esbuild-wrapper.js build",
"build:admin": "node scripts/esbuild-wrapper.js build esbuild.config.admin.js",
"build:legacy": "node scripts/esbuild-wrapper.js build esbuild.config.legacy.js && babel _temp/index.js -o _temp/index.babeled.js && esbuild _temp/index.babeled.js --outfile=frontend/dist/index.es5.js --target=es5 --bundle --minify --sourcemap",
"build:server": "node scripts/esbuild-wrapper.js build esbuild.config.server.js && babel _temp/app.server.js -o _temp/app.server.babeled.js && esbuild _temp/app.server.babeled.js --outfile=api/hooks/lib/app.server.js --target=es5 --bundle --sourcemap --platform=node",
"build:server": "node scripts/esbuild-wrapper.js build esbuild.config.server.js && babel --config-file ./babel.config.server.json _temp/app.server.js -o _temp/app.server.babeled.js && esbuild _temp/app.server.babeled.js --outfile=api/hooks/lib/app.server.js --bundle --sourcemap --platform=node",
"build:test": "node scripts/esbuild-wrapper.js build esbuild.config.test.js && babel --config-file ./babel.config.test.json _temp/hook.test.js -o _temp/hook.test.babeled.js && esbuild _temp/hook.test.babeled.js --outfile=api/hooks/lib/hook.test.js --target=es5 --bundle --sourcemap --platform=node",
"upload:sourcemaps": "scripts/upload-sourcemaps.sh"
},

2
types/global.d.ts vendored
View File

@ -2,7 +2,7 @@ interface Ssr {
id?: string
path: string
content: string
validUntil: any // go Time
// validUntil: any // go Time
}
interface ApiOptions {