From 56f13c676c45a3b86190631eb43babb9a72e2c8a Mon Sep 17 00:00:00 2001 From: chloe caruso Date: Mon, 11 Aug 2025 22:38:02 -0700 Subject: [PATCH] finish assets --- flake.nix | 46 ++-- framework/bundle.ts | 135 +++++---- framework/css.ts | 15 +- framework/debug.safe.ts | 2 +- framework/engine/jsx-runtime.ts | 2 +- framework/engine/render.test.tsx | 2 +- framework/engine/render.ts | 38 ++- framework/engine/suspense.ts | 4 +- framework/esbuild-support.ts | 15 +- framework/generate.ts | 89 +++--- framework/hot.ts | 14 +- framework/incremental.test.ts | 22 +- framework/incremental.ts | 194 +++++++------ framework/lib/assets.ts | 212 +++++++++++--- framework/lib/async.ts | 4 +- framework/lib/fs.ts | 4 +- framework/lib/markdown.tsx | 20 +- framework/lib/sitegen.ts | 23 +- framework/lib/sqlite.ts | 4 +- framework/lib/testing.ts | 10 +- framework/lib/view.ts | 68 +++-- framework/marko.ts | 23 +- framework/watch.ts | 19 +- src/backend.ts | 6 +- src/blog/pages/25/file-viewer-internals.mdx | 10 - src/blog/pages/25/marko-intro.markodown | 290 -------------------- src/file-viewer/backend.tsx | 24 +- src/file-viewer/bin/scan3.ts | 34 ++- src/file-viewer/format.ts | 54 ++-- src/file-viewer/rsync.ts | 8 +- src/global.css | 26 +- src/q+a/backend.ts | 42 ++- src/q+a/clover-markdown.tsx | 4 +- src/q+a/image.tsx | 9 +- src/q+a/pages/admin.q+a.marko | 2 +- src/q+a/views/backend-inbox.marko | 4 +- src/source-of-truth.ts | 8 +- tsconfig.json | 5 +- 38 files changed, 736 insertions(+), 755 deletions(-) delete mode 100644 src/blog/pages/25/file-viewer-internals.mdx delete mode 100644 src/blog/pages/25/marko-intro.markodown diff --git a/flake.nix b/flake.nix index dafcbdd..e2f36c1 100644 --- a/flake.nix +++ b/flake.nix @@ -3,27 +3,27 @@ nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; utils.url = "github:numtide/flake-utils"; }; - outputs = - { nixpkgs, utils, ... }: - utils.lib.eachDefaultSystem ( - system: - let - pkgs = nixpkgs.legacyPackages.${system}; - in - { - devShells.default = pkgs.mkShell { - buildInputs = [ - pkgs.nodejs_24 # runtime - pkgs.deno # formatter - (pkgs.ffmpeg.override { - withOpus = true; - withSvtav1 = true; - withJxl = true; - withWebp = true; - }) - pkgs.rsync - ]; - }; - } - ); + outputs = inputs: inputs.utils.lib.eachDefaultSystem (system: + with inputs.nixpkgs.legacyPackages.${system}; { + devShells.default = pkgs.mkShell { + buildInputs = [ + pkgs.nodejs_24 # runtime + pkgs.deno # formatter + (pkgs.ffmpeg.override { + withOpus = true; + withSvtav1 = true; + withJxl = true; + withWebp = true; + }) + pkgs.rsync + ]; + }; + devShells.min = pkgs.mkShell { + buildInputs = [ + pkgs.nodejs_24 # runtime + pkgs.deno # formatter + pkgs.rsync + ]; + }; + }); } diff --git a/framework/bundle.ts b/framework/bundle.ts index ba2d007..3a4f3a1 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -88,19 +88,19 @@ export async function bundleClientJavaScript( for (const file of outputFiles) { const { text } = file; let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/"); - const { inputs } = UNWRAP(metafile.outputs["out!" + route]); - const sources = Object.keys(inputs).filter((x) => !isIgnoredSource(x)); + const { entryPoint } = UNWRAP(metafile.outputs["out!" + route]); // Register non-chunks as script entries. const chunk = route.startsWith("/js/c."); if (!chunk) { - const key = hot.getScriptId(path.resolve(sources[sources.length - 1])); + const key = hot.getScriptId(toAbs(UNWRAP(entryPoint))); + console.log(route, key); route = "/js/" + key.replace(/\.client\.tsx?/, ".js"); scripts[key] = text; } // Register chunks and public scripts as assets. if (chunk || publicScriptRoutes.includes(route)) { - p.push(io.writeAsset(route, text)); + p.push(io.writeAsset({ pathname: route, buffer: text })); } } await Promise.all(p); @@ -124,48 +124,86 @@ export async function bundleServerJavaScript({ entries, platform, }: ServerSideOptions) { - const wViewSource = incr.work(async (_, viewItems: sg.FileItem[]) => { - const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_"); - return { - magicWord, - file: [ - ...viewItems.map( - (view, i) => `import * as view${i} from ${JSON.stringify(view.file)}`, - ), - `const styles = ${magicWord}[-2]`, - `export const scripts = ${magicWord}[-1]`, - "export const views = {", - ...viewItems.map((view, i) => - [ - ` ${JSON.stringify(view.id)}: {`, - ` component: view${i}.default,`, - ` meta: view${i}.meta,`, - ` layout: view${i}.layout?.default ?? null,`, - ` inlineCss: styles[${magicWord}[${i}]]`, - ` },`, - ].join("\n"), - ), - "}", - ].join("\n"), - }; - }, viewItems); + const regenKeys: Record = {}; + const regenTtls: view.Ttl[] = []; + for (const ref of viewRefs) { + const value = UNWRAP(ref.value); + if (value.type === "page" && (value.regenerate?.tags?.length ?? 0) > 0) { + for (const tag of value.regenerate!.tags!) { + (regenKeys[tag] ??= []).push(`page:${value.id}`); + } + } + if (value.type === "page" && (value.regenerate?.seconds ?? 0) > 0) { + regenTtls.push({ + key: `page:${value.id}` as view.Key, + seconds: value.regenerate!.seconds!, + }); + } + } - await incr.work(async (io) => { - io.writeFile( - "../ts/view.d.ts", - [ - "export interface RegisteredViews {", - ...viewItems - .filter((view) => !view.id.startsWith("page:")) - .map( - (view) => - ` ${JSON.stringify(view.id)}: ` + - `typeof import(${JSON.stringify(path.relative(".clover/ts", toAbs(view.file)))}),`, + const wViewSource = incr.work( + async ( + _, + { viewItems, regenKeys, regenTtls }: { + viewItems: sg.FileItem[]; + regenKeys: Record; + regenTtls: view.Ttl[]; + }, + ) => { + const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_"); + return { + magicWord, + file: [ + ...viewItems.map( + (view, i) => + `import * as view${i} from ${JSON.stringify(view.file)}`, ), - "}", - ].join("\n"), - ); - }); + `const styles = ${magicWord}[-2]`, + `export const scripts = ${magicWord}[-1]`, + "export const views = {", + ...viewItems.map((view, i) => + [ + ` ${JSON.stringify(view.id)}: {`, + ` component: view${i}.default,`, + ` meta: view${i}.meta,`, + ` layout: view${i}.layout?.default ?? null,`, + ` inlineCss: styles[${magicWord}[${i}]]`, + ` },`, + ].join("\n") + ), + "}", + `export const regenTags = ${JSON.stringify(regenKeys)};`, + `export const regenTtls = ${JSON.stringify(regenTtls)};`, + ].join("\n"), + }; + }, + { viewItems, regenKeys, regenTtls }, + ); + + await incr.work( + async (io, { regenKeys, viewItems }) => { + io.writeFile( + "../ts/view.d.ts", + [ + "export interface RegisteredViews {", + ...viewItems + .filter((view) => !view.id.startsWith("page:")) + .map( + (view) => + ` ${JSON.stringify(view.id)}: ` + + `typeof import(${ + JSON.stringify(path.relative(".clover/ts", toAbs(view.file))) + }),`, + ), + "}", + "export type RegenKey = " + + (regenKeys.map((key) => JSON.stringify(key)).join(" | ") || + "never"), + ].join("\n"), + ); + }, + { regenKeys: Object.keys(regenKeys), viewItems }, + ); const wBundles = entries.map((entry) => incr.work(async (io, entry) => { @@ -255,7 +293,7 @@ export async function bundleServerJavaScript({ // file in more than one chunk. if ( magicWord && - metafile.outputs[key].inputs["framework/lib/view.ts"] + UNWRAP(metafile.outputs[key]).inputs["framework/lib/view.ts"] ) { ASSERT(!fileWithMagicWord); fileWithMagicWord = { @@ -268,7 +306,7 @@ export async function bundleServerJavaScript({ } } return fileWithMagicWord; - }, entry), + }, entry) ); const wProcessed = wBundles.map(async (wBundle) => { @@ -288,7 +326,7 @@ export async function bundleServerJavaScript({ const viewStyleKeys = views.map((view) => view.styleKey); const viewCssBundles = await Promise.all( viewStyleKeys.map((key) => - io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key)), + io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key)) ), ); const styleList = Array.from(new Set(viewCssBundles)); @@ -307,7 +345,7 @@ export async function bundleServerJavaScript({ return JSON.stringify(Object.fromEntries(neededScripts)); } // Reference an index into `styleList` - return `${styleList.indexOf(viewCssBundles[i])}`; + return `${styleList.indexOf(UNWRAP(viewCssBundles[i]))}`; }); io.writeFile(basename, text); @@ -333,3 +371,4 @@ import * as mime from "#sitegen/mime"; import * as incr from "./incremental.ts"; import * as sg from "#sitegen"; import type { PageOrView } from "./generate.ts"; +import type * as view from "#sitegen/view"; diff --git a/framework/css.ts b/framework/css.ts index 8d441a9..f52e0a7 100644 --- a/framework/css.ts +++ b/framework/css.ts @@ -57,12 +57,14 @@ export function styleKey( export async function bundleCssFiles( io: Io, { cssImports, theme, dev }: { - cssImports: string[], - theme: Theme, - dev: boolean, - } + cssImports: string[]; + theme: Theme; + dev: boolean; + }, ) { - cssImports = await Promise.all(cssImports.map((file) => io.trackFile('src/' + file))); + cssImports = await Promise.all( + cssImports.map((file) => io.trackFile("src/" + file)), + ); const plugin = { name: "clover css", setup(b) { @@ -111,4 +113,5 @@ import * as esbuild from "esbuild"; import * as fs from "#sitegen/fs"; import * as hot from "./hot.ts"; import * as path from "node:path"; -import { virtualFiles } from "./esbuild-support.ts";import type { Io } from "./incremental.ts"; +import { virtualFiles } from "./esbuild-support.ts"; +import type { Io } from "./incremental.ts"; diff --git a/framework/debug.safe.ts b/framework/debug.safe.ts index 211e9eb..3b3643f 100644 --- a/framework/debug.safe.ts +++ b/framework/debug.safe.ts @@ -9,4 +9,4 @@ globalThis.UNWRAP = (t, ...args) => { globalThis.ASSERT = assert.ok; import * as util from "node:util"; -import * as assert from 'node:assert' +import * as assert from "node:assert"; diff --git a/framework/engine/jsx-runtime.ts b/framework/engine/jsx-runtime.ts index a41d6ff..f21a6b7 100644 --- a/framework/engine/jsx-runtime.ts +++ b/framework/engine/jsx-runtime.ts @@ -51,4 +51,4 @@ declare global { } } -import * as render from "./render.ts"; +import * as render from "#engine/render"; diff --git a/framework/engine/render.test.tsx b/framework/engine/render.test.tsx index 8bada16..f22be74 100644 --- a/framework/engine/render.test.tsx +++ b/framework/engine/render.test.tsx @@ -1,5 +1,5 @@ import { test } from "node:test"; -import * as render from "./render.ts"; +import * as render from "#engine/render"; test("sanity", (t) => t.assert.equal(render.sync("gm <3").text, "gm <3")); test("simple tree", (t) => diff --git a/framework/engine/render.ts b/framework/engine/render.ts index dbb51e7..836ef4b 100644 --- a/framework/engine/render.ts +++ b/framework/engine/render.ts @@ -118,8 +118,11 @@ export function resolveNode(r: State, node: unknown): ResolvedNode { if (!node && node !== 0) return ""; // falsy, non numeric if (typeof node !== "object") { if (node === true) return ""; // booleans are ignored - if (typeof node === "string") return escapeHtml(node); + if (typeof node === "string") return escapeHtmlContent(node); if (typeof node === "number") return String(node); // no escaping ever + if (typeof node === "symbol" && node.toString() === kElement.toString()) { + throw new Error(`There are two instances of Clover SSR loaded!`); + } throw new Error(`Cannot render ${inspect(node)} to HTML`); } if (node instanceof Promise) { @@ -217,12 +220,14 @@ function stringifyElement(element: ResolvedElement) { let attr; switch (prop) { default: - attr = `${prop}=${quoteIfNeeded(escapeHtml(String(value)))}`; + attr = `${prop}=${quoteIfNeeded(escapeAttribute(String(value)))}`; break; case "className": // Legacy React Compat case "class": - attr = `class=${quoteIfNeeded(escapeHtml(clsx(value as ClsxInput)))}`; + attr = `class=${ + quoteIfNeeded(escapeAttribute(clsx(value as ClsxInput))) + }`; break; case "htmlFor": throw new Error("Do not use the `htmlFor` attribute. Use `for`"); @@ -233,7 +238,7 @@ function stringifyElement(element: ResolvedElement) { case "key": continue; } - if (needSpace) (out += " "), (needSpace = !attr.endsWith('"')); + if (needSpace) ((out += " "), (needSpace = !attr.endsWith('"'))); out += attr; } out += ">"; @@ -254,14 +259,16 @@ export function stringifyStyleAttribute(style: Record) { let out = ``; for (const styleName in style) { if (out) out += ";"; - out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${escapeHtml( - String(style[styleName]), - )}`; + out += `${styleName.replace(/[A-Z]/g, "-$&").toLowerCase()}:${ + escapeAttribute( + String(style[styleName]), + ) + }`; } return "style=" + quoteIfNeeded(out); } export function quoteIfNeeded(text: string) { - if (text.includes(" ")) return '"' + text + '"'; + if (text.match(/["/>]/)) return '"' + text + '"'; return text; } @@ -303,6 +310,21 @@ export function clsx(mix: ClsxInput) { return str; } +export const escapeHtmlContent = (unsafeText: string) => + String(unsafeText) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +// TODO: combine into one function which decides if an attribute needs quotes +// and escapes it correctly depending on the context. +const escapeAttribute = (unsafeText: string) => + String(unsafeText) + .replace(/&/g, "&") + .replace(//g, ">") + .replace(/"/g, """); +/** @deprecated */ export const escapeHtml = (unsafeText: string) => String(unsafeText) .replace(/&/g, "&") diff --git a/framework/engine/suspense.ts b/framework/engine/suspense.ts index d0c7625..7ed7cce 100644 --- a/framework/engine/suspense.ts +++ b/framework/engine/suspense.ts @@ -42,7 +42,7 @@ export function Suspense({ children, fallback }: SuspenseProps): render.Node { r.asyncDone = () => { const rejections = r.rejections; if (rejections && rejections.length > 0) throw new Error("TODO"); - state.pushChunk?.(name, (ip[0] = resolved)); + state.pushChunk?.(name, ip[0] = resolved); }; return render.raw(ip); } @@ -99,4 +99,4 @@ export async function* renderStreaming< return addonOutput as unknown as T; } -import * as render from "./render.ts"; +import * as render from "#engine/render"; diff --git a/framework/esbuild-support.ts b/framework/esbuild-support.ts index 5a39fe1..c352b19 100644 --- a/framework/esbuild-support.ts +++ b/framework/esbuild-support.ts @@ -1,7 +1,12 @@ type Awaitable = T | Promise; export function virtualFiles( - map: Record Awaitable)>, + map: Record< + string, + | string + | esbuild.OnLoadResult + | (() => Awaitable) + >, ) { return { name: "clover vfs", @@ -22,7 +27,7 @@ export function virtualFiles( { filter: /./, namespace: "vfs" }, async ({ path }) => { let entry = map[path]; - if (typeof entry === 'function') entry = await entry(); + if (typeof entry === "function") entry = await entry(); return ({ resolveDir: ".", loader: "ts", @@ -88,7 +93,6 @@ export function markoViaBuildCache(): esbuild.Plugin { if (!fs.existsSync(file)) { console.warn(`File does not exist: ${file}`); } - console.log(markoCache.keys()); throw new Error("Marko file not in cache: " + file); } return ({ @@ -106,7 +110,7 @@ export function isIgnoredSource(source: string) { return source.includes(" incr.work( async (io, { id, file }) => - void (await io.writeAsset(id, await io.readFile(file))), + void (await io.writeAsset({ + pathname: id, + buffer: await io.readFile(file), + })), item, - ), + ) ), ); const routes = await Promise.all([...builtViews, ...builtPages]); @@ -63,7 +67,7 @@ export async function generate() { // -- assemble page assets -- const pAssemblePages = builtPages.map((page) => - assembleAndWritePage(page, styleMap, scriptMap), + assembleAndWritePage(page, styleMap, scriptMap) ); await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]); @@ -85,7 +89,7 @@ export async function discoverAllFiles( return ( await Promise.all( siteConfig.siteSections.map(({ root: sectionRoot }) => - incr.work(scanSiteSection, toAbs(sectionRoot)), + incr.work(scanSiteSection, toAbs(sectionRoot)) ), ) ).reduce((acc, next) => ({ @@ -113,10 +117,9 @@ export async function scanSiteSection(io: Io, sectionRoot: string) { let scripts: FileItem[] = []; const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); - const rootPrefix = - hot.projectSrc === sectionRoot - ? "" - : path.relative(hot.projectSrc, sectionRoot) + "/"; + const rootPrefix = hot.projectSrc === sectionRoot + ? "" + : path.relative(hot.projectSrc, sectionRoot) + "/"; const kinds = [ { dir: sectionPath("pages"), @@ -190,7 +193,7 @@ export async function preparePage(io: Io, item: sg.FileItem) { theme: pageTheme, layout, regenerate, - } = await io.import(item.file); + } = await io.import(item.file); if (!Page) throw new Error("Page is missing a 'default' export."); if (!metadata) throw new Error("Page is missing 'meta' export with a title."); @@ -229,23 +232,12 @@ export async function preparePage(io: Io, item: sg.FileItem) { ); } - // -- regeneration -- - let regeneration: Regeneration | null = null; - if (typeof regenerate?.seconds === "number") { - regeneration ??= {}; - regeneration.seconds = regenerate.seconds; - } - if (regenerate?.tags) { - regeneration ??= {}; - regeneration.tags = regenerate.tags; - } - const styleKey = css.styleKey(cssImports, theme); return { type: "page", id: item.id, file: item.file, - regenerate: regeneration, + regenerate, html: text, meta: renderedMeta, cssImports, @@ -255,11 +247,6 @@ export async function preparePage(io: Io, item: sg.FileItem) { } as const; } -interface Regeneration { - seconds?: number; - tags?: string[]; -} - export async function prepareView(io: Io, item: sg.FileItem) { const module = await io.import(item.file); if (!module.meta) throw new Error(`View is missing 'export const meta'`); @@ -305,26 +292,34 @@ export async function assembleAndWritePage( scriptWork: incr.Ref>, ) { const page = await pageWork; - return incr.work(async (io, { id, html, meta, styleKey, clientRefs }) => { - const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey))); + return incr.work( + async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => { + const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey))); - const scriptIds = clientRefs.map(hot.getScriptId); - const scriptMap = await io.readWork(scriptWork); - const scripts = scriptIds - .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`)) - .map((x) => `{${x}}`) - .join("\n"); + const scriptIds = clientRefs.map(hot.getScriptId); + const scriptMap = await io.readWork(scriptWork); + const scripts = scriptIds + .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`)) + .map((x) => `{${x}}`) + .join("\n"); - const doc = sg.wrapDocument({ - body: html, - head: meta, - inlineCss, - scripts, - }); - await io.writeAsset(id, doc, { - "Content-Type": "text/html", - }); - }, page); + const buffer = sg.wrapDocument({ + body: html, + head: meta, + inlineCss, + scripts, + }); + await io.writeAsset({ + pathname: id, + buffer, + headers: { + "Content-Type": "text/html", + }, + regenerative: !!regenerate, + }); + }, + page, + ); } export type PageOrView = PreparedPage | PreparedView; diff --git a/framework/hot.ts b/framework/hot.ts index c5560d7..1eb5c78 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -78,10 +78,9 @@ Module.prototype._compile = function ( } } fileStats.set(filename, { - cssImportsRecursive: - cssImportsMaybe.length > 0 - ? Array.from(new Set(cssImportsMaybe)) - : null, + cssImportsRecursive: cssImportsMaybe.length > 0 + ? Array.from(new Set(cssImportsMaybe)) + : null, imports, lastModified: Math.floor(stat.mtimeMs), }); @@ -137,14 +136,13 @@ export function loadEsbuildCode( src = code; } if (src.includes("import.meta")) { - src = - ` + src = ` import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())}; import.meta.dirname = ${JSON.stringify(path.dirname(filepath))}; import.meta.filename = ${JSON.stringify(filepath)}; ` - .trim() - .replace(/[\n\s]/g, "") + src; + .trim() + .replace(/[\n\s]/g, "") + src; } src = esbuild.transformSync(src, { loader, diff --git a/framework/incremental.test.ts b/framework/incremental.test.ts index 234bf6a..0972a51 100644 --- a/framework/incremental.test.ts +++ b/framework/incremental.test.ts @@ -7,38 +7,38 @@ test("trivial case", async () => { async function compilation() { const first = incr.work({ label: "first compute", - async run (io) { + async run(io) { await setTimeout(1000); const contents = await io.readFile(file1.path); return [contents, Math.random()] as const; - } + }, }); const second = incr.work({ label: "second compute", wait: first, - async run (io) { + async run(io) { await setTimeout(1000); return io.readWork(first)[0].toUpperCase(); - } + }, }); const third = incr.work({ label: "third compute", wait: first, - async run (io) { + async run(io) { await setTimeout(1000); return io.readWork(first)[1] * 1000; - } + }, }); return incr.work({ label: "last compute", wait: [second, third], - async run (io) { + async run(io) { await setTimeout(1000); return { second: io.readWork(second), third: io.readWork(third), - } - } + }; + }, }); } const { value: first } = await incr.compile(compilation); @@ -52,5 +52,5 @@ test("trivial case", async () => { import * as incr from "./incremental2.ts"; import { beforeEach, test } from "node:test"; -import { tmpFile } from "#sitegen/testing";import { setTimeout } from "node:timers/promises"; - +import { tmpFile } from "#sitegen/testing"; +import { setTimeout } from "node:timers/promises"; diff --git a/framework/incremental.ts b/framework/incremental.ts index 28c9004..c94d34f 100644 --- a/framework/incremental.ts +++ b/framework/incremental.ts @@ -2,8 +2,7 @@ // See `work()`, `compile()`, and `invalidate()` for details. // // All state is serializable to allow recovering state across sessions. -// This framework special-cases the asset map, but is otherwise -// agnostic of what it is a compiler for. +// This library special-cases the asset map, but is otherwise agnostic. let running = false; let jobs = 0; let newKeys = 0; @@ -15,28 +14,25 @@ let writes = new Map(); let assets = new Map(); // keyed by hash export interface Ref { - /** This method is compatible with `await` syntax */ - then( - onFulfilled: (value: T) => void, - onRejected: (error: unknown) => void, - ): void; key: string; + /** This method is compatible with `await` syntax */ + then(resolve: (value: T) => void, reject: (error: unknown) => void): void; get value(): T | null; } type Job = (io: Io, input: I) => Promise; /** - * Declare and a unit of work. Return value is memoized and - * only rebuilt when inputs (declared via `Io`) change. Outputs - * are written at the end of a compilation (see `compile`). + * Declare and a unit of work. Return value is memoized and only rebuilt when + * inputs change. Inputs are tracked via the `io` interface, as well as a hash + * of the `input` param and caller source code. Outputs are written at the end + * of a compilation (see `compile`). * - * If the returned `Ref` is not awaited or read - * via io.readWork, the job is never started. + * Work items are lazy, only started when `Ref` is awaited or `io.readWork`ed. */ export function work(job: Job): Ref; export function work(job: Job, input: I): Ref; export function work(job: Job, input: I = null as I): Ref { - const source = JSON.stringify(util.getCallSites(2)[1]); + const source = JSON.stringify(UNWRAP(util.getCallSites(2)[1])); const keySource = [source, util.inspect(input)].join(":"); const key = crypto.createHash("sha1").update(keySource).digest("base64url"); ASSERT(running); @@ -60,13 +56,7 @@ export function work(job: Job, input: I = null as I): Ref { const value = await job(io, input); validateSerializable(value, ""); const { reads, writes } = io; - works.set(key, { - value, - affects: [], - reads, - writes, - debug: source, - }); + works.set(key, { value, affects: [], reads, writes }); for (const add of reads.files) { const { affects } = UNWRAP(files.get(add)); ASSERT(!affects.includes(key)); @@ -133,12 +123,8 @@ export async function flush(start: number) { // Trim const detachedFiles = new Set(); const referencedAssets = new Set(); - for (const [ - k, - { - writes: { assets }, - }, - ] of works) { + for (const [k, v] of works) { + const assets = v.writes.assets; if (seenWorks.has(k)) { for (const asset of assets.values()) referencedAssets.add(asset.hash); continue; @@ -150,11 +136,9 @@ export async function flush(start: number) { files.delete(k); detachedFiles.add(k); } - for (const k of assets.keys()) { - if (!referencedAssets.has(k)) assets.delete(k); - } + for (const k of assets.keys()) if (!referencedAssets.has(k)) assets.delete(k); - const p = []; + const p: Promise[] = []; // File writes let dist = 0; for (const [key, { buffer, size }] of writes) { @@ -163,10 +147,14 @@ export async function flush(start: number) { } // Asset map { - const { json, blob } = getAssetManifest(); + const { json, blob, dynamic, dts } = getAssetManifest(); const jsonString = Buffer.from(JSON.stringify(json)); - p.push(fs.writeMkdir(".clover/o/static.json", jsonString)); - p.push(fs.writeMkdir(".clover/o/static.blob", blob)); + p.push(fs.writeMkdir(".clover/o/asset.json", jsonString)); + p.push(fs.writeMkdir(".clover/o/asset.blob", blob)); + p.push(fs.writeMkdir(".clover/ts/asset.d.ts", dts)); + for (const [k, v] of dynamic) { + p.push(fs.writeMkdir(`.clover/o/dynamic/${k}`, v)); + } dist += blob.byteLength + jsonString.byteLength; } await Promise.all(p); @@ -179,9 +167,8 @@ export async function flush(start: number) { console.writeLine(` - ${works.size} keys (${works.size - newKeys} cached)`); console.writeLine(` - ${assets.size} static assets`); console.writeLine( - ` - dist: ${formatSize(dist)}, incremental: ${formatSize( - serialized.byteLength, - )}`, + ` - dist: ${formatSize(dist)},` + + ` incremental: ${formatSize(serialized.byteLength)}`, ); } @@ -337,7 +324,7 @@ async function deserialize(buffer: Buffer) { if (err.code === "ENOENT") return null; throw err; }) - .then((stat) => ({ k, size, work, stat })), + .then((stat) => ({ k, size, work, stat })) ), ); for (const { k, stat, work, size } of statFiles) { @@ -356,19 +343,23 @@ async function deserialize(buffer: Buffer) { } await Promise.all( - Array.from(files, ([key, file]) => invalidateEntry(key, file)), + Array.from(files, ([key, file]) => invalidateEntry(key, file, false)), ); } -export async function invalidate(filePath: string): Promise { +export async function invalidate( + filePath: string, + unloadModule: boolean = true, +): Promise { const key = toRel(toAbs(filePath)); const file = UNWRAP(files.get(key), `Untracked file '${key}'`); - return invalidateEntry(key, file); + return invalidateEntry(key, file, unloadModule); } export async function invalidateEntry( key: string, file: TrackedFile, + unloadModule: boolean, ): Promise { try { if (file.type === "d") { @@ -399,36 +390,62 @@ export async function invalidateEntry( return false; } catch (e) { forceInvalidate(file); - hot.unload(toAbs(key)); + if (unloadModule) { + // TODO: handle when this triggers unloading of `generate.ts` + hot.unload(toAbs(key)); + } if (file.type === "null") files.delete(key); return true; } } export function getAssetManifest() { + const dynamic = new Map(); const writer = new BufferWriter(); - const asset = Object.fromEntries( + const assetMap = Object.fromEntries( Array.from(works, (work) => work[1].writes.assets) .filter((map) => map.size > 0) .flatMap((map) => - Array.from(map, ([key, { hash, headers }]) => { + Array.from(map, ([key, { hash, headers, regenerative }]) => { const { raw, gzip, zstd } = UNWRAP( assets.get(hash), `Asset ${key} (${hash})`, ); + if (regenerative) { + const id = crypto + .createHash("sha1") + .update(key) + .digest("hex") + .slice(0, 16); /* TODO */ + dynamic.set( + id, + manifest.packDynamicBuffer(raw, gzip, zstd, headers), + ); + return [key, { type: 1, id }] as const; + } return [ key, { + type: 0, raw: writer.write(raw, "raw:" + hash), gzip: writer.write(gzip, "gzip:" + hash), zstd: writer.write(zstd, "zstd:" + hash), headers, }, ] as const; - }), + }) ), - ) satisfies BuiltAssetMap; - return { json: asset, blob: writer.get() }; + ) satisfies manifest.Manifest; + return { + json: assetMap, + blob: writer.get(), + dynamic, + dts: "export type AssetKey = " + + Object.keys(assetMap) + .map((key) => JSON.stringify(key)) + .join(" | ") + + "\n", + }; } /* Input/Output with automatic tracking. @@ -502,7 +519,7 @@ export class Io { const stat = await fs.stat(abs); if (stat.isDirectory()) { return (await this.readDirRecursive(abs)).map((grand) => - path.join(child, grand), + path.join(child, grand) ); } else { return child; @@ -540,26 +557,27 @@ export class Io { } } } - async writeAsset( - pathname: string, - blob: string | Buffer, - headersOption?: HeadersInit, - ) { - ASSERT(pathname.startsWith("/")); - ASSERT(!seenWrites.has("a:" + pathname)); - - const buffer = typeof blob === "string" ? Buffer.from(blob) : blob; - - const headers = new Headers(headersOption ?? {}); + async writeAsset(asset: { + pathname: string; + buffer: string | Buffer; + regenerative?: boolean; + headers?: HeadersInit; + }) { + ASSERT(asset.pathname.startsWith("/")); + ASSERT(!seenWrites.has("a:" + asset.pathname)); + const buffer = typeof asset.buffer === "string" + ? Buffer.from(asset.buffer) + : asset.buffer; + const headers = new Headers(asset.headers ?? {}); const hash = crypto.createHash("sha1").update(buffer).digest("hex"); if (!headers.has("Content-Type")) { - headers.set("Content-Type", mime.contentTypeFor(pathname)); + headers.set("Content-Type", mime.contentTypeFor(asset.pathname)); } - headers.set("ETag", JSON.stringify(hash)); - this.writes.assets.set(pathname, { + headers.set("etag", JSON.stringify(hash)); + this.writes.assets.set(asset.pathname, { hash, - // @ts-expect-error TODO headers: Object.fromEntries(headers), + regenerative: !!asset.regenerative, }); if (!assets.has(hash)) { jobs += 1; @@ -600,7 +618,7 @@ class BufferWriter { write(buffer: Buffer, hash: string): BufferView { let view = this.seen.get(hash); if (view) return view; - view = [this.size, (this.size += buffer.byteLength)]; + view = [this.size, this.size += buffer.byteLength]; this.seen.set(hash, view); this.buffers.push(buffer); return view; @@ -626,7 +644,7 @@ export function validateSerializable(value: unknown, key: string) { Buffer.isBuffer(value) ) { Object.entries(value).forEach(([k, v]) => - validateSerializable(v, `${key}.${k}`), + validateSerializable(v, `${key}.${k}`) ); } else { throw new Error( @@ -662,54 +680,48 @@ interface FileWrite { } interface Writes { files: Set; - assets: Map< - string, - { - hash: string; - headers: Record; - } - >; + assets: Map; } interface Asset { raw: Buffer; gzip: Buffer; zstd: Buffer; } +interface AssetWrite { + hash: string; + headers: Record; + regenerative: boolean; +} interface Work { - debug?: string; value: T; reads: Reads; writes: Writes; affects: string[]; } -type TrackedFile = { - affects: string[]; -} & ( - | { type: "f"; lastModified: number } - | { type: "d"; contentHash: string; contents: string[] } - | { type: "null" } -); -export interface BuiltAssetMap { - [route: string]: BuiltAsset; -} -export interface BuiltAsset { - raw: BufferView; - gzip: BufferView; - zstd: BufferView; - headers: Record; -} +type TrackedFile = + & { affects: string[] } + & ( + | { type: "f"; lastModified: number } + | { type: "d"; contentHash: string; contents: string[] } + | { type: "null" } + ); const gzip = util.promisify(zlib.gzip); const zstdCompress = util.promisify(zlib.zstdCompress); -import * as fs from "#sitegen/fs"; -import * as path from "node:path"; import * as hot from "./hot.ts"; + +import * as fs from "#sitegen/fs"; +import * as mime from "#sitegen/mime"; +import * as manifest from "#sitegen/assets"; + +import * as path from "node:path"; import * as util from "node:util"; import * as crypto from "node:crypto"; -import * as mime from "#sitegen/mime"; import * as zlib from "node:zlib"; + import * as console from "@paperclover/console"; import { Spinner } from "@paperclover/console/Spinner"; import { formatSize } from "@/file-viewer/format.ts"; + import * as msgpackr from "msgpackr"; diff --git a/framework/lib/assets.ts b/framework/lib/assets.ts index 73e0d30..ac77d49 100644 --- a/framework/lib/assets.ts +++ b/framework/lib/assets.ts @@ -1,73 +1,114 @@ -interface Loaded { - map: BuiltAssetMap; - buf: Buffer; -} -let assets: Loaded | null = null; +// Static and dynamic assets are built alongside the server binary. +// This module implements decoding and serving of the asset blobs, +// but also implements patching of dynamic assets. The `Manifest` +// is generated by `incremental.ts` +const root = import.meta.dirname; +let current: Loaded | null = null; -export type StaticPageId = string; +// TODO: rename all these types +type DynamicId = string; +export type { Key }; +export type Manifest = + & { + [K in Key]: StaticAsset | DynamicAsset; + } + & { + [string: string]: StaticAsset | DynamicAsset; + }; +export interface StaticAsset extends AssetBase { + type: 0; +} +interface AssetBase { + headers: Record & { etag: string }; + raw: BufferView; + gzip: BufferView; + zstd: BufferView; +} +export interface DynamicAsset { + type: 1; + id: DynamicId; +} +interface Loaded { + map: Manifest; + static: Buffer; + dynamic: Map; +} +export interface DynamicEntry extends AssetBase { + buffer: Buffer; +} export async function reload() { - const [map, buf] = await Promise.all([ - fs.readFile(path.join(import.meta.dirname, "static.json"), "utf8"), - fs.readFile(path.join(import.meta.dirname, "static.blob")), - ]); - return (assets = { map: JSON.parse(map), buf }); -} - -export function reloadSync() { - const map = fs.readFileSync( - path.join(import.meta.dirname, "static.json"), - "utf8", + const map = await fs.readJson(path.join(root, "asset.json")); + const statics = await fs.readFile(path.join(root, "asset.blob")); + const dynamic = new Map( + await Promise.all( + Object.entries(map) + .filter((entry): entry is [string, DynamicAsset] => entry[1].type === 1) + .map(async ([k, v]) => + [ + v.id, + await fs.readFile(path.join(root, "dynamic", v.id)) + .then(loadRegenerative), + ] as const + ), + ), ); - const buf = fs.readFileSync(path.join(import.meta.dirname, "static.blob")); - return (assets = { map: JSON.parse(map), buf }); + return (current = { map, static: statics, dynamic }); } export async function middleware(c: Context, next: Next) { - if (!assets) await reload(); - const asset = assets!.map[c.req.path]; + if (!current) current = await reload(); + const asset = current.map[c.req.path]; if (asset) return assetInner(c, asset, 200); return next(); } export async function notFound(c: Context) { - if (!assets) await reload(); + if (!current) current = await reload(); let pathname = c.req.path; do { - const asset = assets!.map[pathname + "/404"]; + const asset = current.map[pathname + "/404"]; if (asset) return assetInner(c, asset, 404); pathname = pathname.slice(0, pathname.lastIndexOf("/")); } while (pathname); - const asset = assets!.map["/404"]; + const asset = current.map["/404"]; if (asset) return assetInner(c, asset, 404); return c.text("the 'Not Found' page was not found", 404); } -export async function serveAsset( - c: Context, - id: StaticPageId, - status: StatusCode, -) { - return assetInner(c, (assets ?? (await reload())).map[id], status); +export async function serveAsset(c: Context, id: Key, status: StatusCode) { + return assetInner(c, (current ?? (await reload())).map[id], status); } +/** @deprecated */ export function hasAsset(id: string) { - return (assets ?? reloadSync()).map[id] !== undefined; + return UNWRAP(current).map[id] !== undefined; } export function etagMatches(etag: string, ifNoneMatch: string) { return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; } -function subarrayAsset([start, end]: BufferView) { - return assets!.buf.subarray(start, end); +function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) { + ASSERT(current); + if (asset.type === 0) { + return respondWithBufferAndViews(c, current.static, asset, status); + } else { + const entry = UNWRAP(current.dynamic.get(asset.id)); + return respondWithBufferAndViews(c, entry.buffer, entry, status); + } } -function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) { - const ifnonematch = c.req.header("If-None-Match"); - if (ifnonematch) { - const etag = asset.headers.ETag; - if (etagMatches(etag, ifnonematch)) { +function respondWithBufferAndViews( + c: Context, + buffer: Buffer, + asset: AssetBase, + status: StatusCode, +) { + const ifNoneMatch = c.req.header("If-None-Match"); + if (ifNoneMatch) { + const etag = asset.headers.etag; + if (etagMatches(etag, ifNoneMatch)) { return (c.res = new Response(null, { status: 304, statusText: "Not Modified", @@ -80,24 +121,103 @@ function assetInner(c: Context, asset: BuiltAsset, status: StatusCode) { const acceptEncoding = c.req.header("Accept-Encoding") ?? ""; let body; let headers = asset.headers; - if (acceptEncoding.includes("zstd") && asset.zstd) { - body = subarrayAsset(asset.zstd); + if (acceptEncoding.includes("zstd")) { + body = buffer.subarray(...asset.zstd); headers = { ...asset.headers, "Content-Encoding": "zstd", }; - } else if (acceptEncoding.includes("gzip") && asset.gzip) { - body = subarrayAsset(asset.gzip); + } else if (acceptEncoding.includes("gzip")) { + body = buffer.subarray(...asset.gzip); headers = { ...asset.headers, "Content-Encoding": "gzip", }; } else { - body = subarrayAsset(asset.raw); + body = buffer.subarray(...asset.raw); } return (c.res = new Response(body, { headers, status })); } +export function packDynamicBuffer( + raw: Buffer, + gzip: Buffer, + zstd: Buffer, + headers: Record, +) { + const headersBuffer = Buffer.from( + Object.entries(headers) + .map((entry) => entry.join(":")) + .join("\n"), + "utf-8", + ); + const header = new Uint32Array(3); + header[0] = headersBuffer.byteLength + header.byteLength; + header[1] = header[0] + raw.byteLength; + header[2] = header[1] + gzip.byteLength; + return Buffer.concat([ + Buffer.from(header.buffer), + headersBuffer, + raw, + gzip, + zstd, + ]); +} + +function loadRegenerative(buffer: Buffer): DynamicEntry { + const headersEnd = buffer.readUInt32LE(0); + const headers = Object.fromEntries( + buffer + .subarray(3 * 4, headersEnd) + .toString("utf-8") + .split("\n") + .map((line) => { + const i = line.indexOf(":"); + return [line.slice(0, i), line.slice(i + 1)]; + }), + ); + const raw = buffer.readUInt32LE(4); + const gzip = buffer.readUInt32LE(8); + const hasEtag = (v: object): v is typeof v & { etag: string } => + "etag" in v && typeof v.etag === "string"; + ASSERT(hasEtag(headers)); + return { + headers, + buffer, + raw: [headersEnd, raw], + gzip: [raw, gzip], + zstd: [gzip, buffer.byteLength], + }; +} + +const gzip = util.promisify(zlib.gzip); +const zstdCompress = util.promisify(zlib.zstdCompress); + +export async function overwriteDynamic( + key: Key, + value: string | Buffer, + headers: Record, +) { + if (!current) current = await reload(); + const asset = UNWRAP(current.map[key]); + ASSERT(asset.type === 1); + UNWRAP(current.dynamic.has(asset.id)); + const buffer = Buffer.from(value); + const etag = JSON.stringify( + crypto.createHash("sha1").update(buffer).digest("hex"), + ); + const [gzipBuffer, zstdBuffer] = await Promise.all([ + gzip(buffer), + zstdCompress(buffer), + ]); + const packed = packDynamicBuffer(buffer, gzipBuffer, zstdBuffer, { + ...headers, + etag, + }); + current.dynamic.set(asset.id, loadRegenerative(packed)); + await fs.writeFile(path.join(root, "dynamic", asset.id), packed); +} + process.on("message", (msg: any) => { if (msg?.type === "clover.assets.reload") reload(); }); @@ -105,6 +225,10 @@ process.on("message", (msg: any) => { import * as fs from "#sitegen/fs"; import type { Context, Next } from "hono"; import type { StatusCode } from "hono/utils/http-status"; -import type { BuiltAsset, BuiltAssetMap, BufferView } from "../incremental.ts"; +import type { BufferView } from "../incremental.ts"; import { Buffer } from "node:buffer"; import * as path from "node:path"; +import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts"; +import * as crypto from "node:crypto"; +import * as zlib from "node:zlib"; +import * as util from "node:util"; diff --git a/framework/lib/async.ts b/framework/lib/async.ts index 881fd48..09b0713 100644 --- a/framework/lib/async.ts +++ b/framework/lib/async.ts @@ -51,8 +51,8 @@ export class Queue { let n = 0; for (const item of active) { let itemText = "- " + item.format(now); - text += - `\n` + itemText.slice(0, Math.max(0, process.stdout.columns - 1)); + text += `\n` + + itemText.slice(0, Math.max(0, process.stdout.columns - 1)); if (n > 10) { text += `\n ... + ${active.length - n} more`; break; diff --git a/framework/lib/fs.ts b/framework/lib/fs.ts index c6d9c17..b44d99e 100644 --- a/framework/lib/fs.ts +++ b/framework/lib/fs.ts @@ -4,6 +4,7 @@ export { createReadStream, createWriteStream, existsSync, + type FileHandle, open, readdir, readdirSync, @@ -15,7 +16,6 @@ export { statSync, writeFile, writeFileSync, - type FileHandle, }; export function mkdir(dir: string) { @@ -98,6 +98,7 @@ import { writeFileSync, } from "node:fs"; import { + type FileHandle, mkdir as nodeMkdir, open, readdir, @@ -106,6 +107,5 @@ import { rmdir, stat, writeFile, - type FileHandle, } from "node:fs/promises"; export { Stats } from "node:fs"; diff --git a/framework/lib/markdown.tsx b/framework/lib/markdown.tsx index 333b95c..938b709 100644 --- a/framework/lib/markdown.tsx +++ b/framework/lib/markdown.tsx @@ -1,9 +1,11 @@ -/* Impementation of CommonMark specification for markdown with support +/* Implementation of [CommonMark] specification for markdown with support * for custom syntax extensions via the parser options. Instead of * returning an AST that has a second conversion pass to JSX, the * returned value of 'parse' is 'engine.Node' which can be stringified - * via clover's SSR engine. This way, generation optimizations, async + * via Clover's SSR engine. This way, generation optimizations, async * components, and other features are gained for free here. + * + * [CommonMark]: https://spec.commonmark.org/0.31.2/ */ function parse(src: string, options: Partial = {}) {} @@ -16,6 +18,9 @@ export function Markdown({ return parse(src, options); } +// TODO: This implementation is flawed because it is impossible to sanely handle +// emphasis and strong emphasis, and all their edge cases. Instead of making these +// using extensions interface, they should be special cased. function parseInline(src: string, options: Partial = {}) { const { rules = inlineRules, links = new Map() } = options; const opts: InlineOpts = { rules, links }; @@ -110,12 +115,11 @@ export const inlineRules: Record = { } else if (afterText[0] === "[") { const splitTarget = splitFirst(afterText.slice(1), /]/); if (!splitTarget) return null; - const name = - splitTarget.first.trim().length === 0 - ? // Collapsed reference link - textSrc.trim() - : // Full Reference Link - splitTarget.first.trim(); + const name = splitTarget.first.trim().length === 0 + // Collapsed reference link + ? textSrc.trim() + // Full Reference Link + : splitTarget.first.trim(); const target = opts.links.get(name); if (!target) return null; ({ href, title } = target); diff --git a/framework/lib/sitegen.ts b/framework/lib/sitegen.ts index fd3950f..af99dca 100644 --- a/framework/lib/sitegen.ts +++ b/framework/lib/sitegen.ts @@ -1,6 +1,25 @@ -// Import this file with 'import * as sg from "#sitegen";' export type ScriptId = string; +export interface PageExports extends ViewExports { + regenerate?: PageRegenerateOptions; +} +export interface ViewExports { + default: render.Component; + meta: meta.Meta | ((props: { ssr: true }) => Promise | meta.Meta); + theme?: css.Theme; + layout?: Layout; +} +export interface Layout { + default: render.Component; + theme?: css.Theme; + // TODO: nested layout +} +export interface PageRegenerateOptions { + tags?: string[]; + seconds?: number; + debounce?: number; +} + /** * A filesystem object associated with some ID, * such as a page's route to it's source file. @@ -49,3 +68,5 @@ export function wrapDocument({ } import * as render from "#engine/render"; +import type * as meta from "./meta.ts"; +import type * as css from "../css.ts"; diff --git a/framework/lib/sqlite.ts b/framework/lib/sqlite.ts index 5d3822a..782f003 100644 --- a/framework/lib/sqlite.ts +++ b/framework/lib/sqlite.ts @@ -10,7 +10,9 @@ export function getDb(file: string) { if (db) return db; const fileWithExt = file.includes(".") ? file : file + ".sqlite"; db = new WrappedDatabase( - new DatabaseSync(path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt)), + new DatabaseSync( + path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt), + ), ); map.set(file, db); return db; diff --git a/framework/lib/testing.ts b/framework/lib/testing.ts index 36ad50f..3bdc773 100644 --- a/framework/lib/testing.ts +++ b/framework/lib/testing.ts @@ -1,5 +1,9 @@ export function tmpFile(basename: string) { - const file = path.join(import.meta.dirname, '../../.clover/testing', basename); + const file = path.join( + import.meta.dirname, + "../../.clover/testing", + basename, + ); return { path: file, read: fs.readFile.bind(fs, file), @@ -7,5 +11,5 @@ export function tmpFile(basename: string) { }; } -import * as path from 'node:path'; -import * as fs from './fs.ts'; +import * as path from "node:path"; +import * as fs from "./fs.ts"; diff --git a/framework/lib/view.ts b/framework/lib/view.ts index 7cf007b..b1c8ab5 100644 --- a/framework/lib/view.ts +++ b/framework/lib/view.ts @@ -1,6 +1,6 @@ // The "view" system allows rendering dynamic pages within backends. // This is done by scanning all `views` dirs, bundling their client -// resources, and then providing `renderView` which renders a page. +// resources, and then providing `serve` which renders a page. // // This system also implements page regeneration. let codegen: Codegen; @@ -12,53 +12,53 @@ try { // Generated in `bundle.ts` export interface Codegen { - views: Record; + views: { [K in Key]: View> }; scripts: Record; regenTtls: Ttl[]; - regenTags: Record; + regenTags: Record; } -export interface View { +// The view contains pre-bundled CSS and scripts, but keeps the scripts +// separate for run-time dynamic scripts. For example, the file viewer +// includes the canvas for the current page, but only the current page. +export interface View> { component: render.Component; - meta: - | meta.Meta - | ((props: { context?: hono.Context }) => Promise | meta.Meta); + meta: meta.Meta | ((props: Props) => Promise | meta.Meta); layout?: render.Component; inlineCss: string; scripts: Record; } export interface Ttl { seconds: number; - key: ViewKey; + key: Key; } -type ViewKey = keyof ViewMap; +export type Key = keyof ViewMap; -export async function renderView( +export async function serve( context: hono.Context, id: K, props: PropsFromModule, ) { - return context.html(await renderViewToString(id, { context, ...props })); + return context.html(await renderToString(id, { context, ...props })); } type PropsFromModule = M extends { default: (props: infer T) => render.Node; -} - ? T +} ? T : never; -export async function renderViewToString( +export async function renderToString( id: K, props: PropsFromModule, ) { - // The view contains pre-bundled CSS and scripts, but keeps the scripts - // separate for run-time dynamic scripts. For example, the file viewer - // includes the canvas for the current page, but only the current page. const { component, inlineCss, layout, meta: metadata, - }: View = UNWRAP(codegen.views[id], `Missing view ${id}`); + }: View> = UNWRAP( + codegen.views[id], + `Missing view ${id}`, + ); // -- metadata -- const renderedMetaPromise = Promise.resolve( @@ -79,21 +79,43 @@ export async function renderViewToString( head: await renderedMetaPromise, inlineCss, scripts: joinScripts( - Array.from(sitegen.scripts, (id) => - UNWRAP(codegen.scripts[id], `Missing script ${id}`), + Array.from( + sitegen!.scripts, + (id) => UNWRAP(codegen.scripts[id], `Missing script ${id}`), ), ), }); } -export function joinScripts(scriptSources: string[]) { +export function regenerate(tag: RegenKey) { + for (const view of codegen.regenTags[tag]) { + const key = view.slice("page:".length); + renderToString(view, {}) + .then((result) => { + console.info(`regenerate ${key}`); + asset.overwriteDynamic(key as asset.Key, result, { + "content-type": "text/html", + }); + }) + .catch((e) => { + console.error(`Failed regenerating ${view} from tag ${tag}`, e); + }); + } +} + +function joinScripts(scriptSources: string[]) { const { length } = scriptSources; if (length === 0) return ""; - if (length === 1) return scriptSources[0]; + if (0 in scriptSources) return scriptSources[0]; return scriptSources.map((source) => `{${source}}`).join(";"); } + import * as meta from "./meta.ts"; import type * as hono from "#hono"; import * as render from "#engine/render"; import * as sg from "./sitegen.ts"; -import type { RegisteredViews as ViewMap } from "../../.clover/ts/view.d.ts"; +import * as asset from "./assets.ts"; +import type { + RegenKey, + RegisteredViews as ViewMap, +} from "../../.clover/ts/view.d.ts"; diff --git a/framework/marko.ts b/framework/marko.ts index f3c79a0..c62f8b8 100644 --- a/framework/marko.ts +++ b/framework/marko.ts @@ -14,18 +14,19 @@ export function loadMarko(module: NodeJS.Module, filepath: string) { // bare client import statements to it's own usage. const scannedClientRefs = new Set(); if (src.match(/^\s*client\s+import\s+["']/m)) { - src = - src.replace( - /^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m, - (_, src) => { - const ref = JSON.parse(`"${src.slice(1, -1)}"`); - const resolved = hot.resolveClientRef(filepath, ref); - scannedClientRefs.add(resolved); - return ` { + const ref = JSON.parse(`"${src.slice(1, -1)}"`); + const resolved = hot.resolveClientRef(filepath, ref); + scannedClientRefs.add(resolved); + return ``; - }, - ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n'; + ) + } />`; + }, + ) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n'; } src = marko.compileSync(src, filepath).code; diff --git a/framework/watch.ts b/framework/watch.ts index 9bc7bf7..9fe2157 100644 --- a/framework/watch.ts +++ b/framework/watch.ts @@ -11,7 +11,7 @@ let watch: Watch; export async function main() { // Catch up state by running a main build. - await incr.restore(); + if (!process.argv.includes("-f")) await incr.restore(); watch = new Watch(rebuild); rebuild([]); } @@ -36,15 +36,16 @@ function onSubprocessClose(code: number | null, signal: string | null) { } async function rebuild(files: string[]) { - const hasInvalidated = files.length === 0 - || (await Promise.all(files.map(incr.invalidate))).some(Boolean); + const hasInvalidated = files.length === 0 || + (await Promise.all(files.map((file) => incr.invalidate(file)))) + .some(Boolean); if (!hasInvalidated) return; incr.compile(generate.generate).then(({ watchFiles, newOutputs, - newAssets + newAssets, }) => { - const removeWatch = [...watch.files].filter(x => !watchFiles.has(x)) + const removeWatch = [...watch.files].filter((x) => !watchFiles.has(x)); for (const file of removeWatch) watch.remove(file); watch.add(...watchFiles); // Restart the server if it was changed or not running. @@ -60,8 +61,8 @@ async function rebuild(files: string[]) { function statusLine() { console.info( - `Watching ${watch.files.size} files ` - + `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`, + `Watching ${watch.files.size} files ` + + `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`, ); } @@ -142,7 +143,7 @@ class Watch { #getFiles(absPath: string, event: fs.WatchEventType) { const files = []; if (this.files.has(absPath)) files.push(absPath); - if (event === 'rename') { + if (event === "rename") { const dir = path.dirname(absPath); if (this.files.has(dir)) files.push(dir); } @@ -153,7 +154,7 @@ class Watch { if (!subPath) return; const files = this.#getFiles(path.join(root, subPath), event); if (files.length === 0) return; - for(const file of files) this.stale.add(file); + for (const file of files) this.stale.add(file); const { debounce } = this; if (debounce !== null) clearTimeout(debounce); this.debounce = setTimeout(() => { diff --git a/src/backend.ts b/src/backend.ts index 396b597..48b5a11 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -1,6 +1,6 @@ -// This is the main file for the backend +// This is the main file for paperclover.net's server. const app = new Hono(); -const logHttp = scoped("http", { color: "magenta" }); +const logHttp = console.scoped("http", { color: "magenta" }); // Middleware app.use(trimTrailingSlash()); @@ -38,4 +38,4 @@ import { logger } from "hono/logger"; import { trimTrailingSlash } from "hono/trailing-slash"; import * as assets from "#sitegen/assets"; import * as admin from "./admin.ts"; -import { scoped } from "@paperclover/console"; +import * as console from "@paperclover/console"; diff --git a/src/blog/pages/25/file-viewer-internals.mdx b/src/blog/pages/25/file-viewer-internals.mdx deleted file mode 100644 index 5f1a1cb..0000000 --- a/src/blog/pages/25/file-viewer-internals.mdx +++ /dev/null @@ -1,10 +0,0 @@ -export const blog: BlogMeta = { - title: "Marko is the coziest HTML templating language", - desc: "...todo...", - date: "2025-07-08", - draft: true, -}; -export const meta = formatBlogMeta(blob); -export * as layout from "@/blog/layout.tsx"; - - diff --git a/src/blog/pages/25/marko-intro.markodown b/src/blog/pages/25/marko-intro.markodown deleted file mode 100644 index 14a0983..0000000 --- a/src/blog/pages/25/marko-intro.markodown +++ /dev/null @@ -1,290 +0,0 @@ -export const blog: BlogMeta = { - title: "Marko is the coziest HTML templating language", - desc: "...todo...", - date: "2025-06-13", - draft: true, -}; -export const meta = formatBlogMeta(blob); -export * as layout from "@/blog/layout.tsx"; - -I've been recently playing around [Marko][1], and after adding limited support -for it in my website generator, [sitegen][2], I instantly fell in love with how -minimalistic it is in comparison to JSX, Astro components, and Svelte. - -## Introduction - -If JSX was taking HTML and shoving its syntax into JavaScript, Marko is shoving -JavaScript into HTML. Attributes are JavaScript expressions. - -```marko -
- // `input` is like props, but given in the top-level scope - - - - // Capital letter variables for imported components - - - // Components also can be auto-imported by lowercase. - // This will look upwards for a `tags/` folder containing - // "custom-footer.marko", similar to how Node.js finds - // package names in all upwards `node_modules` folders. - -
- -// ESM `import` / `export` just work as expected. -// I prefer my imports at the end, to highlight the markup. -import MarkdownContent from "./MarkdownContent.marko"; -import { formatTimeNicely } from "../date-helpers.ts"; -``` - -Tags with the `value` attribute have a shorthand, which is used by the built-in -`` for conditional rendering. - -```marko -// Sugar for - - -// and it composes amazingly to the 'if' built-in - - - -``` - -Tags can also return values into the scope for use in the template using `/`, such as `` for unique ID generation. This is available to components that ``. - -``` - - - -