clo/sitegen
1
0
Fork 0
Code Issues 20 Pull requests Projects Releases Packages Wiki Activity Actions

rewrite incremental.ts (#21)

Browse source 
the problems with the original implementation was mostly around error
handling. sources had to be tracked manually and provided to each
incremental output. the `hasArtifact` check was frequently forgotten.
this has been re-abstracted through `incr.work()`, which is given an
`io` object. all fs reads and module loads go through this interface,
which allows the sources to be properly tracked, even if it throws.

closes #12
This commit is contained in:
clover caruso 2025-08-02 17:31:58 -07:00
parent ff5e207f94
commit 8c0bd4c6c6
19 changed files with 1653 additions and 1560 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

248
framework/bundle.ts
View file

@ -1,17 +1,25 @@
async function trackEsbuild(io: Io, metafile: esbuild.Metafile) {
await Promise.all(Object.keys(metafile.inputs)
.filter(file => !isIgnoredSource(file))
.map(file => io.trackFile(file)));
}
// This file implements client-side bundling, mostly wrapping esbuild. // This file implements client-side bundling, mostly wrapping esbuild.
export async function bundleClientJavaScript( export async function bundleClientJavaScript(
referencedScripts: string[], io: Io,
extraPublicScripts: string[], { clientRefs, extraPublicScripts, dev = false }: {
incr: Incremental, clientRefs: string[];
dev: boolean = false, extraPublicScripts: string[];
dev: boolean;
},
) { ) {
const entryPoints = [ const entryPoints = [
...new Set([ ...new Set([
...referencedScripts.map((file) => path.resolve(hot.projectSrc, file)), ...clientRefs.map((x) => `src/${x}`),
...extraPublicScripts, ...extraPublicScripts,
]), ].map(toAbs)),
]; ];
if (entryPoints.length === 0) return; if (entryPoints.length === 0) return {};
const invalidFiles = entryPoints const invalidFiles = entryPoints
.filter((file) => !file.match(/\.client\.[tj]sx?/)); .filter((file) => !file.match(/\.client\.[tj]sx?/));
if (invalidFiles.length > 0) { if (invalidFiles.length > 0) {
@ -24,7 +32,7 @@ export async function bundleClientJavaScript(
const clientPlugins: esbuild.Plugin[] = [ const clientPlugins: esbuild.Plugin[] = [
projectRelativeResolution(), projectRelativeResolution(),
markoViaBuildCache(incr), markoViaBuildCache(),
]; ];
const bundle = await esbuild.build({ const bundle = await esbuild.build({
@ -65,78 +73,81 @@ export async function bundleClientJavaScript(
) )
); );
const { metafile, outputFiles } = bundle; const { metafile, outputFiles } = bundle;
const promises: Promise<void>[] = []; const p = []
p.push(trackEsbuild(io, metafile));
const scripts: Record<string, string> = {};
for (const file of outputFiles) { for (const file of outputFiles) {
const { text } = file; const { text } = file;
let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/"); let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
const { inputs } = UNWRAP(metafile.outputs["out!" + route]); const { inputs } = UNWRAP(metafile.outputs["out!" + route]);
const sources = Object.keys(inputs) const sources = Object.keys(inputs).filter((x) => !isIgnoredSource(x));
.filter((x) => !x.startsWith("<define:"));
// Register non-chunks as script entries. // Register non-chunks as script entries.
const chunk = route.startsWith("/js/c."); const chunk = route.startsWith("/js/c.");
if (!chunk) { if (!chunk) {
const key = hot.getScriptId(path.resolve(sources[sources.length - 1])); const key = hot.getScriptId(path.resolve(sources[sources.length - 1]));
route = "/js/" + key.replace(/\.client\.tsx?/, ".js"); route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
incr.put({ scripts[key] = text;
sources,
kind: "script",
key,
value: text,
});
} }
// Register chunks and public scripts as assets. // Register chunks and public scripts as assets.
if (chunk || publicScriptRoutes.includes(route)) { if (chunk || publicScriptRoutes.includes(route)) {
promises.push(incr.putAsset({ p.push(io.writeAsset(route, text));
sources,
key: route,
body: text,
}));
} }
} }
await Promise.all(promises); await Promise.all(p);
return scripts;
} }
export type ServerPlatform = "node" | "passthru"; export type ServerPlatform = "node" | "passthru";
export interface ServerSideOptions {
entries: string[],
viewItems: sg.FileItem[]
viewRefs: incr.Ref<PreparedView>[],
styleMap: Map<string, incr.Ref<string>>;
scriptMap: incr.Ref<Record<string, string>>;
platform: ServerPlatform,
}
export async function bundleServerJavaScript( export async function bundleServerJavaScript(
incr: Incremental, { viewItems, viewRefs, styleMap, scriptMap: wScriptMap, entries, platform }: ServerSideOptions
platform: ServerPlatform = "node",
) { ) {
if (incr.hasArtifact("backendBundle", platform)) return; const wViewSource = incr.work(async (_, viewItems: sg.FileItem[]) => {
// Comment
const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_"); const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
return {
const viewSource = [ magicWord,
...Array.from( file: [
incr.out.viewMetadata, ...viewItems.map((view, i) => `import * as view${i} from ${JSON.stringify(view.file)}`),
([, view], i) => `import * as view${i} from ${JSON.stringify(view.file)}`,
),
`const styles = ${magicWord}[-2]`, `const styles = ${magicWord}[-2]`,
`export const scripts = ${magicWord}[-1]`, `export const scripts = ${magicWord}[-1]`,
"export const views = {", "export const views = {",
...Array.from(incr.out.viewMetadata, ([key, view], i) => ...viewItems.map((view, i) => [
[ ` ${JSON.stringify(view.id)}: {`,
` ${JSON.stringify(key)}: {`,
` component: view${i}.default,`, ` component: view${i}.default,`,
// ` meta: ${
// view.staticMeta ? JSON.stringify(view.staticMeta) : `view${i}.meta`
// },`,
` meta: view${i}.meta,`, ` meta: view${i}.meta,`,
` layout: ${view.hasLayout ? `view${i}.layout?.default` : "null"},`, ` layout: view${i}.layout?.default ?? null,`,
` inlineCss: styles[${magicWord}[${i}]]`, ` inlineCss: styles[${magicWord}[${i}]]`,
` },`, ` },`,
].join("\n")), ].join("\n")),
"}", "}",
].join("\n"); ].join("\n")
};
}, viewItems)
const wBundles = entries.map(entry => [entry, incr.work(async (io, entry) => {
const pkg = await io.readJson<{ dependencies: Record<string, string>; }>("package.json");
let magicWord = null as string | null;
// -- plugins -- // -- plugins --
const serverPlugins: esbuild.Plugin[] = [ const serverPlugins: esbuild.Plugin[] = [
virtualFiles({ virtualFiles({
"$views": viewSource, // only add dependency when imported.
"$views": async () => {
const view = await io.readWork(wViewSource);
({ magicWord } = view);
return view.file;
},
}), }),
projectRelativeResolution(), projectRelativeResolution(),
markoViaBuildCache(incr), markoViaBuildCache(),
{ {
name: "replace client references", name: "replace client references",
setup(b) { setup(b) {
@ -161,13 +172,11 @@ export async function bundleServerJavaScript(
}, },
}, },
]; ];
const pkg = await fs.readJson("package.json") as {
dependencies: Record<string, string>; const { metafile, outputFiles, errors, warnings } = await esbuild.build({
};
const { metafile, outputFiles } = await esbuild.build({
bundle: true, bundle: true,
chunkNames: "c.[hash]", chunkNames: "c.[hash]",
entryNames: "server", entryNames: path.basename(entry, path.extname(entry)),
entryPoints: [ entryPoints: [
path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"), path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
], ],
@ -185,71 +194,60 @@ export async function bundleServerJavaScript(
jsxDev: false, jsxDev: false,
define: { define: {
MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText), MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
'globalThis.CLOVER_SERVER_ENTRY': JSON.stringify(entry),
}, },
external: Object.keys(pkg.dependencies) external: Object.keys(pkg.dependencies)
.filter((x) => !x.startsWith("@paperclover")), .filter((x) => !x.startsWith("@paperclover")),
}); });
await trackEsbuild(io, metafile)
const files: Record<string, Buffer> = {}; let fileWithMagicWord: {
let fileWithMagicWord: string | null = null; bytes: Buffer;
basename: string;
magicWord: string;
} | null = null;
for (const output of outputFiles) { for (const output of outputFiles) {
const basename = output.path.replace(/^.*?!/, ""); const basename = output.path.replace(/^.*?!(?:\/|\\)/, "");
const key = "out!" + basename.replaceAll("\\", "/"); const key = "out!/" + basename.replaceAll("\\", "/");
// If this contains the generated "$views" file, then // If this contains the generated "$views" file, then
// mark this file as the one for replacement. Because // mark this file as the one for replacement. Because
// `splitting` is `true`, esbuild will not emit this // `splitting` is `true`, esbuild will not emit this
// file in more than one chunk. // file in more than one chunk.
if (metafile.outputs[key].inputs["framework/lib/view.ts"]) { if (magicWord && metafile.outputs[key].inputs["framework/lib/view.ts"]) {
fileWithMagicWord = basename; ASSERT(!fileWithMagicWord);
} fileWithMagicWord = {
files[basename] = Buffer.from(output.contents); basename,
} bytes: Buffer.from(output.contents),
incr.put({
kind: "backendBundle",
key: platform,
value: {
magicWord, magicWord,
files, };
fileWithMagicWord, } else {
}, io.writeFile(basename, Buffer.from(output.contents))
sources: Object.keys(metafile.inputs).filter((x) => }
!x.includes("<define:") && }
!x.startsWith("vfs:") && return fileWithMagicWord;
!x.startsWith("dropped:") && }, entry)] as const);
!x.includes("node_modules")
),
});
}
export async function finalizeServerJavaScript( const wProcessed = wBundles.map(async([entry, wBundle]) => {
incr: Incremental, if (!await wBundle) return;
platform: ServerPlatform, await incr.work(async (io) => {
) { // Only the reachable resources need to be read and inserted into the bundle.
if (incr.hasArtifact("backendReplace", platform)) return; // This is what Map<string, incr.Ref> is for
const { const { basename, bytes, magicWord } = UNWRAP(await io.readWork(wBundle));
files, const views = await Promise.all(viewRefs.map(ref => io.readWork(ref)));
fileWithMagicWord,
magicWord,
} = UNWRAP(incr.getArtifact("backendBundle", platform));
if (!fileWithMagicWord) return; // Client JS
const scriptList = Object.entries(await io.readWork(wScriptMap));
const viewScriptsList = new Set(views.flatMap(view => view.clientRefs));
const neededScripts = scriptList.filter(([k]) => viewScriptsList.has(k));
// Only the reachable resources need to be inserted into the bundle. // CSS
const viewScriptsList = new Set( const viewStyleKeys = views.map((view) => view.styleKey);
Array.from(incr.out.viewMetadata.values()) const viewCssBundles = await Promise.all(
.flatMap((view) => view.clientRefs), viewStyleKeys.map((key) => io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key))));
);
const viewStyleKeys = Array.from(incr.out.viewMetadata.values())
.map((view) => css.styleKey(view.cssImports, view.theme));
const viewCssBundles = viewStyleKeys
.map((key) => UNWRAP(incr.out.style.get(key), "Style key: " + key));
// Deduplicate styles
const styleList = Array.from(new Set(viewCssBundles)); const styleList = Array.from(new Set(viewCssBundles));
// Replace the magic word // Replace the magic word
let text = files[fileWithMagicWord].toString("utf-8"); const text = bytes.toString("utf-8").replace(
text = text.replace(
new RegExp(magicWord + "\\[(-?\\d+)\\]", "gs"), new RegExp(magicWord + "\\[(-?\\d+)\\]", "gs"),
(_, i) => { (_, i) => {
i = Number(i); i = Number(i);
@ -259,62 +257,34 @@ export async function finalizeServerJavaScript(
} }
// Inline the script data // Inline the script data
if (i === -1) { if (i === -1) {
return JSON.stringify(Object.fromEntries(incr.out.script)); return JSON.stringify(Object.fromEntries(neededScripts));
} }
// Reference an index into `styleList` // Reference an index into `styleList`
return `${styleList.indexOf(viewCssBundles[i])}`; return `${styleList.indexOf(viewCssBundles[i])}`;
}, },
); );
incr.put({ io.writeFile(basename, text);
kind: "backendReplace",
key: platform,
sources: [
// Backend input code (includes view code)
...incr.sourcesFor("backendBundle", platform),
// Script
...Array.from(viewScriptsList)
.flatMap((key) => incr.sourcesFor("script", hot.getScriptId(key))),
// Style
...viewStyleKeys.flatMap((key) => incr.sourcesFor("style", key)),
],
value: Buffer.from(text),
}); });
})
await Promise.all(wProcessed);
} }
function markoViaBuildCache(incr: Incremental): esbuild.Plugin {
return {
name: "marko via build cache",
setup(b) {
b.onLoad(
{ filter: /\.marko$/ },
async ({ path: file }) => {
const key = path.relative(hot.projectRoot, file)
.replaceAll("\\", "/");
const cacheEntry = incr.out.serverMarko.get(key);
if (!cacheEntry) {
if (!fs.existsSync(file)) {
console.log(`File does not exist: ${file}`);
}
throw new Error("Marko file not in cache: " + file);
}
return ({
loader: "ts",
contents: cacheEntry.src,
resolveDir: path.dirname(file),
});
},
);
},
};
}
import * as esbuild from "esbuild"; import * as esbuild from "esbuild";
import * as path from "node:path"; import * as path from "node:path";
import process from "node:process"; import process from "node:process";
import * as hot from "./hot.ts"; import * as hot from "./hot.ts";
import { projectRelativeResolution, virtualFiles } from "./esbuild-support.ts"; import {
import { Incremental } from "./incremental.ts"; isIgnoredSource,
markoViaBuildCache,
projectRelativeResolution,
virtualFiles,
} from "./esbuild-support.ts";
import { Io, toAbs, toRel } from "./incremental.ts";
import * as css from "./css.ts"; import * as css from "./css.ts";
import * as fs from "#sitegen/fs"; import * as fs from "#sitegen/fs";
import * as mime from "#sitegen/mime"; import * as mime from "#sitegen/mime";
import * as incr from "./incremental.ts";
import * as sg from "#sitegen";import type { PreparedView } from "./generate2.ts";import { meta } from "@/file-viewer/pages/file.cotyledon_speedbump.tsx";

22
framework/css.ts
View file

@ -40,11 +40,6 @@ export function preprocess(css: string, theme: Theme): string {
); );
} }
export interface Output {
text: string;
sources: string[];
}
export function styleKey( export function styleKey(
cssImports: string[], cssImports: string[],
theme: Theme, theme: Theme,
@ -60,11 +55,14 @@ export function styleKey(
} }
export async function bundleCssFiles( export async function bundleCssFiles(
io: Io,
{ cssImports, theme, dev }: {
cssImports: string[], cssImports: string[],
theme: Theme, theme: Theme,
dev: boolean = false, dev: boolean,
): Promise<Output> { }
cssImports = cssImports.map((file) => path.resolve(hot.projectSrc, file)); ) {
cssImports = await Promise.all(cssImports.map((file) => io.trackFile('src/' + file)));
const plugin = { const plugin = {
name: "clover css", name: "clover css",
setup(b) { setup(b) {
@ -106,15 +104,11 @@ export async function bundleCssFiles(
throw new AggregateError(warnings, "CSS Build Failed"); throw new AggregateError(warnings, "CSS Build Failed");
} }
if (outputFiles.length > 1) throw new Error("Too many output files"); if (outputFiles.length > 1) throw new Error("Too many output files");
return { return outputFiles[0].text;
text: outputFiles[0].text,
sources: Object.keys(metafile.outputs["$input$.css"].inputs)
.filter((x) => !x.startsWith("vfs:")),
};
} }
import * as esbuild from "esbuild"; import * as esbuild from "esbuild";
import * as fs from "#sitegen/fs"; import * as fs from "#sitegen/fs";
import * as hot from "./hot.ts"; import * as hot from "./hot.ts";
import * as path from "node:path"; import * as path from "node:path";
import { virtualFiles } from "./esbuild-support.ts"; import { virtualFiles } from "./esbuild-support.ts";import type { Io } from "./incremental.ts";

9
framework/debug.safe.ts
View file

@ -6,12 +6,7 @@ globalThis.UNWRAP = (t, ...args) => {
} }
return t; return t;
}; };
globalThis.ASSERT = (t, ...args) => { globalThis.ASSERT = assert.ok;
if (!t) {
throw new Error(
args.length > 0 ? util.format(...args) : "Assertion Failed",
);
}
};
import * as util from "node:util"; import * as util from "node:util";
import * as assert from 'node:assert'

2
framework/definitions.d.ts vendored
View file

@ -1,4 +1,4 @@
declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T; declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
declare function ASSERT(value: unknown, ...log: unknown[]): asserts value; declare function ASSERT(value: unknown, message?: string): asserts value;
type Timer = ReturnType<typeof setTimeout>; type Timer = ReturnType<typeof setTimeout>;

4
framework/engine/ssr.ts
View file

@ -13,6 +13,7 @@ export function ssrSync<A extends Addons>(node: Node, addon: A = {} as A) {
const resolved = resolveNode(r, node); const resolved = resolveNode(r, node);
return { text: renderNode(resolved), addon }; return { text: renderNode(resolved), addon };
} }
export { ssrSync as sync };
export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) { export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
const r = initRender(true, addon); const r = initRender(true, addon);
@ -20,7 +21,7 @@ export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
if (r.async === 0) { if (r.async === 0) {
return Promise.resolve({ text: renderNode(resolved), addon }); return Promise.resolve({ text: renderNode(resolved), addon });
} }
const { resolve, reject, promise } = Promise.withResolvers<Result>(); const { resolve, reject, promise } = Promise.withResolvers<Result<A>>();
r.asyncDone = () => { r.asyncDone = () => {
const rejections = r.rejections; const rejections = r.rejections;
if (!rejections) return resolve({ text: renderNode(resolved), addon }); if (!rejections) return resolve({ text: renderNode(resolved), addon });
@ -29,6 +30,7 @@ export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
}; };
return promise; return promise;
} }
export { ssrAsync as async };
/** Inline HTML into a render without escaping it */ /** Inline HTML into a render without escaping it */
export function html(rawText: ResolvedNode): DirectHtml { export function html(rawText: ResolvedNode): DirectHtml {

46
framework/esbuild-support.ts
View file

@ -1,5 +1,7 @@
type Awaitable<T> = T | Promise<T>;
export function virtualFiles( export function virtualFiles(
map: Record<string, string | esbuild.OnLoadResult>, map: Record<string, string | esbuild.OnLoadResult | (() => Awaitable<string | esbuild.OnLoadResult>)>,
) { ) {
return { return {
name: "clover vfs", name: "clover vfs",
@ -18,8 +20,9 @@ export function virtualFiles(
); );
b.onLoad( b.onLoad(
{ filter: /./, namespace: "vfs" }, { filter: /./, namespace: "vfs" },
({ path }) => { async ({ path }) => {
const entry = map[path]; let entry = map[path];
if (typeof entry === 'function') entry = await entry();
return ({ return ({
resolveDir: ".", resolveDir: ".",
loader: "ts", loader: "ts",
@ -73,7 +76,42 @@ export function projectRelativeResolution(root = process.cwd() + "/src") {
} satisfies esbuild.Plugin; } satisfies esbuild.Plugin;
} }
export function markoViaBuildCache(): esbuild.Plugin {
return {
name: "marko via build cache",
setup(b) {
b.onLoad(
{ filter: /\.marko$/ },
async ({ path: file }) => {
const cacheEntry = markoCache.get(file);
if (!cacheEntry) {
if (!fs.existsSync(file)) {
console.warn(`File does not exist: ${file}`);
}
console.log(markoCache.keys());
throw new Error("Marko file not in cache: " + file);
}
return ({
loader: "ts",
contents: cacheEntry.src,
resolveDir: path.dirname(file),
});
},
);
},
};
}
export function isIgnoredSource(source: string) {
return source.includes("<define:") ||
source.startsWith("vfs:") ||
source.startsWith("dropped:") ||
source.includes("node_modules")
}
import * as esbuild from "esbuild"; import * as esbuild from "esbuild";
import * as string from "#sitegen/string"; import * as string from "#sitegen/string";
import * as path from "node:path"; import * as path from "node:path";
import * as hot from "./hot.ts"; import * as fs from "#sitegen/fs";
import * as incr from "./incremental.ts";
import * as hot from "./hot.ts";import { markoCache } from "./marko.ts";

471
framework/generate.ts
View file

@ -1,51 +1,99 @@
// This file contains the main site generator build process. // This file contains the main site generator build process.
// By using `Incremental`'s ability to automatically purge stale // By using `incr.work`'s ability to cache work between runs,
// assets, the `sitegen` function performs partial rebuilds. // the site generator is very fast to re-run.
//
// See `watch.ts` for a live development environment.
const { toRel, toAbs } = incr;
const globalCssPath = toAbs("src/global.css");
export function main() { export async function main() {
return withSpinner<Record<string, unknown>, any>({ await incr.restore();
text: "Recovering State", await incr.compile(generate);
successText,
failureText: () => "sitegen FAIL",
}, async (spinner) => {
// const incr = Incremental.fromDisk();
// await incr.statAllFiles();
const incr = new Incremental();
const result = await sitegen(spinner, incr);
incr.toDisk(); // Allows picking up this state again
return result;
}) as ReturnType<typeof sitegen>;
} }
export function successText({ export async function generate() {
elapsed, // -- read config and discover files --
inserted, const siteConfig = await incr.work(readManifest);
referenced, const {
unreferenced, staticFiles,
}: Awaited<ReturnType<typeof sitegen>>) { scripts,
const s = (array: unknown[]) => array.length === 1 ? "" : "s"; views,
const kind = inserted.length === referenced.length ? "build" : "update"; pages,
const status = inserted.length > 0 } = await discoverAllFiles(siteConfig);
? `${kind} ${inserted.length} key${s(inserted)}`
: unreferenced.length > 0 // TODO: make sure that `static` and `pages` does not overlap
? `pruned ${unreferenced.length} key${s(unreferenced)}`
: `checked ${referenced.length} key${s(referenced)}`; // TODO: loadMarkoCache
return `sitegen! ${status} in ${elapsed.toFixed(1)}s`;
// -- perform build-time rendering --
const builtPages = pages.map((item) => incr.work(preparePage, item));
const builtViews = views.map((item) => incr.work(prepareView, item));
const builtStaticFiles = Promise.all((staticFiles.map((item) =>
incr.work(
async (io, { id, file }) => void await io.writeAsset(id, await io.readFile(file)),
item,
)
)));
const routes = await Promise.all([...builtViews, ...builtPages]);
// -- page resources --
const scriptMap = incr.work(bundle.bundleClientJavaScript, {
clientRefs: routes.flatMap((x) => x.clientRefs),
extraPublicScripts: scripts.map((entry) => entry.file),
dev: false,
});
const styleMap = prepareInlineCss(routes);
// -- backend --
const builtBackend = bundle.bundleServerJavaScript({
entries: siteConfig.backends,
platform: 'node',
styleMap,
scriptMap,
viewItems: views,
viewRefs: builtViews,
})
// -- assemble page assets --
const pAssemblePages = builtPages.map((page) =>
assembleAndWritePage(page, styleMap, scriptMap)
);
await Promise.all([
builtBackend,
builtStaticFiles,
...pAssemblePages,
]);
} }
export async function sitegen( export async function readManifest(io: Io) {
status: Spinner, const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
incr: Incremental, return {
siteSections: cfg.siteSections.map((section) => ({
root: toRel(section.root),
})),
backends: cfg.backends.map(toRel),
};
}
export async function discoverAllFiles(
siteConfig: Awaited<ReturnType<typeof readManifest>>,
) { ) {
const startTime = performance.now(); return (
await Promise.all(
let root = path.resolve(import.meta.dirname, "../src"); siteConfig.siteSections.map(({ root: sectionRoot }) =>
const join = (...sub: string[]) => path.join(root, ...sub); incr.work(scanSiteSection, toAbs(sectionRoot))
),
// Sitegen reviews every defined section for resources to process )
const sections: sg.Section[] = ).reduce((acc, next) => ({
require(path.join(root, "site.ts")).siteSections; staticFiles: acc.staticFiles.concat(next.staticFiles),
pages: acc.pages.concat(next.pages),
views: acc.views.concat(next.views),
scripts: acc.scripts.concat(next.scripts),
}));
}
export async function scanSiteSection(io: Io, sectionRoot: string) {
// Static files are compressed and served as-is. // Static files are compressed and served as-is.
// - "{section}/static/*.png" // - "{section}/static/*.png"
let staticFiles: FileItem[] = []; let staticFiles: FileItem[] = [];
@ -61,14 +109,10 @@ export async function sitegen(
// - "{section}/scripts/*.client.ts" // - "{section}/scripts/*.client.ts"
let scripts: FileItem[] = []; let scripts: FileItem[] = [];
// -- Scan for files --
status.text = "Scanning Project";
for (const section of sections) {
const { root: sectionRoot } = section;
const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub); const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
const rootPrefix = root === sectionRoot const rootPrefix = hot.projectSrc === sectionRoot
? "" ? ""
: path.relative(root, sectionRoot) + "/"; : path.relative(hot.projectSrc, sectionRoot) + "/";
const kinds = [ const kinds = [
{ {
dir: sectionPath("pages"), dir: sectionPath("pages"),
@ -97,11 +141,23 @@ export async function sitegen(
exclude: [".client.ts", ".client.tsx"], exclude: [".client.ts", ".client.tsx"],
}, },
]; ];
for ( for (const kind of kinds) {
const { dir, list, prefix, include = [""], exclude = [], ext = false } const {
of kinds dir,
) { list,
const items = fs.readDirRecOptionalSync(dir); prefix,
include = [""],
exclude = [],
ext = false,
} = kind;
let items;
try {
items = await io.readDirRecursive(dir);
} catch (err: any) {
if (err.code === "ENOENT") continue;
throw err;
}
for (const subPath of items) { for (const subPath of items) {
const file = path.join(dir, subPath); const file = path.join(dir, subPath);
const stat = fs.statSync(file); const stat = fs.statSync(file);
@ -110,67 +166,26 @@ export async function sitegen(
if (exclude.some((e) => subPath.endsWith(e))) continue; if (exclude.some((e) => subPath.endsWith(e))) continue;
const trim = ext const trim = ext
? subPath ? subPath
: subPath.slice(0, -path.extname(subPath).length).replaceAll( : subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
".",
"/",
);
let id = prefix + trim.replaceAll("\\", "/"); let id = prefix + trim.replaceAll("\\", "/");
if (prefix === "/" && id.endsWith("/index")) { if (prefix === "/" && id.endsWith("/index")) {
id = id.slice(0, -"/index".length) || "/"; id = id.slice(0, -"/index".length) || "/";
} }
list.push({ id, file: file }); list.push({ id, file: path.relative(hot.projectRoot, file) });
} }
} }
}
const globalCssPath = join("global.css");
// TODO: make sure that `static` and `pages` does not overlap
// -- inline style sheets, used and shared by pages and views --
status.text = "Building";
const cssOnce = new OnceMap();
const cssQueue = new Queue({
name: "Bundle",
async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
const { text, sources } = await css.bundleCssFiles(files, theme);
incr.put({
kind: "style",
key,
sources,
value: text,
});
},
passive: true,
getItemText: ([id]) => id,
maxJobs: 2,
});
function ensureCssGetsBuilt(
cssImports: string[],
theme: css.Theme,
referrer: string,
) {
const key = css.styleKey(cssImports, theme);
cssOnce.get(
key,
async () => {
incr.getArtifact("style", key) ??
await cssQueue.add([referrer, key, cssImports, theme]);
},
);
}
// -- server side render pages -- return { staticFiles, pages, views, scripts };
async function loadPageModule({ file }: FileItem) { }
require(file);
} export async function preparePage(io: Io, item: sg.FileItem) {
async function renderPage(item: FileItem) {
// -- load and validate module -- // -- load and validate module --
let { let {
default: Page, default: Page,
meta: metadata, meta: metadata,
theme: pageTheme, theme: pageTheme,
layout, layout,
} = require(item.file); } = await io.import<any>(item.file);
if (!Page) { if (!Page) {
throw new Error("Page is missing a 'default' export."); throw new Error("Page is missing a 'default' export.");
} }
@ -188,7 +203,6 @@ export async function sitegen(
new Set([globalCssPath, ...hot.getCssImports(item.file)]), new Set([globalCssPath, ...hot.getCssImports(item.file)]),
(file) => path.relative(hot.projectSrc, file), (file) => path.relative(hot.projectSrc, file),
); );
ensureCssGetsBuilt(cssImports, theme, item.id);
// -- metadata -- // -- metadata --
const renderedMetaPromise = Promise.resolve( const renderedMetaPromise = Promise.resolve(
@ -210,27 +224,23 @@ export async function sitegen(
]); ]);
if (!renderedMeta.includes("<title>")) { if (!renderedMeta.includes("<title>")) {
throw new Error( throw new Error(
"Page is missing 'meta.title'. " + "Page is missing 'meta.title'. " + "All pages need a title tag.",
"All pages need a title tag.",
); );
} }
incr.put({ const styleKey = css.styleKey(cssImports, theme);
kind: "pageMetadata", return {
key: item.id, id: item.id,
// Incremental integrates with `hot.ts` + `require`
// to trace all the needed source files here.
sources: [item.file],
value: {
html: text, html: text,
meta: renderedMeta, meta: renderedMeta,
cssImports, cssImports,
theme: theme ?? null, theme: theme ?? null,
styleKey,
clientRefs: Array.from(addon.sitegen.scripts), clientRefs: Array.from(addon.sitegen.scripts),
}, };
}); }
}
async function prepareView(item: FileItem) { export async function prepareView(io: Io, item: sg.FileItem) {
const module = require(item.file); const module = await io.import<any>(item.file);
if (!module.meta) { if (!module.meta) {
throw new Error(`${item.file} is missing 'export const meta'`); throw new Error(`${item.file} is missing 'export const meta'`);
} }
@ -246,209 +256,80 @@ export async function sitegen(
new Set([globalCssPath, ...hot.getCssImports(item.file)]), new Set([globalCssPath, ...hot.getCssImports(item.file)]),
(file) => path.relative(hot.projectSrc, file), (file) => path.relative(hot.projectSrc, file),
); );
ensureCssGetsBuilt(cssImports, theme, item.id); const styleKey = css.styleKey(cssImports, theme);
incr.put({ return {
kind: "viewMetadata",
key: item.id,
sources: [item.file],
value: {
file: path.relative(hot.projectRoot, item.file), file: path.relative(hot.projectRoot, item.file),
cssImports, cssImports,
theme, theme,
clientRefs: hot.getClientScriptRefs(item.file), clientRefs: hot.getClientScriptRefs(item.file),
hasLayout: !!module.layout?.default, hasLayout: !!module.layout?.default,
}, styleKey,
}); };
} }
export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
// Of the pages that are already built, a call to 'ensureCssGetsBuilt' is export function prepareInlineCss(
// required so that it's (1) re-built if needed, (2) not pruned from build. items: Array<{
const neededPages = pages.filter((page) => { styleKey: string;
const existing = incr.getArtifact("pageMetadata", page.id); cssImports: string[];
if (existing) { theme: css.Theme;
const { cssImports, theme } = existing; }>,
ensureCssGetsBuilt(cssImports, theme, page.id); ) {
} const map = new Map<string, incr.Ref<string>>();
return !existing; for (const { styleKey, cssImports, theme } of items) {
}); if (map.has(styleKey)) continue;
const neededViews = views.filter((view) => { map.set(
const existing = incr.getArtifact("viewMetadata", view.id); styleKey,
if (existing) { incr.work(css.bundleCssFiles, {
const { cssImports, theme } = existing;
ensureCssGetsBuilt(cssImports, theme, view.id);
}
return !existing;
});
// Load the marko cache before render modules are loaded
incr.loadMarkoCache();
// This is done in two passes so that a page that throws during evaluation
// will report "Load Render Module" instead of "Render Static Page".
const spinnerFormat = status.format;
status.format = () => "";
const moduleLoadQueue = new Queue({
name: "Load Render Module",
fn: loadPageModule,
getItemText,
maxJobs: 1,
});
moduleLoadQueue.addMany(neededPages);
moduleLoadQueue.addMany(neededViews);
await moduleLoadQueue.done({ method: "stop" });
const pageQueue = new Queue({
name: "Render Static Page",
fn: renderPage,
getItemText,
maxJobs: 2,
});
pageQueue.addMany(neededPages);
const viewQueue = new Queue({
name: "Build Dynamic View",
fn: prepareView,
getItemText,
maxJobs: 2,
});
viewQueue.addMany(neededViews);
const pageAndViews = [
pageQueue.done({ method: "stop" }),
viewQueue.done({ method: "stop" }),
];
await Promise.allSettled(pageAndViews);
await Promise.all(pageAndViews);
status.format = spinnerFormat;
// -- bundle server javascript (backend and views) --
status.text = "Bundle JavaScript";
incr.snapshotMarkoCache();
const serverJavaScriptPromise = bundle.bundleServerJavaScript(incr, "node");
// -- bundle client javascript --
const referencedScripts = Array.from(
new Set(
[
...pages.map((item) =>
UNWRAP(
incr.getArtifact("pageMetadata", item.id),
`Missing pageMetadata ${item.id}`,
)
),
...views.map((item) =>
UNWRAP(
incr.getArtifact("viewMetadata", item.id),
`Missing viewMetadata ${item.id}`,
)
),
].flatMap((item) => item.clientRefs),
),
(script) => path.resolve(hot.projectSrc, script),
).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
const extraPublicScripts = scripts.map((entry) => entry.file);
const clientJavaScriptPromise = bundle.bundleClientJavaScript(
referencedScripts,
extraPublicScripts,
incr,
);
await Promise.all([
serverJavaScriptPromise,
clientJavaScriptPromise,
cssQueue.done({ method: "stop" }),
]);
await bundle.finalizeServerJavaScript(incr, "node");
// -- copy/compress static files --
async function doStaticFile(item: FileItem) {
const body = await fs.readFile(item.file);
await incr.putAsset({
sources: [item.file],
key: item.id,
body,
});
}
const staticQueue = new Queue({
name: "Load Static",
fn: doStaticFile,
getItemText,
maxJobs: 16,
});
status.format = () => "";
staticQueue.addMany(
staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
);
await staticQueue.done({ method: "stop" });
status.format = spinnerFormat;
// -- concatenate static rendered pages --
status.text = `Concat Pages`;
await Promise.all(pages.map(async (page) => {
if (incr.hasArtifact("asset", page.id)) return;
const {
html,
meta,
cssImports, cssImports,
theme, theme,
clientRefs, dev: false,
} = UNWRAP(incr.out.pageMetadata.get(page.id)); }),
const scriptIds = clientRefs.map(hot.getScriptId);
const styleKey = css.styleKey(cssImports, theme);
const style = UNWRAP(
incr.out.style.get(styleKey),
`Missing style ${styleKey}`,
); );
}
return map;
}
export type PreparedPage = Awaited<ReturnType<typeof preparePage>>;
export async function assembleAndWritePage(
pageWork: incr.Ref<PreparedPage>,
styleMap: Map<string, incr.Ref<string>>,
scriptWork: incr.Ref<Record<string, string>>,
) {
const page = await pageWork;
return incr.work(
async (io, { id, html, meta, styleKey, clientRefs }) => {
const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
const scriptIds = clientRefs.map(hot.getScriptId);
const scriptMap = await io.readWork(scriptWork);
const scripts = scriptIds.map((ref) =>
UNWRAP(scriptMap[ref], `Missing script ${ref}`)
)
.map((x) => `{${x}}`).join("\n");
const doc = wrapDocument({ const doc = wrapDocument({
body: html, body: html,
head: meta, head: meta,
inlineCss: style, inlineCss,
scripts: scriptIds.map( scripts,
(ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
).map((x) => `{${x}}`).join("\n"),
}); });
await incr.putAsset({ await io.writeAsset(id, doc, {
sources: [
page.file,
...incr.sourcesFor("style", styleKey),
...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
],
key: page.id,
body: doc,
headers: {
"Content-Type": "text/html", "Content-Type": "text/html",
},
}); });
})); },
status.format = () => ""; page,
status.text = ``; );
// This will wait for all compression jobs to finish, which up
// to this point have been left as dangling promises.
await incr.wait();
const { inserted, referenced, unreferenced } = incr.shake();
// Flush the site to disk.
status.format = spinnerFormat;
status.text = `Incremental Flush`;
incr.flush("node"); // Write outputs
return {
incr,
inserted,
referenced,
unreferenced,
elapsed: (performance.now() - startTime) / 1000,
};
} }
function getItemText({ file }: FileItem) { import * as sg from "#sitegen";
return path.relative(hot.projectSrc, file).replaceAll("\\", "/"); import * as incr from "./incremental.ts";
} import { Io } from "./incremental.ts";
import { OnceMap, Queue } from "#sitegen/async";
import { Incremental } from "./incremental.ts";
import * as bundle from "./bundle.ts"; import * as bundle from "./bundle.ts";
import * as css from "./css.ts"; import * as css from "./css.ts";
import * as engine from "./engine/ssr.ts"; import * as engine from "./engine/ssr.ts";
import * as hot from "./hot.ts"; import * as hot from "./hot.ts";
import * as fs from "#sitegen/fs"; import * as fs from "#sitegen/fs";
import * as sg from "#sitegen";
import type { FileItem } from "#sitegen"; import type { FileItem } from "#sitegen";
import * as path from "node:path"; import * as path from "node:path";
import * as meta from "#sitegen/meta"; import * as meta from "#sitegen/meta";

74
framework/hot.ts
View file

@ -19,6 +19,8 @@ export const load = createRequire(
}; };
export const { cache } = load; export const { cache } = load;
load<any>("source-map-support").install({hookRequire: true});
// Register extensions by overwriting `require.extensions` // Register extensions by overwriting `require.extensions`
const require = load; const require = load;
const exts = require.extensions; const exts = require.extensions;
@ -42,8 +44,7 @@ export function getFileStat(filepath: string) {
} }
function shouldTrackPath(filename: string) { function shouldTrackPath(filename: string) {
return !filename.includes("node_modules") && return !filename.includes("node_modules");
!filename.includes(import.meta.dirname);
} }
const Module = load<typeof import("node:module")>("node:module"); const Module = load<typeof import("node:module")>("node:module");
@ -59,11 +60,12 @@ Module.prototype._compile = function (
filename, filename,
format, format,
); );
const stat = fs.statSync(filename);
if (shouldTrackPath(filename)) { if (shouldTrackPath(filename)) {
const stat = fs.statSync(filename);
const cssImportsMaybe: string[] = []; const cssImportsMaybe: string[] = [];
const imports: string[] = []; const imports: string[] = [];
for (const { filename: file, cloverClientRefs } of this.children) { for (const childModule of this.children) {
const { filename: file, cloverClientRefs } = childModule;
if (file.endsWith(".css")) cssImportsMaybe.push(file); if (file.endsWith(".css")) cssImportsMaybe.push(file);
else { else {
const child = fileStats.get(file); const child = fileStats.get(file);
@ -71,6 +73,7 @@ Module.prototype._compile = function (
const { cssImportsRecursive } = child; const { cssImportsRecursive } = child;
if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive); if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
imports.push(file); imports.push(file);
(childModule.cloverImporters ??= []).push(this);
if (cloverClientRefs && cloverClientRefs.length > 0) { if (cloverClientRefs && cloverClientRefs.length > 0) {
(this.cloverClientRefs ??= []) (this.cloverClientRefs ??= [])
.push(...cloverClientRefs); .push(...cloverClientRefs);
@ -82,7 +85,7 @@ Module.prototype._compile = function (
? Array.from(new Set(cssImportsMaybe)) ? Array.from(new Set(cssImportsMaybe))
: null, : null,
imports, imports,
lastModified: stat.mtimeMs, lastModified: Math.floor(stat.mtimeMs),
}); });
} }
return result; return result;
@ -113,7 +116,7 @@ function loadEsbuild(module: NodeJS.Module, filepath: string) {
interface LoadOptions { interface LoadOptions {
scannedClientRefs?: string[]; scannedClientRefs?: string[];
} }
function loadEsbuildCode( export function loadEsbuildCode(
module: NodeJS.Module, module: NodeJS.Module,
filepath: string, filepath: string,
src: string, src: string,
@ -139,7 +142,7 @@ function loadEsbuildCode(
import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())}; import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
import.meta.dirname = ${JSON.stringify(path.dirname(filepath))}; import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
import.meta.filename = ${JSON.stringify(filepath)}; import.meta.filename = ${JSON.stringify(filepath)};
`.trim().replace(/\n/g, "") + src; `.trim().replace(/[\n\s]/g, "") + src;
} }
src = esbuild.transformSync(src, { src = esbuild.transformSync(src, {
loader, loader,
@ -149,11 +152,12 @@ function loadEsbuildCode(
jsxImportSource: "#ssr", jsxImportSource: "#ssr",
jsxDev: true, jsxDev: true,
sourcefile: filepath, sourcefile: filepath,
sourcemap: 'inline',
}).code; }).code;
return module._compile(src, filepath, "commonjs"); return module._compile(src, filepath, "commonjs");
} }
function resolveClientRef(sourcePath: string, ref: string) { export function resolveClientRef(sourcePath: string, ref: string) {
const filePath = resolveFrom(sourcePath, ref); const filePath = resolveFrom(sourcePath, ref);
if ( if (
!filePath.endsWith(".client.ts") && !filePath.endsWith(".client.ts") &&
@ -164,44 +168,10 @@ function resolveClientRef(sourcePath: string, ref: string) {
return path.relative(projectSrc, filePath); return path.relative(projectSrc, filePath);
} }
// TODO: extract the marko compilation tools out, lazy load them let lazyMarko: typeof import('./marko.ts') | null = null;
export interface MarkoCacheEntry {
src: string;
scannedClientRefs: string[];
}
export const markoCache = new Map<string, MarkoCacheEntry>();
function loadMarko(module: NodeJS.Module, filepath: string) { function loadMarko(module: NodeJS.Module, filepath: string) {
let cache = markoCache.get(filepath); lazyMarko ??= require<typeof import('./marko.ts')>("./framework/marko.ts");
if (!cache) { lazyMarko.loadMarko(module, filepath);
let src = fs.readFileSync(filepath, "utf8");
// A non-standard thing here is Clover Sitegen implements
// its own client side scripting stuff, so it overrides
// bare client import statements to it's own usage.
const scannedClientRefs = new Set<string>();
if (src.match(/^\s*client\s+import\s+["']/m)) {
src = src.replace(
/^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
(_, src) => {
const ref = JSON.parse(`"${src.slice(1, -1)}"`);
const resolved = resolveClientRef(filepath, ref);
scannedClientRefs.add(resolved);
return `<CloverScriptInclude=${
JSON.stringify(getScriptId(resolved))
} />`;
},
) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
}
src = marko.compileSync(src, filepath).code;
src = src.replace("marko/debug/html", "#ssr/marko");
cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
markoCache.set(filepath, cache);
}
const { src, scannedClientRefs } = cache;
return loadEsbuildCode(module, filepath, src, {
scannedClientRefs,
});
} }
function loadMdx(module: NodeJS.Module, filepath: string) { function loadMdx(module: NodeJS.Module, filepath: string) {
@ -224,10 +194,14 @@ export function reloadRecursive(filepath: string) {
} }
export function unload(filepath: string) { export function unload(filepath: string) {
lazyMarko?.markoCache.delete(filepath)
filepath = path.resolve(filepath); filepath = path.resolve(filepath);
const existing = cache[filepath]; const module = cache[filepath];
if (existing) delete cache[filepath]; if (!module) return;
fileStats.delete(filepath); delete cache[filepath];
for (const importer of module.cloverImporters ?? []) {
unload(importer.filename);
}
} }
function deleteRecursiveInner(id: string, module: any) { function deleteRecursiveInner(id: string, module: any) {
@ -326,6 +300,7 @@ declare global {
interface Module { interface Module {
cloverClientRefs?: string[]; cloverClientRefs?: string[];
cloverSourceCode?: string; cloverSourceCode?: string;
cloverImporters?: Module[],
_compile( _compile(
this: NodeJS.Module, this: NodeJS.Module,
@ -343,11 +318,10 @@ declare module "node:module" {
): unknown; ): unknown;
} }
import * as fs from "./lib/fs.ts"; import * as fs from "#sitegen/fs";
import * as path from "node:path"; import * as path from "node:path";
import { pathToFileURL } from "node:url"; import { pathToFileURL } from "node:url";
import * as esbuild from "esbuild"; import * as esbuild from "esbuild";
import * as marko from "@marko/compiler";
import { createRequire } from "node:module"; import { createRequire } from "node:module";
import * as mdx from "@mdx-js/mdx"; import * as mdx from "@mdx-js/mdx";
import * as self from "./hot.ts"; import * as self from "./hot.ts";

56
framework/incremental.test.ts Normal file
View file

@ -0,0 +1,56 @@
test("trivial case", async () => {
incr.reset();
const file1 = tmpFile("example.txt");
file1.write("one");
async function compilation() {
const first = incr.work({
label: "first compute",
async run (io) {
await setTimeout(1000);
const contents = await io.readFile(file1.path);
return [contents, Math.random()] as const;
}
});
const second = incr.work({
label: "second compute",
wait: first,
async run (io) {
await setTimeout(1000);
return io.readWork(first)[0].toUpperCase();
}
});
const third = incr.work({
label: "third compute",
wait: first,
async run (io) {
await setTimeout(1000);
return io.readWork(first)[1] * 1000;
}
});
return incr.work({
label: "last compute",
wait: [second, third],
async run (io) {
await setTimeout(1000);
return {
second: io.readWork(second),
third: io.readWork(third),
}
}
});
}
const { value: first } = await incr.compile(compilation);
const { value: second } = await incr.compile(compilation);
ASSERT(first === second);
incr.forceInvalidate(file1.path);
const { value: third } = await incr.compile(compilation);
ASSERT(first !== third);
ASSERT(first[0] === third[0]);
});
import * as incr from "./incremental2.ts";
import { beforeEach, test } from "node:test";
import { tmpFile } from "#sitegen/testing";import { setTimeout } from "node:timers/promises";

1174
framework/incremental.ts
View file

File diff suppressed because it is too large Load diff

4
framework/lib/assets.ts
View file

@ -69,7 +69,7 @@ export function etagMatches(etag: string, ifNoneMatch: string) {
return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
} }
function subarrayAsset([start, end]: View) { function subarrayAsset([start, end]: BufferView) {
return assets!.buf.subarray(start, end); return assets!.buf.subarray(start, end);
} }
@ -115,6 +115,6 @@ process.on("message", (msg: any) => {
import * as fs from "#sitegen/fs"; import * as fs from "#sitegen/fs";
import type { Context, Next } from "hono"; import type { Context, Next } from "hono";
import type { StatusCode } from "hono/utils/http-status"; import type { StatusCode } from "hono/utils/http-status";
import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts"; import type { BuiltAsset, BuiltAssetMap, BufferView } from "../incremental.ts";
import { Buffer } from "node:buffer"; import { Buffer } from "node:buffer";
import * as path from "node:path"; import * as path from "node:path";

11
framework/lib/testing.ts Normal file
View file

@ -0,0 +1,11 @@
export function tmpFile(basename: string) {
const file = path.join(import.meta.dirname, '../../.clover/testing', basename);
return {
path: file,
read: fs.readFile.bind(fs, file),
write: fs.writeMkdir.bind(fs, file),
};
}
import * as path from 'node:path';
import * as fs from './fs.ts';

46
framework/marko.ts Normal file
View file

@ -0,0 +1,46 @@
console.log("MARKO");
export interface MarkoCacheEntry {
src: string;
scannedClientRefs: string[];
}
export const markoCache = new Map<string, MarkoCacheEntry>();
export function loadMarko(module: NodeJS.Module, filepath: string) {
let cache = markoCache.get(filepath);
console.log({ filepath, has: !!cache })
if (!cache) {
let src = fs.readFileSync(filepath, "utf8");
// A non-standard thing here is Clover Sitegen implements
// its own client side scripting stuff, so it overrides
// bare client import statements to it's own usage.
const scannedClientRefs = new Set<string>();
if (src.match(/^\s*client\s+import\s+["']/m)) {
src = src.replace(
/^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
(_, src) => {
const ref = JSON.parse(`"${src.slice(1, -1)}"`);
const resolved = hot.resolveClientRef(filepath, ref);
scannedClientRefs.add(resolved);
return `<CloverScriptInclude=${
JSON.stringify(hot.getScriptId(resolved))
} />`;
},
) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
}
src = marko.compileSync(src, filepath).code;
src = src.replace("marko/debug/html", "#ssr/marko");
cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
markoCache.set(filepath, cache);
}
const { src, scannedClientRefs } = cache;
return hot.loadEsbuildCode(module, filepath, src, {
scannedClientRefs,
});
}
import * as marko from "@marko/compiler";
import * as hot from "./hot.ts";
import * as fs from "#sitegen/fs";

120
framework/watch.ts
View file

@ -2,102 +2,67 @@
const debounceMilliseconds = 25; const debounceMilliseconds = 25;
let subprocess: child_process.ChildProcess | null = null;
process.on("beforeExit", () => {
subprocess?.removeListener("close", onSubprocessClose);
});
let watch: Watch;
export async function main() { export async function main() {
let subprocess: child_process.ChildProcess | null = null;
// Catch up state by running a main build. // Catch up state by running a main build.
const { incr } = await generate.main(); await incr.restore();
// ...and watch the files that cause invals. watch = new Watch(rebuild);
const watch = new Watch(rebuild); rebuild([]);
watch.add(...incr.invals.keys()); }
statusLine();
// ... and then serve it!
serve();
function serve() { function serve() {
if (subprocess) { if (subprocess) {
subprocess.removeListener("close", onSubprocessClose); subprocess.removeListener("close", onSubprocessClose);
subprocess.kill(); subprocess.kill();
} }
subprocess = child_process.fork(".clover/out/server.js", [ subprocess = child_process.fork(".clover/o/backend.js", [
"--development", "--development",
], { ], {
stdio: "inherit", stdio: "inherit",
}); });
subprocess.on("close", onSubprocessClose); subprocess.on("close", onSubprocessClose);
} }
function onSubprocessClose(code: number | null, signal: string | null) { function onSubprocessClose(code: number | null, signal: string | null) {
subprocess = null; subprocess = null;
const status = code != null ? `code ${code}` : `signal ${signal}`; const status = code != null ? `code ${code}` : `signal ${signal}`;
console.error(`Backend process exited with ${status}`); console.error(`Backend process exited with ${status}`);
} }
process.on("beforeExit", () => { async function rebuild(files: string[]) {
subprocess?.removeListener("close", onSubprocessClose); const hasInvalidated = files.length === 0
}); || (await Promise.all(files.map(incr.invalidate))).some(Boolean);
if (!hasInvalidated) return;
function rebuild(files: string[]) { incr.compile(generate.generate).then(({
files = files.map((file) => path.relative(hot.projectRoot, file)); watchFiles,
const changed: string[] = []; newOutputs,
for (const file of files) { newAssets
let mtimeMs: number | null = null; }) => {
try { const removeWatch = [...watch.files].filter(x => !watchFiles.has(x))
mtimeMs = fs.statSync(file).mtimeMs; for (const file of removeWatch) watch.remove(file);
} catch (err: any) { watch.add(...watchFiles);
if (err?.code !== "ENOENT") throw err;
}
if (incr.updateStat(file, mtimeMs)) changed.push(file);
}
if (changed.length === 0) {
console.warn("Files were modified but the 'modify' time did not change.");
return;
}
withSpinner<any, Awaited<ReturnType<typeof generate.sitegen>>>({
text: "Rebuilding",
successText: generate.successText,
failureText: () => "sitegen FAIL",
}, async (spinner) => {
console.info("---");
console.info(
"Updated" +
(changed.length === 1
? " " + changed[0]
: changed.map((file) => "\n- " + file)),
);
const result = await generate.sitegen(spinner, incr);
incr.toDisk(); // Allows picking up this state again
for (const file of watch.files) {
const relative = path.relative(hot.projectRoot, file);
if (!incr.invals.has(relative)) watch.remove(file);
}
return result;
}).then((result) => {
// Restart the server if it was changed or not running. // Restart the server if it was changed or not running.
if ( if (!subprocess || newOutputs.includes("backend.js")) {
!subprocess ||
result.inserted.some(({ kind }) => kind === "backendReplace")
) {
serve(); serve();
} else if ( } else if (subprocess && newAssets) {
subprocess &&
result.inserted.some(({ kind }) => kind === "asset")
) {
subprocess.send({ type: "clover.assets.reload" }); subprocess.send({ type: "clover.assets.reload" });
} }
return result;
}).catch((err) => { }).catch((err) => {
console.error(util.inspect(err)); console.error(util.inspect(err));
}).finally(statusLine); }).finally(statusLine);
} }
function statusLine() { function statusLine() {
console.info( console.info(
`Watching ${incr.invals.size} files \x1b[36m[last change: ${ `Watching ${watch.files.size} files `
new Date().toLocaleTimeString() + `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`,
}]\x1b[39m`,
); );
}
} }
class Watch { class Watch {
@ -174,11 +139,21 @@ class Watch {
for (const w of this.watchers) w.close(); for (const w of this.watchers) w.close();
} }
#getFiles(absPath: string, event: fs.WatchEventType) {
const files = [];
if (this.files.has(absPath)) files.push(absPath);
if (event === 'rename') {
const dir = path.dirname(absPath);
if (this.files.has(dir)) files.push(dir);
}
return files;
}
#handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) { #handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) {
if (!subPath) return; if (!subPath) return;
const file = path.join(root, subPath); const files = this.#getFiles(path.join(root, subPath), event);
if (!this.files.has(file)) return; if (files.length === 0) return;
this.stale.add(file); for(const file of files) this.stale.add(file);
const { debounce } = this; const { debounce } = this;
if (debounce !== null) clearTimeout(debounce); if (debounce !== null) clearTimeout(debounce);
this.debounce = setTimeout(() => { this.debounce = setTimeout(() => {
@ -192,6 +167,7 @@ class Watch {
import * as fs from "node:fs"; import * as fs from "node:fs";
import { withSpinner } from "@paperclover/console/Spinner"; import { withSpinner } from "@paperclover/console/Spinner";
import * as generate from "./generate.ts"; import * as generate from "./generate.ts";
import * as incr from "./incremental.ts";
import * as path from "node:path"; import * as path from "node:path";
import * as util from "node:util"; import * as util from "node:util";
import * as hot from "./hot.ts"; import * as hot from "./hot.ts";

126
package-lock.json generated
View file

@ -15,8 +15,10 @@
"hls.js": "^1.6.5", "hls.js": "^1.6.5",
"hono": "^4.7.11", "hono": "^4.7.11",
"marko": "^6.0.20", "marko": "^6.0.20",
"msgpackr": "^1.11.5",
"puppeteer": "^24.10.1", "puppeteer": "^24.10.1",
"sharp": "^0.34.2", "sharp": "^0.34.2",
"source-map-support": "^0.5.21",
"unique-names-generator": "^4.7.1", "unique-names-generator": "^4.7.1",
"vscode-oniguruma": "^2.0.1", "vscode-oniguruma": "^2.0.1",
"vscode-textmate": "^9.2.0" "vscode-textmate": "^9.2.0"
@ -1478,6 +1480,84 @@
"url": "https://opencollective.com/unified" "url": "https://opencollective.com/unified"
} }
}, },
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
"integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
"integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"darwin"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
"integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
"cpu": [
"arm"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
"integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
"cpu": [
"arm64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
"integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"linux"
]
},
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
"integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
"cpu": [
"x64"
],
"license": "MIT",
"optional": true,
"os": [
"win32"
]
},
"node_modules/@paperclover/console": { "node_modules/@paperclover/console": {
"resolved": "git+https://git.paperclover.net/clo/console.git#1a6ac2b79fdd8a21a1c57d25723975872bc07e3e", "resolved": "git+https://git.paperclover.net/clo/console.git#1a6ac2b79fdd8a21a1c57d25723975872bc07e3e",
"dependencies": { "dependencies": {
@ -3765,6 +3845,37 @@
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
"license": "MIT" "license": "MIT"
}, },
"node_modules/msgpackr": {
"version": "1.11.5",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz",
"integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==",
"license": "MIT",
"optionalDependencies": {
"msgpackr-extract": "^3.0.2"
}
},
"node_modules/msgpackr-extract": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
"integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
"hasInstallScript": true,
"license": "MIT",
"optional": true,
"dependencies": {
"node-gyp-build-optional-packages": "5.2.2"
},
"bin": {
"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
},
"optionalDependencies": {
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
}
},
"node_modules/netmask": { "node_modules/netmask": {
"version": "2.0.2", "version": "2.0.2",
"resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz", "resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz",
@ -3774,6 +3885,21 @@
"node": ">= 0.4.0" "node": ">= 0.4.0"
} }
}, },
"node_modules/node-gyp-build-optional-packages": {
"version": "5.2.2",
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
"integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
"license": "MIT",
"optional": true,
"dependencies": {
"detect-libc": "^2.0.1"
},
"bin": {
"node-gyp-build-optional-packages": "bin.js",
"node-gyp-build-optional-packages-optional": "optional.js",
"node-gyp-build-optional-packages-test": "build-test.js"
}
},
"node_modules/node-releases": { "node_modules/node-releases": {
"version": "2.0.19", "version": "2.0.19",
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz", "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",

2
package.json
View file

@ -11,8 +11,10 @@
"hls.js": "^1.6.5", "hls.js": "^1.6.5",
"hono": "^4.7.11", "hono": "^4.7.11",
"marko": "^6.0.20", "marko": "^6.0.20",
"msgpackr": "^1.11.5",
"puppeteer": "^24.10.1", "puppeteer": "^24.10.1",
"sharp": "^0.34.2", "sharp": "^0.34.2",
"source-map-support": "^0.5.21",
"unique-names-generator": "^4.7.1", "unique-names-generator": "^4.7.1",
"vscode-oniguruma": "^2.0.1", "vscode-oniguruma": "^2.0.1",
"vscode-textmate": "^9.2.0" "vscode-textmate": "^9.2.0"

146
src/file-viewer/bin/scan3.ts
View file

@ -17,9 +17,9 @@ export async function main() {
const start = performance.now(); const start = performance.now();
const timerSpinner = new Spinner({ const timerSpinner = new Spinner({
text: () => text: () =>
`paper clover's scan3 [${ `paper clover's scan3 [${((performance.now() - start) / 1000).toFixed(
((performance.now() - start) / 1000).toFixed(1) 1,
}s]`, )}s]`,
fps: 10, fps: 10,
}); });
using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() }; using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
@ -38,20 +38,23 @@ export async function main() {
qList.addMany(items.map((subPath) => path.join(absPath, subPath))); qList.addMany(items.map((subPath) => path.join(absPath, subPath)));
if (mediaFile) { if (mediaFile) {
const deleted = mediaFile.getChildren() const deleted = mediaFile
.getChildren()
.filter((child) => !items.includes(child.basename)) .filter((child) => !items.includes(child.basename))
.flatMap((child) => .flatMap((child) =>
child.kind === MediaFileKind.directory child.kind === MediaFileKind.directory
? child.getRecursiveFileChildren() ? child.getRecursiveFileChildren()
: child : child,
); );
qMeta.addMany(deleted.map((mediaFile) => ({ qMeta.addMany(
deleted.map((mediaFile) => ({
absPath: path.join(root, mediaFile.path), absPath: path.join(root, mediaFile.path),
publicPath: mediaFile.path, publicPath: mediaFile.path,
stat: null, stat: null,
mediaFile, mediaFile,
}))); })),
);
} }
return; return;
@ -96,13 +99,13 @@ export async function main() {
if ( if (
mediaFile && mediaFile &&
mediaFile.date.getTime() < stat.mtime.getTime() && mediaFile.date.getTime() < stat.mtime.getTime() &&
(Date.now() - stat.mtime.getTime()) < monthMilliseconds Date.now() - stat.mtime.getTime() < monthMilliseconds
) { ) {
date = mediaFile.date; date = mediaFile.date;
console.warn( console.warn(
`M-time on ${publicPath} was likely corrupted. ${ `M-time on ${publicPath} was likely corrupted. ${formatDate(
formatDate(mediaFile.date) mediaFile.date,
} -> ${formatDate(stat.mtime)}`, )} -> ${formatDate(stat.mtime)}`,
); );
} }
mediaFile = MediaFile.createFile({ mediaFile = MediaFile.createFile({
@ -129,7 +132,10 @@ export async function main() {
await processor.run({ absPath, stat, mediaFile, spin }); await processor.run({ absPath, stat, mediaFile, spin });
mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index))); mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
for (const dependantJob of after) { for (const dependantJob of after) {
ASSERT(dependantJob.needs > 0, `dependantJob.needs > 0, ${dependantJob.needs}`); ASSERT(
dependantJob.needs > 0,
`dependantJob.needs > 0, ${dependantJob.needs}`,
);
dependantJob.needs -= 1; dependantJob.needs -= 1;
if (dependantJob.needs == 0) qProcess.add(dependantJob); if (dependantJob.needs == 0) qProcess.add(dependantJob);
} }
@ -149,25 +155,27 @@ export async function main() {
})); }));
} }
async function queueProcessors( async function queueProcessors({
{ absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">, absPath,
) { stat,
mediaFile,
}: Omit<ProcessFileArgs, "spin">) {
const ext = mediaFile.extensionNonEmpty.toLowerCase(); const ext = mediaFile.extensionNonEmpty.toLowerCase();
let possible = processors.filter((p) => let possible = processors.filter((p) =>
p.include ? p.include.has(ext) : !p.exclude?.has(ext) p.include ? p.include.has(ext) : !p.exclude?.has(ext),
); );
if (possible.length === 0) return; if (possible.length === 0) return;
const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1; const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1;
ASSERT(hash <= 0xFFFF, `${hash.toString(16)} has no bits above 16 set`); ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
let processed = mediaFile.processed; let processed = mediaFile.processed;
// If the hash has changed, migrate the bitfield over. // If the hash has changed, migrate the bitfield over.
// This also runs when the processor hash is in it's initial 0 state. // This also runs when the processor hash is in it's initial 0 state.
const order = decodeProcessors(mediaFile.processors); const order = decodeProcessors(mediaFile.processors);
if ((processed & 0xFFFF) !== hash) { if ((processed & 0xffff) !== hash) {
const previous = order.filter((_, i) => const previous = order.filter(
(processed & (1 << (16 + i))) !== 0 (_, i) => (processed & (1 << (16 + i))) !== 0,
); );
processed = hash; processed = hash;
for (const { id, hash } of previous) { for (const { id, hash } of previous) {
@ -182,13 +190,13 @@ export async function main() {
} }
mediaFile.setProcessors( mediaFile.setProcessors(
processed, processed,
possible.map((p) => possible
p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF) .map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
).join(";"), .join(";"),
); );
} else { } else {
possible = order.map(({ id }) => possible = order.map(({ id }) =>
UNWRAP(possible.find((p) => p.id === id)) UNWRAP(possible.find((p) => p.id === id)),
); );
} }
@ -225,8 +233,9 @@ export async function main() {
async function runUndoProcessors(mediaFile: MediaFile) { async function runUndoProcessors(mediaFile: MediaFile) {
const { processed } = mediaFile; const { processed } = mediaFile;
const previous = decodeProcessors(mediaFile.processors) const previous = decodeProcessors(mediaFile.processors).filter(
.filter((_, i) => (processed & (1 << (16 + i))) !== 0); (_, i) => (processed & (1 << (16 + i))) !== 0,
);
for (const { id } of previous) { for (const { id } of previous) {
const p = processors.find((p) => p.id === id); const p = processors.find((p) => p.id === id);
if (!p) continue; if (!p) continue;
@ -244,22 +253,23 @@ export async function main() {
await qProcess.done(); await qProcess.done();
// Update directory metadata // Update directory metadata
const dirs = MediaFile.getDirectoriesToReindex() const dirs = MediaFile.getDirectoriesToReindex().sort(
.sort((a, b) => b.path.length - a.path.length); (a, b) => b.path.length - a.path.length,
);
for (const dir of dirs) { for (const dir of dirs) {
const children = dir.getChildren(); const children = dir.getChildren();
// readme.txt // readme.txt
const readmeContent = children.find((x) => const readmeContent =
x.basename === "readme.txt" children.find((x) => x.basename === "readme.txt")?.contents ?? "";
)?.contents ?? "";
// dirsort // dirsort
let dirsort: string[] | null = null; let dirsort: string[] | null = null;
const dirSortRaw = const dirSortRaw =
children.find((x) => x.basename === ".dirsort")?.contents ?? ""; children.find((x) => x.basename === ".dirsort")?.contents ?? "";
if (dirSortRaw) { if (dirSortRaw) {
dirsort = dirSortRaw.split("\n") dirsort = dirSortRaw
.split("\n")
.map((x) => x.trim()) .map((x) => x.trim())
.filter(Boolean); .filter(Boolean);
} }
@ -284,7 +294,8 @@ export async function main() {
} }
} }
const dirHash = crypto.createHash("sha1") const dirHash = crypto
.createHash("sha1")
.update(dir.path + allHashes) .update(dir.path + allHashes)
.digest("hex"); .digest("hex");
@ -323,19 +334,21 @@ export async function main() {
console.info( console.info(
"Updated file viewer index in \x1b[1m" + "Updated file viewer index in \x1b[1m" +
((performance.now() - start) / 1000).toFixed(1) + "s\x1b[0m", ((performance.now() - start) / 1000).toFixed(1) +
"s\x1b[0m",
); );
MediaFile.db.prepare("VACUUM").run(); MediaFile.db.prepare("VACUUM").run();
const { duration, count } = MediaFile.db.prepare< const { duration, count } = MediaFile.db
[], .prepare<[], { count: number; duration: number }>(
{ count: number; duration: number } `
>(`
select select
count(*) as count, count(*) as count,
sum(duration) as duration sum(duration) as duration
from media_files from media_files
`).getNonNull(); `,
)
.getNonNull();
console.info(); console.info();
console.info( console.info(
@ -365,7 +378,7 @@ const execFile: typeof execFileRaw = ((
) => ) =>
execFileRaw(...args).catch((e: any) => { execFileRaw(...args).catch((e: any) => {
if (e?.message?.startsWith?.("Command failed")) { if (e?.message?.startsWith?.("Command failed")) {
if (e.code > (2 ** 31)) e.code |= 0; if (e.code > 2 ** 31) e.code |= 0;
const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
} }
@ -374,11 +387,7 @@ const execFile: typeof execFileRaw = ((
const ffprobeBin = testProgram("ffprobe", "--help"); const ffprobeBin = testProgram("ffprobe", "--help");
const ffmpegBin = testProgram("ffmpeg", "--help"); const ffmpegBin = testProgram("ffmpeg", "--help");
const ffmpegOptions = [ const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
"-hide_banner",
"-loglevel",
"warning",
];
const procDuration: Process = { const procDuration: Process = {
name: "calculate duration", name: "calculate duration",
@ -496,13 +505,10 @@ const procImageSubsets: Process = {
for (const size of targetSizes) { for (const size of targetSizes) {
const { w, h } = resizeDimensions(width, height, size); const { w, h } = resizeDimensions(width, height, size);
for (const { ext, args } of transcodeRules.imagePresets) { for (const { ext, args } of transcodeRules.imagePresets) {
spin.text = baseStatus + spin.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
stack.use( stack.use(
await produceAsset( await produceAsset(`${mediaFile.hash}/${size}${ext}`, async (out) => {
`${mediaFile.hash}/${size}${ext}`,
async (out) => {
await fs.mkdir(path.dirname(out)); await fs.mkdir(path.dirname(out));
await fs.rm(out, { force: true }); await fs.rm(out, { force: true });
await execFile(ffmpegBin!, [ await execFile(ffmpegBin!, [
@ -515,8 +521,7 @@ const procImageSubsets: Process = {
out, out,
]); ]);
return [out]; return [out];
}, }),
),
); );
} }
} }
@ -561,21 +566,17 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
if (config.encoder && typeof config.encoder.videoSrc === "string") { if (config.encoder && typeof config.encoder.videoSrc === "string") {
const { videoSrc, audioSrc, rate } = config.encoder; const { videoSrc, audioSrc, rate } = config.encoder;
inputArgs = [ inputArgs = [
...rate ? ["-r", String(rate)] : [], ...(rate ? ["-r", String(rate)] : []),
"-i", "-i",
videoSrc, videoSrc,
...audioSrc ? ["-i", audioSrc] : [], ...(audioSrc ? ["-i", audioSrc] : []),
]; ];
} }
} catch (err: any) { } catch (err: any) {
if (err?.code !== "ENOENT") throw err; if (err?.code !== "ENOENT") throw err;
} }
const args = transcodeRules.getVideoArgs( const args = transcodeRules.getVideoArgs(preset, base, inputArgs);
preset,
base,
inputArgs,
);
try { try {
const fakeProgress = new Progress({ text: spin.text, spinner: null }); const fakeProgress = new Progress({ text: spin.text, spinner: null });
fakeProgress.stop(); fakeProgress.stop();
@ -612,7 +613,9 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
const procCompression = [ const procCompression = [
{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) }, { name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
{ name: "zstd", fn: () => zlib.createZstdCompress() }, { name: "zstd", fn: () => zlib.createZstdCompress() },
].map(({ name, fn }) => ({ ].map(
({ name, fn }) =>
({
name: `compress ${name}`, name: `compress ${name}`,
exclude: rules.extsPreCompressed, exclude: rules.extsPreCompressed,
async run({ absPath, mediaFile }) { async run({ absPath, mediaFile }) {
@ -627,7 +630,8 @@ const procCompression = [
return [base]; return [base];
}); });
}, },
} satisfies Process as Process)); }) satisfies Process as Process,
);
const processors = [ const processors = [
procDimensions, procDimensions,
@ -637,16 +641,15 @@ const processors = [
procImageSubsets, procImageSubsets,
...procVideos, ...procVideos,
...procCompression, ...procCompression,
] ].map((process, id, all) => {
.map((process, id, all) => { const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
const strIndex = (id: number) =>
String.fromCharCode("a".charCodeAt(0) + id);
return { return {
...process as Process, ...(process as Process),
id: strIndex(id), id: strIndex(id),
// Create a unique key. // Create a unique key.
hash: new Uint16Array( hash: new Uint16Array(
crypto.createHash("sha1") crypto
.createHash("sha1")
.update( .update(
process.run.toString() + process.run.toString() +
(process.version ? String(process.version) : ""), (process.version ? String(process.version) : ""),
@ -660,7 +663,7 @@ const processors = [
return strIndex(index); return strIndex(index);
}), }),
}; };
}); });
function resizeDimensions(w: number, h: number, desiredWidth: number) { function resizeDimensions(w: number, h: number, desiredWidth: number) {
ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`); ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
@ -676,10 +679,7 @@ async function produceAsset(
if (asset.refs === 1) { if (asset.refs === 1) {
const paths = await builder(path.join(workDir, key)); const paths = await builder(path.join(workDir, key));
asset.addFiles( asset.addFiles(
paths.map((file) => paths.map((file) => path.relative(workDir, file).replaceAll("\\", "/")),
path.relative(workDir, file)
.replaceAll("\\", "/")
),
); );
} }
return { return {
@ -719,7 +719,7 @@ interface ProcessJob {
absPath: string; absPath: string;
stat: fs.Stats; stat: fs.Stats;
mediaFile: MediaFile; mediaFile: MediaFile;
processor: typeof processors[0]; processor: (typeof processors)[0];
index: number; index: number;
after: ProcessJob[]; after: ProcessJob[];
needs: number; needs: number;

26
src/global.css
View file

@ -4,7 +4,9 @@
font-weight: 400 750; font-weight: 400 750;
font-style: normal; font-style: normal;
font-display: swap; font-display: swap;
font-variation-settings: "CASL" 0.25, "MONO" 0; font-variation-settings:
"CASL" 0.25,
"MONO" 0;
font-style: oblique -15deg 0deg; font-style: oblique -15deg 0deg;
unicode-range: U+0020-007E; unicode-range: U+0020-007E;
} }
@ -14,7 +16,9 @@
font-weight: 400 800; font-weight: 400 800;
font-style: normal; font-style: normal;
font-display: swap; font-display: swap;
font-variation-settings: "CASL" 0.25, "MONO" 1; font-variation-settings:
"CASL" 0.25,
"MONO" 1;
font-style: oblique -15deg 0deg; font-style: oblique -15deg 0deg;
unicode-range: U+0020-007E; unicode-range: U+0020-007E;
} }
@ -24,21 +28,13 @@
font-weight: 400 800; font-weight: 400 800;
font-style: normal; font-style: normal;
font-display: swap; font-display: swap;
font-variation-settings: "CASL" 0.25, "MONO" 1; font-variation-settings:
"CASL" 0.25,
"MONO" 1;
font-style: oblique -15deg 0deg; font-style: oblique -15deg 0deg;
unicode-range: unicode-range:
U+00C0-00FF, U+00C0-00FF, U+00A9, U+2190-2193, U+2018, U+2019, U+201C, U+201D, U+2022,
U+00A9, U+00A0-00A8, U+00AA-00BF, U+2194-2199, U+0100-017F;
U+2190-2193,
U+2018,
U+2019,
U+201C,
U+201D,
U+2022,
U+00A0-00A8,
U+00AA-00BF,
U+2194-2199,
U+0100-017F;
} }
*, *,

2
src/pages/index.marko
View file

@ -29,7 +29,7 @@ export const meta: Meta = {
<main> <main>
<div> <div>
<h2>posts</h2> <h2>posts</h2>
<p>song: <span>in the summer</span> (coming soon, 2025-07-12)</p> <p>song: <a href="/in-the-summer">in the summer</a> (2025-01-01)</p>
<p>song: <a href="/waterfalls">waterfalls</a> (2025-01-01)</p> <p>song: <a href="/waterfalls">waterfalls</a> (2025-01-01)</p>
<h2>things</h2> <h2>things</h2>
<p><a href="/q+a">questions and answers</a></p> <p><a href="/q+a">questions and answers</a></p>