rewrite incremental #21
19 changed files with 1653 additions and 1560 deletions
|
|
@ -1,17 +1,25 @@
|
|||
async function trackEsbuild(io: Io, metafile: esbuild.Metafile) {
|
||||
await Promise.all(Object.keys(metafile.inputs)
|
||||
.filter(file => !isIgnoredSource(file))
|
||||
.map(file => io.trackFile(file)));
|
||||
}
|
||||
|
||||
// This file implements client-side bundling, mostly wrapping esbuild.
|
||||
export async function bundleClientJavaScript(
|
||||
referencedScripts: string[],
|
||||
extraPublicScripts: string[],
|
||||
incr: Incremental,
|
||||
dev: boolean = false,
|
||||
io: Io,
|
||||
{ clientRefs, extraPublicScripts, dev = false }: {
|
||||
clientRefs: string[];
|
||||
extraPublicScripts: string[];
|
||||
dev: boolean;
|
||||
},
|
||||
) {
|
||||
const entryPoints = [
|
||||
...new Set([
|
||||
...referencedScripts.map((file) => path.resolve(hot.projectSrc, file)),
|
||||
...clientRefs.map((x) => `src/${x}`),
|
||||
...extraPublicScripts,
|
||||
]),
|
||||
].map(toAbs)),
|
||||
];
|
||||
if (entryPoints.length === 0) return;
|
||||
if (entryPoints.length === 0) return {};
|
||||
const invalidFiles = entryPoints
|
||||
.filter((file) => !file.match(/\.client\.[tj]sx?/));
|
||||
if (invalidFiles.length > 0) {
|
||||
|
|
@ -24,7 +32,7 @@ export async function bundleClientJavaScript(
|
|||
|
||||
const clientPlugins: esbuild.Plugin[] = [
|
||||
projectRelativeResolution(),
|
||||
markoViaBuildCache(incr),
|
||||
markoViaBuildCache(),
|
||||
];
|
||||
|
||||
const bundle = await esbuild.build({
|
||||
|
|
@ -65,78 +73,81 @@ export async function bundleClientJavaScript(
|
|||
)
|
||||
);
|
||||
const { metafile, outputFiles } = bundle;
|
||||
const promises: Promise<void>[] = [];
|
||||
const p = []
|
||||
p.push(trackEsbuild(io, metafile));
|
||||
const scripts: Record<string, string> = {};
|
||||
for (const file of outputFiles) {
|
||||
const { text } = file;
|
||||
let route = file.path.replace(/^.*!/, "").replaceAll("\\", "/");
|
||||
const { inputs } = UNWRAP(metafile.outputs["out!" + route]);
|
||||
const sources = Object.keys(inputs)
|
||||
.filter((x) => !x.startsWith("<define:"));
|
||||
const sources = Object.keys(inputs).filter((x) => !isIgnoredSource(x));
|
||||
|
||||
// Register non-chunks as script entries.
|
||||
const chunk = route.startsWith("/js/c.");
|
||||
if (!chunk) {
|
||||
const key = hot.getScriptId(path.resolve(sources[sources.length - 1]));
|
||||
route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
|
||||
incr.put({
|
||||
sources,
|
||||
kind: "script",
|
||||
key,
|
||||
value: text,
|
||||
});
|
||||
scripts[key] = text;
|
||||
}
|
||||
// Register chunks and public scripts as assets.
|
||||
if (chunk || publicScriptRoutes.includes(route)) {
|
||||
promises.push(incr.putAsset({
|
||||
sources,
|
||||
key: route,
|
||||
body: text,
|
||||
}));
|
||||
p.push(io.writeAsset(route, text));
|
||||
}
|
||||
}
|
||||
await Promise.all(promises);
|
||||
await Promise.all(p);
|
||||
return scripts;
|
||||
}
|
||||
|
||||
export type ServerPlatform = "node" | "passthru";
|
||||
export interface ServerSideOptions {
|
||||
entries: string[],
|
||||
viewItems: sg.FileItem[]
|
||||
viewRefs: incr.Ref<PreparedView>[],
|
||||
styleMap: Map<string, incr.Ref<string>>;
|
||||
scriptMap: incr.Ref<Record<string, string>>;
|
||||
platform: ServerPlatform,
|
||||
}
|
||||
export async function bundleServerJavaScript(
|
||||
incr: Incremental,
|
||||
platform: ServerPlatform = "node",
|
||||
{ viewItems, viewRefs, styleMap, scriptMap: wScriptMap, entries, platform }: ServerSideOptions
|
||||
) {
|
||||
if (incr.hasArtifact("backendBundle", platform)) return;
|
||||
|
||||
// Comment
|
||||
const wViewSource = incr.work(async (_, viewItems: sg.FileItem[]) => {
|
||||
const magicWord = "C_" + crypto.randomUUID().replaceAll("-", "_");
|
||||
|
||||
const viewSource = [
|
||||
...Array.from(
|
||||
incr.out.viewMetadata,
|
||||
([, view], i) => `import * as view${i} from ${JSON.stringify(view.file)}`,
|
||||
),
|
||||
return {
|
||||
magicWord,
|
||||
file: [
|
||||
...viewItems.map((view, i) => `import * as view${i} from ${JSON.stringify(view.file)}`),
|
||||
`const styles = ${magicWord}[-2]`,
|
||||
`export const scripts = ${magicWord}[-1]`,
|
||||
"export const views = {",
|
||||
...Array.from(incr.out.viewMetadata, ([key, view], i) =>
|
||||
[
|
||||
` ${JSON.stringify(key)}: {`,
|
||||
...viewItems.map((view, i) => [
|
||||
` ${JSON.stringify(view.id)}: {`,
|
||||
` component: view${i}.default,`,
|
||||
// ` meta: ${
|
||||
// view.staticMeta ? JSON.stringify(view.staticMeta) : `view${i}.meta`
|
||||
// },`,
|
||||
` meta: view${i}.meta,`,
|
||||
` layout: ${view.hasLayout ? `view${i}.layout?.default` : "null"},`,
|
||||
` layout: view${i}.layout?.default ?? null,`,
|
||||
` inlineCss: styles[${magicWord}[${i}]]`,
|
||||
` },`,
|
||||
].join("\n")),
|
||||
"}",
|
||||
].join("\n");
|
||||
].join("\n")
|
||||
};
|
||||
}, viewItems)
|
||||
|
||||
const wBundles = entries.map(entry => [entry, incr.work(async (io, entry) => {
|
||||
const pkg = await io.readJson<{ dependencies: Record<string, string>; }>("package.json");
|
||||
|
||||
let magicWord = null as string | null;
|
||||
// -- plugins --
|
||||
const serverPlugins: esbuild.Plugin[] = [
|
||||
virtualFiles({
|
||||
"$views": viewSource,
|
||||
// only add dependency when imported.
|
||||
"$views": async () => {
|
||||
const view = await io.readWork(wViewSource);
|
||||
({ magicWord } = view);
|
||||
return view.file;
|
||||
},
|
||||
}),
|
||||
projectRelativeResolution(),
|
||||
markoViaBuildCache(incr),
|
||||
markoViaBuildCache(),
|
||||
{
|
||||
name: "replace client references",
|
||||
setup(b) {
|
||||
|
|
@ -161,13 +172,11 @@ export async function bundleServerJavaScript(
|
|||
},
|
||||
},
|
||||
];
|
||||
const pkg = await fs.readJson("package.json") as {
|
||||
dependencies: Record<string, string>;
|
||||
};
|
||||
const { metafile, outputFiles } = await esbuild.build({
|
||||
|
||||
const { metafile, outputFiles, errors, warnings } = await esbuild.build({
|
||||
bundle: true,
|
||||
chunkNames: "c.[hash]",
|
||||
entryNames: "server",
|
||||
entryNames: path.basename(entry, path.extname(entry)),
|
||||
entryPoints: [
|
||||
path.join(import.meta.dirname, "backend/entry-" + platform + ".ts"),
|
||||
],
|
||||
|
|
@ -185,71 +194,60 @@ export async function bundleServerJavaScript(
|
|||
jsxDev: false,
|
||||
define: {
|
||||
MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText),
|
||||
'globalThis.CLOVER_SERVER_ENTRY': JSON.stringify(entry),
|
||||
},
|
||||
external: Object.keys(pkg.dependencies)
|
||||
.filter((x) => !x.startsWith("@paperclover")),
|
||||
});
|
||||
await trackEsbuild(io, metafile)
|
||||
|
||||
const files: Record<string, Buffer> = {};
|
||||
let fileWithMagicWord: string | null = null;
|
||||
let fileWithMagicWord: {
|
||||
bytes: Buffer;
|
||||
basename: string;
|
||||
magicWord: string;
|
||||
} | null = null;
|
||||
for (const output of outputFiles) {
|
||||
const basename = output.path.replace(/^.*?!/, "");
|
||||
const key = "out!" + basename.replaceAll("\\", "/");
|
||||
const basename = output.path.replace(/^.*?!(?:\/|\\)/, "");
|
||||
const key = "out!/" + basename.replaceAll("\\", "/");
|
||||
// If this contains the generated "$views" file, then
|
||||
// mark this file as the one for replacement. Because
|
||||
// `splitting` is `true`, esbuild will not emit this
|
||||
// file in more than one chunk.
|
||||
if (metafile.outputs[key].inputs["framework/lib/view.ts"]) {
|
||||
fileWithMagicWord = basename;
|
||||
}
|
||||
files[basename] = Buffer.from(output.contents);
|
||||
}
|
||||
incr.put({
|
||||
kind: "backendBundle",
|
||||
key: platform,
|
||||
value: {
|
||||
if (magicWord && metafile.outputs[key].inputs["framework/lib/view.ts"]) {
|
||||
ASSERT(!fileWithMagicWord);
|
||||
fileWithMagicWord = {
|
||||
basename,
|
||||
bytes: Buffer.from(output.contents),
|
||||
magicWord,
|
||||
files,
|
||||
fileWithMagicWord,
|
||||
},
|
||||
sources: Object.keys(metafile.inputs).filter((x) =>
|
||||
!x.includes("<define:") &&
|
||||
!x.startsWith("vfs:") &&
|
||||
!x.startsWith("dropped:") &&
|
||||
!x.includes("node_modules")
|
||||
),
|
||||
});
|
||||
}
|
||||
};
|
||||
} else {
|
||||
io.writeFile(basename, Buffer.from(output.contents))
|
||||
}
|
||||
}
|
||||
return fileWithMagicWord;
|
||||
}, entry)] as const);
|
||||
|
||||
export async function finalizeServerJavaScript(
|
||||
incr: Incremental,
|
||||
platform: ServerPlatform,
|
||||
) {
|
||||
if (incr.hasArtifact("backendReplace", platform)) return;
|
||||
const {
|
||||
files,
|
||||
fileWithMagicWord,
|
||||
magicWord,
|
||||
} = UNWRAP(incr.getArtifact("backendBundle", platform));
|
||||
const wProcessed = wBundles.map(async([entry, wBundle]) => {
|
||||
if (!await wBundle) return;
|
||||
await incr.work(async (io) => {
|
||||
// Only the reachable resources need to be read and inserted into the bundle.
|
||||
// This is what Map<string, incr.Ref> is for
|
||||
const { basename, bytes, magicWord } = UNWRAP(await io.readWork(wBundle));
|
||||
const views = await Promise.all(viewRefs.map(ref => io.readWork(ref)));
|
||||
|
||||
if (!fileWithMagicWord) return;
|
||||
// Client JS
|
||||
const scriptList = Object.entries(await io.readWork(wScriptMap));
|
||||
const viewScriptsList = new Set(views.flatMap(view => view.clientRefs));
|
||||
const neededScripts = scriptList.filter(([k]) => viewScriptsList.has(k));
|
||||
|
||||
// Only the reachable resources need to be inserted into the bundle.
|
||||
const viewScriptsList = new Set(
|
||||
Array.from(incr.out.viewMetadata.values())
|
||||
.flatMap((view) => view.clientRefs),
|
||||
);
|
||||
const viewStyleKeys = Array.from(incr.out.viewMetadata.values())
|
||||
.map((view) => css.styleKey(view.cssImports, view.theme));
|
||||
const viewCssBundles = viewStyleKeys
|
||||
.map((key) => UNWRAP(incr.out.style.get(key), "Style key: " + key));
|
||||
|
||||
// Deduplicate styles
|
||||
// CSS
|
||||
const viewStyleKeys = views.map((view) => view.styleKey);
|
||||
const viewCssBundles = await Promise.all(
|
||||
viewStyleKeys.map((key) => io.readWork(UNWRAP(styleMap.get(key), "Style key: " + key))));
|
||||
const styleList = Array.from(new Set(viewCssBundles));
|
||||
|
||||
// Replace the magic word
|
||||
let text = files[fileWithMagicWord].toString("utf-8");
|
||||
text = text.replace(
|
||||
const text = bytes.toString("utf-8").replace(
|
||||
new RegExp(magicWord + "\\[(-?\\d+)\\]", "gs"),
|
||||
(_, i) => {
|
||||
i = Number(i);
|
||||
|
|
@ -259,62 +257,34 @@ export async function finalizeServerJavaScript(
|
|||
}
|
||||
// Inline the script data
|
||||
if (i === -1) {
|
||||
return JSON.stringify(Object.fromEntries(incr.out.script));
|
||||
return JSON.stringify(Object.fromEntries(neededScripts));
|
||||
}
|
||||
// Reference an index into `styleList`
|
||||
return `${styleList.indexOf(viewCssBundles[i])}`;
|
||||
},
|
||||
);
|
||||
|
||||
incr.put({
|
||||
kind: "backendReplace",
|
||||
key: platform,
|
||||
sources: [
|
||||
// Backend input code (includes view code)
|
||||
...incr.sourcesFor("backendBundle", platform),
|
||||
// Script
|
||||
...Array.from(viewScriptsList)
|
||||
.flatMap((key) => incr.sourcesFor("script", hot.getScriptId(key))),
|
||||
// Style
|
||||
...viewStyleKeys.flatMap((key) => incr.sourcesFor("style", key)),
|
||||
],
|
||||
value: Buffer.from(text),
|
||||
io.writeFile(basename, text);
|
||||
});
|
||||
})
|
||||
|
||||
await Promise.all(wProcessed);
|
||||
}
|
||||
|
||||
function markoViaBuildCache(incr: Incremental): esbuild.Plugin {
|
||||
return {
|
||||
name: "marko via build cache",
|
||||
setup(b) {
|
||||
b.onLoad(
|
||||
{ filter: /\.marko$/ },
|
||||
async ({ path: file }) => {
|
||||
const key = path.relative(hot.projectRoot, file)
|
||||
.replaceAll("\\", "/");
|
||||
const cacheEntry = incr.out.serverMarko.get(key);
|
||||
if (!cacheEntry) {
|
||||
if (!fs.existsSync(file)) {
|
||||
console.log(`File does not exist: ${file}`);
|
||||
}
|
||||
throw new Error("Marko file not in cache: " + file);
|
||||
}
|
||||
return ({
|
||||
loader: "ts",
|
||||
contents: cacheEntry.src,
|
||||
resolveDir: path.dirname(file),
|
||||
});
|
||||
},
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
import * as esbuild from "esbuild";
|
||||
import * as path from "node:path";
|
||||
import process from "node:process";
|
||||
import * as hot from "./hot.ts";
|
||||
import { projectRelativeResolution, virtualFiles } from "./esbuild-support.ts";
|
||||
import { Incremental } from "./incremental.ts";
|
||||
import {
|
||||
isIgnoredSource,
|
||||
markoViaBuildCache,
|
||||
projectRelativeResolution,
|
||||
virtualFiles,
|
||||
} from "./esbuild-support.ts";
|
||||
import { Io, toAbs, toRel } from "./incremental.ts";
|
||||
import * as css from "./css.ts";
|
||||
import * as fs from "#sitegen/fs";
|
||||
import * as mime from "#sitegen/mime";
|
||||
import * as incr from "./incremental.ts";
|
||||
import * as sg from "#sitegen";import type { PreparedView } from "./generate2.ts";import { meta } from "@/file-viewer/pages/file.cotyledon_speedbump.tsx";
|
||||
|
|
|
|||
|
|
@ -40,11 +40,6 @@ export function preprocess(css: string, theme: Theme): string {
|
|||
);
|
||||
}
|
||||
|
||||
export interface Output {
|
||||
text: string;
|
||||
sources: string[];
|
||||
}
|
||||
|
||||
export function styleKey(
|
||||
cssImports: string[],
|
||||
theme: Theme,
|
||||
|
|
@ -60,11 +55,14 @@ export function styleKey(
|
|||
}
|
||||
|
||||
export async function bundleCssFiles(
|
||||
io: Io,
|
||||
{ cssImports, theme, dev }: {
|
||||
cssImports: string[],
|
||||
theme: Theme,
|
||||
dev: boolean = false,
|
||||
): Promise<Output> {
|
||||
cssImports = cssImports.map((file) => path.resolve(hot.projectSrc, file));
|
||||
dev: boolean,
|
||||
}
|
||||
) {
|
||||
cssImports = await Promise.all(cssImports.map((file) => io.trackFile('src/' + file)));
|
||||
const plugin = {
|
||||
name: "clover css",
|
||||
setup(b) {
|
||||
|
|
@ -106,15 +104,11 @@ export async function bundleCssFiles(
|
|||
throw new AggregateError(warnings, "CSS Build Failed");
|
||||
}
|
||||
if (outputFiles.length > 1) throw new Error("Too many output files");
|
||||
return {
|
||||
text: outputFiles[0].text,
|
||||
sources: Object.keys(metafile.outputs["$input$.css"].inputs)
|
||||
.filter((x) => !x.startsWith("vfs:")),
|
||||
};
|
||||
return outputFiles[0].text;
|
||||
}
|
||||
|
||||
import * as esbuild from "esbuild";
|
||||
import * as fs from "#sitegen/fs";
|
||||
import * as hot from "./hot.ts";
|
||||
import * as path from "node:path";
|
||||
import { virtualFiles } from "./esbuild-support.ts";
|
||||
import { virtualFiles } from "./esbuild-support.ts";import type { Io } from "./incremental.ts";
|
||||
|
|
|
|||
|
|
@ -6,12 +6,7 @@ globalThis.UNWRAP = (t, ...args) => {
|
|||
}
|
||||
return t;
|
||||
};
|
||||
globalThis.ASSERT = (t, ...args) => {
|
||||
if (!t) {
|
||||
throw new Error(
|
||||
args.length > 0 ? util.format(...args) : "Assertion Failed",
|
||||
);
|
||||
}
|
||||
};
|
||||
globalThis.ASSERT = assert.ok;
|
||||
|
||||
import * as util from "node:util";
|
||||
import * as assert from 'node:assert'
|
||||
|
|
|
|||
2
framework/definitions.d.ts
vendored
2
framework/definitions.d.ts
vendored
|
|
@ -1,4 +1,4 @@
|
|||
declare function UNWRAP<T>(value: T | null | undefined, ...log: unknown[]): T;
|
||||
declare function ASSERT(value: unknown, ...log: unknown[]): asserts value;
|
||||
declare function ASSERT(value: unknown, message?: string): asserts value;
|
||||
|
||||
type Timer = ReturnType<typeof setTimeout>;
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ export function ssrSync<A extends Addons>(node: Node, addon: A = {} as A) {
|
|||
const resolved = resolveNode(r, node);
|
||||
return { text: renderNode(resolved), addon };
|
||||
}
|
||||
export { ssrSync as sync };
|
||||
|
||||
export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
|
||||
const r = initRender(true, addon);
|
||||
|
|
@ -20,7 +21,7 @@ export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
|
|||
if (r.async === 0) {
|
||||
return Promise.resolve({ text: renderNode(resolved), addon });
|
||||
}
|
||||
const { resolve, reject, promise } = Promise.withResolvers<Result>();
|
||||
const { resolve, reject, promise } = Promise.withResolvers<Result<A>>();
|
||||
r.asyncDone = () => {
|
||||
const rejections = r.rejections;
|
||||
if (!rejections) return resolve({ text: renderNode(resolved), addon });
|
||||
|
|
@ -29,6 +30,7 @@ export function ssrAsync<A extends Addons>(node: Node, addon: A = {} as A) {
|
|||
};
|
||||
return promise;
|
||||
}
|
||||
export { ssrAsync as async };
|
||||
|
||||
/** Inline HTML into a render without escaping it */
|
||||
export function html(rawText: ResolvedNode): DirectHtml {
|
||||
|
|
|
|||
|
|
@ -1,5 +1,7 @@
|
|||
type Awaitable<T> = T | Promise<T>;
|
||||
|
||||
export function virtualFiles(
|
||||
map: Record<string, string | esbuild.OnLoadResult>,
|
||||
map: Record<string, string | esbuild.OnLoadResult | (() => Awaitable<string | esbuild.OnLoadResult>)>,
|
||||
) {
|
||||
return {
|
||||
name: "clover vfs",
|
||||
|
|
@ -18,8 +20,9 @@ export function virtualFiles(
|
|||
);
|
||||
b.onLoad(
|
||||
{ filter: /./, namespace: "vfs" },
|
||||
({ path }) => {
|
||||
const entry = map[path];
|
||||
async ({ path }) => {
|
||||
let entry = map[path];
|
||||
if (typeof entry === 'function') entry = await entry();
|
||||
return ({
|
||||
resolveDir: ".",
|
||||
loader: "ts",
|
||||
|
|
@ -73,7 +76,42 @@ export function projectRelativeResolution(root = process.cwd() + "/src") {
|
|||
} satisfies esbuild.Plugin;
|
||||
}
|
||||
|
||||
export function markoViaBuildCache(): esbuild.Plugin {
|
||||
return {
|
||||
name: "marko via build cache",
|
||||
setup(b) {
|
||||
b.onLoad(
|
||||
{ filter: /\.marko$/ },
|
||||
async ({ path: file }) => {
|
||||
const cacheEntry = markoCache.get(file);
|
||||
if (!cacheEntry) {
|
||||
if (!fs.existsSync(file)) {
|
||||
console.warn(`File does not exist: ${file}`);
|
||||
}
|
||||
console.log(markoCache.keys());
|
||||
throw new Error("Marko file not in cache: " + file);
|
||||
}
|
||||
return ({
|
||||
loader: "ts",
|
||||
contents: cacheEntry.src,
|
||||
resolveDir: path.dirname(file),
|
||||
});
|
||||
},
|
||||
);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export function isIgnoredSource(source: string) {
|
||||
return source.includes("<define:") ||
|
||||
source.startsWith("vfs:") ||
|
||||
source.startsWith("dropped:") ||
|
||||
source.includes("node_modules")
|
||||
}
|
||||
|
||||
import * as esbuild from "esbuild";
|
||||
import * as string from "#sitegen/string";
|
||||
import * as path from "node:path";
|
||||
import * as hot from "./hot.ts";
|
||||
import * as fs from "#sitegen/fs";
|
||||
import * as incr from "./incremental.ts";
|
||||
import * as hot from "./hot.ts";import { markoCache } from "./marko.ts";
|
||||
|
|
|
|||
|
|
@ -1,51 +1,99 @@
|
|||
// This file contains the main site generator build process.
|
||||
// By using `Incremental`'s ability to automatically purge stale
|
||||
// assets, the `sitegen` function performs partial rebuilds.
|
||||
// By using `incr.work`'s ability to cache work between runs,
|
||||
// the site generator is very fast to re-run.
|
||||
//
|
||||
// See `watch.ts` for a live development environment.
|
||||
const { toRel, toAbs } = incr;
|
||||
const globalCssPath = toAbs("src/global.css");
|
||||
|
||||
export function main() {
|
||||
return withSpinner<Record<string, unknown>, any>({
|
||||
text: "Recovering State",
|
||||
successText,
|
||||
failureText: () => "sitegen FAIL",
|
||||
}, async (spinner) => {
|
||||
// const incr = Incremental.fromDisk();
|
||||
// await incr.statAllFiles();
|
||||
const incr = new Incremental();
|
||||
const result = await sitegen(spinner, incr);
|
||||
incr.toDisk(); // Allows picking up this state again
|
||||
return result;
|
||||
}) as ReturnType<typeof sitegen>;
|
||||
export async function main() {
|
||||
await incr.restore();
|
||||
await incr.compile(generate);
|
||||
}
|
||||
|
||||
export function successText({
|
||||
elapsed,
|
||||
inserted,
|
||||
referenced,
|
||||
unreferenced,
|
||||
}: Awaited<ReturnType<typeof sitegen>>) {
|
||||
const s = (array: unknown[]) => array.length === 1 ? "" : "s";
|
||||
const kind = inserted.length === referenced.length ? "build" : "update";
|
||||
const status = inserted.length > 0
|
||||
? `${kind} ${inserted.length} key${s(inserted)}`
|
||||
: unreferenced.length > 0
|
||||
? `pruned ${unreferenced.length} key${s(unreferenced)}`
|
||||
: `checked ${referenced.length} key${s(referenced)}`;
|
||||
return `sitegen! ${status} in ${elapsed.toFixed(1)}s`;
|
||||
export async function generate() {
|
||||
// -- read config and discover files --
|
||||
const siteConfig = await incr.work(readManifest);
|
||||
const {
|
||||
staticFiles,
|
||||
scripts,
|
||||
views,
|
||||
pages,
|
||||
} = await discoverAllFiles(siteConfig);
|
||||
|
||||
// TODO: make sure that `static` and `pages` does not overlap
|
||||
|
||||
// TODO: loadMarkoCache
|
||||
|
||||
// -- perform build-time rendering --
|
||||
const builtPages = pages.map((item) => incr.work(preparePage, item));
|
||||
const builtViews = views.map((item) => incr.work(prepareView, item));
|
||||
const builtStaticFiles = Promise.all((staticFiles.map((item) =>
|
||||
incr.work(
|
||||
async (io, { id, file }) => void await io.writeAsset(id, await io.readFile(file)),
|
||||
item,
|
||||
)
|
||||
)));
|
||||
const routes = await Promise.all([...builtViews, ...builtPages]);
|
||||
|
||||
// -- page resources --
|
||||
const scriptMap = incr.work(bundle.bundleClientJavaScript, {
|
||||
clientRefs: routes.flatMap((x) => x.clientRefs),
|
||||
extraPublicScripts: scripts.map((entry) => entry.file),
|
||||
dev: false,
|
||||
});
|
||||
const styleMap = prepareInlineCss(routes);
|
||||
|
||||
// -- backend --
|
||||
const builtBackend = bundle.bundleServerJavaScript({
|
||||
entries: siteConfig.backends,
|
||||
platform: 'node',
|
||||
styleMap,
|
||||
scriptMap,
|
||||
viewItems: views,
|
||||
viewRefs: builtViews,
|
||||
})
|
||||
|
||||
// -- assemble page assets --
|
||||
const pAssemblePages = builtPages.map((page) =>
|
||||
assembleAndWritePage(page, styleMap, scriptMap)
|
||||
);
|
||||
|
||||
await Promise.all([
|
||||
builtBackend,
|
||||
builtStaticFiles,
|
||||
...pAssemblePages,
|
||||
]);
|
||||
}
|
||||
|
||||
export async function sitegen(
|
||||
status: Spinner,
|
||||
incr: Incremental,
|
||||
export async function readManifest(io: Io) {
|
||||
const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
|
||||
return {
|
||||
siteSections: cfg.siteSections.map((section) => ({
|
||||
root: toRel(section.root),
|
||||
})),
|
||||
backends: cfg.backends.map(toRel),
|
||||
};
|
||||
}
|
||||
|
||||
export async function discoverAllFiles(
|
||||
siteConfig: Awaited<ReturnType<typeof readManifest>>,
|
||||
) {
|
||||
const startTime = performance.now();
|
||||
|
||||
let root = path.resolve(import.meta.dirname, "../src");
|
||||
const join = (...sub: string[]) => path.join(root, ...sub);
|
||||
|
||||
// Sitegen reviews every defined section for resources to process
|
||||
const sections: sg.Section[] =
|
||||
require(path.join(root, "site.ts")).siteSections;
|
||||
return (
|
||||
await Promise.all(
|
||||
siteConfig.siteSections.map(({ root: sectionRoot }) =>
|
||||
incr.work(scanSiteSection, toAbs(sectionRoot))
|
||||
),
|
||||
)
|
||||
).reduce((acc, next) => ({
|
||||
staticFiles: acc.staticFiles.concat(next.staticFiles),
|
||||
pages: acc.pages.concat(next.pages),
|
||||
views: acc.views.concat(next.views),
|
||||
scripts: acc.scripts.concat(next.scripts),
|
||||
}));
|
||||
}
|
||||
|
||||
export async function scanSiteSection(io: Io, sectionRoot: string) {
|
||||
// Static files are compressed and served as-is.
|
||||
// - "{section}/static/*.png"
|
||||
let staticFiles: FileItem[] = [];
|
||||
|
|
@ -61,14 +109,10 @@ export async function sitegen(
|
|||
// - "{section}/scripts/*.client.ts"
|
||||
let scripts: FileItem[] = [];
|
||||
|
||||
// -- Scan for files --
|
||||
status.text = "Scanning Project";
|
||||
for (const section of sections) {
|
||||
const { root: sectionRoot } = section;
|
||||
const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
|
||||
const rootPrefix = root === sectionRoot
|
||||
const rootPrefix = hot.projectSrc === sectionRoot
|
||||
? ""
|
||||
: path.relative(root, sectionRoot) + "/";
|
||||
: path.relative(hot.projectSrc, sectionRoot) + "/";
|
||||
const kinds = [
|
||||
{
|
||||
dir: sectionPath("pages"),
|
||||
|
|
@ -97,11 +141,23 @@ export async function sitegen(
|
|||
exclude: [".client.ts", ".client.tsx"],
|
||||
},
|
||||
];
|
||||
for (
|
||||
const { dir, list, prefix, include = [""], exclude = [], ext = false }
|
||||
of kinds
|
||||
) {
|
||||
const items = fs.readDirRecOptionalSync(dir);
|
||||
for (const kind of kinds) {
|
||||
const {
|
||||
dir,
|
||||
list,
|
||||
prefix,
|
||||
include = [""],
|
||||
exclude = [],
|
||||
ext = false,
|
||||
} = kind;
|
||||
|
||||
let items;
|
||||
try {
|
||||
items = await io.readDirRecursive(dir);
|
||||
} catch (err: any) {
|
||||
if (err.code === "ENOENT") continue;
|
||||
throw err;
|
||||
}
|
||||
for (const subPath of items) {
|
||||
const file = path.join(dir, subPath);
|
||||
const stat = fs.statSync(file);
|
||||
|
|
@ -110,67 +166,26 @@ export async function sitegen(
|
|||
if (exclude.some((e) => subPath.endsWith(e))) continue;
|
||||
const trim = ext
|
||||
? subPath
|
||||
: subPath.slice(0, -path.extname(subPath).length).replaceAll(
|
||||
".",
|
||||
"/",
|
||||
);
|
||||
: subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
|
||||
let id = prefix + trim.replaceAll("\\", "/");
|
||||
if (prefix === "/" && id.endsWith("/index")) {
|
||||
id = id.slice(0, -"/index".length) || "/";
|
||||
}
|
||||
list.push({ id, file: file });
|
||||
list.push({ id, file: path.relative(hot.projectRoot, file) });
|
||||
}
|
||||
}
|
||||
}
|
||||
const globalCssPath = join("global.css");
|
||||
|
||||
// TODO: make sure that `static` and `pages` does not overlap
|
||||
|
||||
// -- inline style sheets, used and shared by pages and views --
|
||||
status.text = "Building";
|
||||
const cssOnce = new OnceMap();
|
||||
const cssQueue = new Queue({
|
||||
name: "Bundle",
|
||||
async fn([, key, files, theme]: [string, string, string[], css.Theme]) {
|
||||
const { text, sources } = await css.bundleCssFiles(files, theme);
|
||||
incr.put({
|
||||
kind: "style",
|
||||
key,
|
||||
sources,
|
||||
value: text,
|
||||
});
|
||||
},
|
||||
passive: true,
|
||||
getItemText: ([id]) => id,
|
||||
maxJobs: 2,
|
||||
});
|
||||
function ensureCssGetsBuilt(
|
||||
cssImports: string[],
|
||||
theme: css.Theme,
|
||||
referrer: string,
|
||||
) {
|
||||
const key = css.styleKey(cssImports, theme);
|
||||
cssOnce.get(
|
||||
key,
|
||||
async () => {
|
||||
incr.getArtifact("style", key) ??
|
||||
await cssQueue.add([referrer, key, cssImports, theme]);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// -- server side render pages --
|
||||
async function loadPageModule({ file }: FileItem) {
|
||||
require(file);
|
||||
}
|
||||
async function renderPage(item: FileItem) {
|
||||
return { staticFiles, pages, views, scripts };
|
||||
}
|
||||
|
||||
export async function preparePage(io: Io, item: sg.FileItem) {
|
||||
// -- load and validate module --
|
||||
let {
|
||||
default: Page,
|
||||
meta: metadata,
|
||||
theme: pageTheme,
|
||||
layout,
|
||||
} = require(item.file);
|
||||
} = await io.import<any>(item.file);
|
||||
if (!Page) {
|
||||
throw new Error("Page is missing a 'default' export.");
|
||||
}
|
||||
|
|
@ -188,7 +203,6 @@ export async function sitegen(
|
|||
new Set([globalCssPath, ...hot.getCssImports(item.file)]),
|
||||
(file) => path.relative(hot.projectSrc, file),
|
||||
);
|
||||
ensureCssGetsBuilt(cssImports, theme, item.id);
|
||||
|
||||
// -- metadata --
|
||||
const renderedMetaPromise = Promise.resolve(
|
||||
|
|
@ -210,27 +224,23 @@ export async function sitegen(
|
|||
]);
|
||||
if (!renderedMeta.includes("<title>")) {
|
||||
throw new Error(
|
||||
"Page is missing 'meta.title'. " +
|
||||
"All pages need a title tag.",
|
||||
"Page is missing 'meta.title'. " + "All pages need a title tag.",
|
||||
);
|
||||
}
|
||||
incr.put({
|
||||
kind: "pageMetadata",
|
||||
key: item.id,
|
||||
// Incremental integrates with `hot.ts` + `require`
|
||||
// to trace all the needed source files here.
|
||||
sources: [item.file],
|
||||
value: {
|
||||
const styleKey = css.styleKey(cssImports, theme);
|
||||
return {
|
||||
id: item.id,
|
||||
html: text,
|
||||
meta: renderedMeta,
|
||||
cssImports,
|
||||
theme: theme ?? null,
|
||||
styleKey,
|
||||
clientRefs: Array.from(addon.sitegen.scripts),
|
||||
},
|
||||
});
|
||||
}
|
||||
async function prepareView(item: FileItem) {
|
||||
const module = require(item.file);
|
||||
};
|
||||
}
|
||||
|
||||
export async function prepareView(io: Io, item: sg.FileItem) {
|
||||
const module = await io.import<any>(item.file);
|
||||
if (!module.meta) {
|
||||
throw new Error(`${item.file} is missing 'export const meta'`);
|
||||
}
|
||||
|
|
@ -246,209 +256,80 @@ export async function sitegen(
|
|||
new Set([globalCssPath, ...hot.getCssImports(item.file)]),
|
||||
(file) => path.relative(hot.projectSrc, file),
|
||||
);
|
||||
ensureCssGetsBuilt(cssImports, theme, item.id);
|
||||
incr.put({
|
||||
kind: "viewMetadata",
|
||||
key: item.id,
|
||||
sources: [item.file],
|
||||
value: {
|
||||
const styleKey = css.styleKey(cssImports, theme);
|
||||
return {
|
||||
file: path.relative(hot.projectRoot, item.file),
|
||||
cssImports,
|
||||
theme,
|
||||
clientRefs: hot.getClientScriptRefs(item.file),
|
||||
hasLayout: !!module.layout?.default,
|
||||
},
|
||||
});
|
||||
}
|
||||
styleKey,
|
||||
};
|
||||
}
|
||||
export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
|
||||
|
||||
// Of the pages that are already built, a call to 'ensureCssGetsBuilt' is
|
||||
// required so that it's (1) re-built if needed, (2) not pruned from build.
|
||||
const neededPages = pages.filter((page) => {
|
||||
const existing = incr.getArtifact("pageMetadata", page.id);
|
||||
if (existing) {
|
||||
const { cssImports, theme } = existing;
|
||||
ensureCssGetsBuilt(cssImports, theme, page.id);
|
||||
}
|
||||
return !existing;
|
||||
});
|
||||
const neededViews = views.filter((view) => {
|
||||
const existing = incr.getArtifact("viewMetadata", view.id);
|
||||
if (existing) {
|
||||
const { cssImports, theme } = existing;
|
||||
ensureCssGetsBuilt(cssImports, theme, view.id);
|
||||
}
|
||||
return !existing;
|
||||
});
|
||||
|
||||
// Load the marko cache before render modules are loaded
|
||||
incr.loadMarkoCache();
|
||||
|
||||
// This is done in two passes so that a page that throws during evaluation
|
||||
// will report "Load Render Module" instead of "Render Static Page".
|
||||
const spinnerFormat = status.format;
|
||||
status.format = () => "";
|
||||
const moduleLoadQueue = new Queue({
|
||||
name: "Load Render Module",
|
||||
fn: loadPageModule,
|
||||
getItemText,
|
||||
maxJobs: 1,
|
||||
});
|
||||
moduleLoadQueue.addMany(neededPages);
|
||||
moduleLoadQueue.addMany(neededViews);
|
||||
await moduleLoadQueue.done({ method: "stop" });
|
||||
const pageQueue = new Queue({
|
||||
name: "Render Static Page",
|
||||
fn: renderPage,
|
||||
getItemText,
|
||||
maxJobs: 2,
|
||||
});
|
||||
pageQueue.addMany(neededPages);
|
||||
const viewQueue = new Queue({
|
||||
name: "Build Dynamic View",
|
||||
fn: prepareView,
|
||||
getItemText,
|
||||
maxJobs: 2,
|
||||
});
|
||||
viewQueue.addMany(neededViews);
|
||||
const pageAndViews = [
|
||||
pageQueue.done({ method: "stop" }),
|
||||
viewQueue.done({ method: "stop" }),
|
||||
];
|
||||
await Promise.allSettled(pageAndViews);
|
||||
await Promise.all(pageAndViews);
|
||||
status.format = spinnerFormat;
|
||||
|
||||
// -- bundle server javascript (backend and views) --
|
||||
status.text = "Bundle JavaScript";
|
||||
incr.snapshotMarkoCache();
|
||||
const serverJavaScriptPromise = bundle.bundleServerJavaScript(incr, "node");
|
||||
|
||||
// -- bundle client javascript --
|
||||
const referencedScripts = Array.from(
|
||||
new Set(
|
||||
[
|
||||
...pages.map((item) =>
|
||||
UNWRAP(
|
||||
incr.getArtifact("pageMetadata", item.id),
|
||||
`Missing pageMetadata ${item.id}`,
|
||||
)
|
||||
),
|
||||
...views.map((item) =>
|
||||
UNWRAP(
|
||||
incr.getArtifact("viewMetadata", item.id),
|
||||
`Missing viewMetadata ${item.id}`,
|
||||
)
|
||||
),
|
||||
].flatMap((item) => item.clientRefs),
|
||||
),
|
||||
(script) => path.resolve(hot.projectSrc, script),
|
||||
).filter((file) => !incr.hasArtifact("script", hot.getScriptId(file)));
|
||||
const extraPublicScripts = scripts.map((entry) => entry.file);
|
||||
const clientJavaScriptPromise = bundle.bundleClientJavaScript(
|
||||
referencedScripts,
|
||||
extraPublicScripts,
|
||||
incr,
|
||||
);
|
||||
await Promise.all([
|
||||
serverJavaScriptPromise,
|
||||
clientJavaScriptPromise,
|
||||
cssQueue.done({ method: "stop" }),
|
||||
]);
|
||||
await bundle.finalizeServerJavaScript(incr, "node");
|
||||
|
||||
// -- copy/compress static files --
|
||||
async function doStaticFile(item: FileItem) {
|
||||
const body = await fs.readFile(item.file);
|
||||
await incr.putAsset({
|
||||
sources: [item.file],
|
||||
key: item.id,
|
||||
body,
|
||||
});
|
||||
}
|
||||
const staticQueue = new Queue({
|
||||
name: "Load Static",
|
||||
fn: doStaticFile,
|
||||
getItemText,
|
||||
maxJobs: 16,
|
||||
});
|
||||
status.format = () => "";
|
||||
staticQueue.addMany(
|
||||
staticFiles.filter((file) => !incr.hasArtifact("asset", file.id)),
|
||||
);
|
||||
await staticQueue.done({ method: "stop" });
|
||||
status.format = spinnerFormat;
|
||||
|
||||
// -- concatenate static rendered pages --
|
||||
status.text = `Concat Pages`;
|
||||
await Promise.all(pages.map(async (page) => {
|
||||
if (incr.hasArtifact("asset", page.id)) return;
|
||||
const {
|
||||
html,
|
||||
meta,
|
||||
export function prepareInlineCss(
|
||||
items: Array<{
|
||||
styleKey: string;
|
||||
cssImports: string[];
|
||||
theme: css.Theme;
|
||||
}>,
|
||||
) {
|
||||
const map = new Map<string, incr.Ref<string>>();
|
||||
for (const { styleKey, cssImports, theme } of items) {
|
||||
if (map.has(styleKey)) continue;
|
||||
map.set(
|
||||
styleKey,
|
||||
incr.work(css.bundleCssFiles, {
|
||||
cssImports,
|
||||
theme,
|
||||
clientRefs,
|
||||
} = UNWRAP(incr.out.pageMetadata.get(page.id));
|
||||
const scriptIds = clientRefs.map(hot.getScriptId);
|
||||
const styleKey = css.styleKey(cssImports, theme);
|
||||
const style = UNWRAP(
|
||||
incr.out.style.get(styleKey),
|
||||
`Missing style ${styleKey}`,
|
||||
dev: false,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
export type PreparedPage = Awaited<ReturnType<typeof preparePage>>;
|
||||
export async function assembleAndWritePage(
|
||||
pageWork: incr.Ref<PreparedPage>,
|
||||
styleMap: Map<string, incr.Ref<string>>,
|
||||
scriptWork: incr.Ref<Record<string, string>>,
|
||||
) {
|
||||
const page = await pageWork;
|
||||
return incr.work(
|
||||
async (io, { id, html, meta, styleKey, clientRefs }) => {
|
||||
const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
|
||||
|
||||
const scriptIds = clientRefs.map(hot.getScriptId);
|
||||
const scriptMap = await io.readWork(scriptWork);
|
||||
const scripts = scriptIds.map((ref) =>
|
||||
UNWRAP(scriptMap[ref], `Missing script ${ref}`)
|
||||
)
|
||||
.map((x) => `{${x}}`).join("\n");
|
||||
|
||||
const doc = wrapDocument({
|
||||
body: html,
|
||||
head: meta,
|
||||
inlineCss: style,
|
||||
scripts: scriptIds.map(
|
||||
(ref) => UNWRAP(incr.out.script.get(ref), `Missing script ${ref}`),
|
||||
).map((x) => `{${x}}`).join("\n"),
|
||||
inlineCss,
|
||||
scripts,
|
||||
});
|
||||
await incr.putAsset({
|
||||
sources: [
|
||||
page.file,
|
||||
...incr.sourcesFor("style", styleKey),
|
||||
...scriptIds.flatMap((ref) => incr.sourcesFor("script", ref)),
|
||||
],
|
||||
key: page.id,
|
||||
body: doc,
|
||||
headers: {
|
||||
await io.writeAsset(id, doc, {
|
||||
"Content-Type": "text/html",
|
||||
},
|
||||
});
|
||||
}));
|
||||
status.format = () => "";
|
||||
status.text = ``;
|
||||
// This will wait for all compression jobs to finish, which up
|
||||
// to this point have been left as dangling promises.
|
||||
await incr.wait();
|
||||
|
||||
const { inserted, referenced, unreferenced } = incr.shake();
|
||||
|
||||
// Flush the site to disk.
|
||||
status.format = spinnerFormat;
|
||||
status.text = `Incremental Flush`;
|
||||
incr.flush("node"); // Write outputs
|
||||
return {
|
||||
incr,
|
||||
inserted,
|
||||
referenced,
|
||||
unreferenced,
|
||||
elapsed: (performance.now() - startTime) / 1000,
|
||||
};
|
||||
},
|
||||
page,
|
||||
);
|
||||
}
|
||||
|
||||
function getItemText({ file }: FileItem) {
|
||||
return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
|
||||
}
|
||||
|
||||
import { OnceMap, Queue } from "#sitegen/async";
|
||||
import { Incremental } from "./incremental.ts";
|
||||
import * as sg from "#sitegen";
|
||||
import * as incr from "./incremental.ts";
|
||||
import { Io } from "./incremental.ts";
|
||||
import * as bundle from "./bundle.ts";
|
||||
import * as css from "./css.ts";
|
||||
import * as engine from "./engine/ssr.ts";
|
||||
import * as hot from "./hot.ts";
|
||||
import * as fs from "#sitegen/fs";
|
||||
import * as sg from "#sitegen";
|
||||
import type { FileItem } from "#sitegen";
|
||||
import * as path from "node:path";
|
||||
import * as meta from "#sitegen/meta";
|
||||
|
|
|
|||
|
|
@ -19,6 +19,8 @@ export const load = createRequire(
|
|||
};
|
||||
export const { cache } = load;
|
||||
|
||||
load<any>("source-map-support").install({hookRequire: true});
|
||||
|
||||
// Register extensions by overwriting `require.extensions`
|
||||
const require = load;
|
||||
const exts = require.extensions;
|
||||
|
|
@ -42,8 +44,7 @@ export function getFileStat(filepath: string) {
|
|||
}
|
||||
|
||||
function shouldTrackPath(filename: string) {
|
||||
return !filename.includes("node_modules") &&
|
||||
!filename.includes(import.meta.dirname);
|
||||
return !filename.includes("node_modules");
|
||||
}
|
||||
|
||||
const Module = load<typeof import("node:module")>("node:module");
|
||||
|
|
@ -59,11 +60,12 @@ Module.prototype._compile = function (
|
|||
filename,
|
||||
format,
|
||||
);
|
||||
const stat = fs.statSync(filename);
|
||||
if (shouldTrackPath(filename)) {
|
||||
const stat = fs.statSync(filename);
|
||||
const cssImportsMaybe: string[] = [];
|
||||
const imports: string[] = [];
|
||||
for (const { filename: file, cloverClientRefs } of this.children) {
|
||||
for (const childModule of this.children) {
|
||||
const { filename: file, cloverClientRefs } = childModule;
|
||||
if (file.endsWith(".css")) cssImportsMaybe.push(file);
|
||||
else {
|
||||
const child = fileStats.get(file);
|
||||
|
|
@ -71,6 +73,7 @@ Module.prototype._compile = function (
|
|||
const { cssImportsRecursive } = child;
|
||||
if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive);
|
||||
imports.push(file);
|
||||
(childModule.cloverImporters ??= []).push(this);
|
||||
if (cloverClientRefs && cloverClientRefs.length > 0) {
|
||||
(this.cloverClientRefs ??= [])
|
||||
.push(...cloverClientRefs);
|
||||
|
|
@ -82,7 +85,7 @@ Module.prototype._compile = function (
|
|||
? Array.from(new Set(cssImportsMaybe))
|
||||
: null,
|
||||
imports,
|
||||
lastModified: stat.mtimeMs,
|
||||
lastModified: Math.floor(stat.mtimeMs),
|
||||
});
|
||||
}
|
||||
return result;
|
||||
|
|
@ -113,7 +116,7 @@ function loadEsbuild(module: NodeJS.Module, filepath: string) {
|
|||
interface LoadOptions {
|
||||
scannedClientRefs?: string[];
|
||||
}
|
||||
function loadEsbuildCode(
|
||||
export function loadEsbuildCode(
|
||||
module: NodeJS.Module,
|
||||
filepath: string,
|
||||
src: string,
|
||||
|
|
@ -139,7 +142,7 @@ function loadEsbuildCode(
|
|||
import.meta.url = ${JSON.stringify(pathToFileURL(filepath).toString())};
|
||||
import.meta.dirname = ${JSON.stringify(path.dirname(filepath))};
|
||||
import.meta.filename = ${JSON.stringify(filepath)};
|
||||
`.trim().replace(/\n/g, "") + src;
|
||||
`.trim().replace(/[\n\s]/g, "") + src;
|
||||
}
|
||||
src = esbuild.transformSync(src, {
|
||||
loader,
|
||||
|
|
@ -149,11 +152,12 @@ function loadEsbuildCode(
|
|||
jsxImportSource: "#ssr",
|
||||
jsxDev: true,
|
||||
sourcefile: filepath,
|
||||
sourcemap: 'inline',
|
||||
}).code;
|
||||
return module._compile(src, filepath, "commonjs");
|
||||
}
|
||||
|
||||
function resolveClientRef(sourcePath: string, ref: string) {
|
||||
export function resolveClientRef(sourcePath: string, ref: string) {
|
||||
const filePath = resolveFrom(sourcePath, ref);
|
||||
if (
|
||||
!filePath.endsWith(".client.ts") &&
|
||||
|
|
@ -164,44 +168,10 @@ function resolveClientRef(sourcePath: string, ref: string) {
|
|||
return path.relative(projectSrc, filePath);
|
||||
}
|
||||
|
||||
// TODO: extract the marko compilation tools out, lazy load them
|
||||
export interface MarkoCacheEntry {
|
||||
src: string;
|
||||
scannedClientRefs: string[];
|
||||
}
|
||||
export const markoCache = new Map<string, MarkoCacheEntry>();
|
||||
let lazyMarko: typeof import('./marko.ts') | null = null;
|
||||
function loadMarko(module: NodeJS.Module, filepath: string) {
|
||||
let cache = markoCache.get(filepath);
|
||||
if (!cache) {
|
||||
let src = fs.readFileSync(filepath, "utf8");
|
||||
// A non-standard thing here is Clover Sitegen implements
|
||||
// its own client side scripting stuff, so it overrides
|
||||
// bare client import statements to it's own usage.
|
||||
const scannedClientRefs = new Set<string>();
|
||||
if (src.match(/^\s*client\s+import\s+["']/m)) {
|
||||
src = src.replace(
|
||||
/^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
|
||||
(_, src) => {
|
||||
const ref = JSON.parse(`"${src.slice(1, -1)}"`);
|
||||
const resolved = resolveClientRef(filepath, ref);
|
||||
scannedClientRefs.add(resolved);
|
||||
return `<CloverScriptInclude=${
|
||||
JSON.stringify(getScriptId(resolved))
|
||||
} />`;
|
||||
},
|
||||
) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
|
||||
}
|
||||
|
||||
src = marko.compileSync(src, filepath).code;
|
||||
src = src.replace("marko/debug/html", "#ssr/marko");
|
||||
cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
|
||||
markoCache.set(filepath, cache);
|
||||
}
|
||||
|
||||
const { src, scannedClientRefs } = cache;
|
||||
return loadEsbuildCode(module, filepath, src, {
|
||||
scannedClientRefs,
|
||||
});
|
||||
lazyMarko ??= require<typeof import('./marko.ts')>("./framework/marko.ts");
|
||||
lazyMarko.loadMarko(module, filepath);
|
||||
}
|
||||
|
||||
function loadMdx(module: NodeJS.Module, filepath: string) {
|
||||
|
|
@ -224,10 +194,14 @@ export function reloadRecursive(filepath: string) {
|
|||
}
|
||||
|
||||
export function unload(filepath: string) {
|
||||
lazyMarko?.markoCache.delete(filepath)
|
||||
filepath = path.resolve(filepath);
|
||||
const existing = cache[filepath];
|
||||
if (existing) delete cache[filepath];
|
||||
fileStats.delete(filepath);
|
||||
const module = cache[filepath];
|
||||
if (!module) return;
|
||||
delete cache[filepath];
|
||||
for (const importer of module.cloverImporters ?? []) {
|
||||
unload(importer.filename);
|
||||
}
|
||||
}
|
||||
|
||||
function deleteRecursiveInner(id: string, module: any) {
|
||||
|
|
@ -326,6 +300,7 @@ declare global {
|
|||
interface Module {
|
||||
cloverClientRefs?: string[];
|
||||
cloverSourceCode?: string;
|
||||
cloverImporters?: Module[],
|
||||
|
||||
_compile(
|
||||
this: NodeJS.Module,
|
||||
|
|
@ -343,11 +318,10 @@ declare module "node:module" {
|
|||
): unknown;
|
||||
}
|
||||
|
||||
import * as fs from "./lib/fs.ts";
|
||||
import * as fs from "#sitegen/fs";
|
||||
import * as path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import * as esbuild from "esbuild";
|
||||
import * as marko from "@marko/compiler";
|
||||
import { createRequire } from "node:module";
|
||||
import * as mdx from "@mdx-js/mdx";
|
||||
import * as self from "./hot.ts";
|
||||
|
|
|
|||
56
framework/incremental.test.ts
Normal file
56
framework/incremental.test.ts
Normal file
|
|
@ -0,0 +1,56 @@
|
|||
test("trivial case", async () => {
|
||||
incr.reset();
|
||||
|
||||
const file1 = tmpFile("example.txt");
|
||||
file1.write("one");
|
||||
|
||||
async function compilation() {
|
||||
const first = incr.work({
|
||||
label: "first compute",
|
||||
async run (io) {
|
||||
await setTimeout(1000);
|
||||
const contents = await io.readFile(file1.path);
|
||||
return [contents, Math.random()] as const;
|
||||
}
|
||||
});
|
||||
const second = incr.work({
|
||||
label: "second compute",
|
||||
wait: first,
|
||||
async run (io) {
|
||||
await setTimeout(1000);
|
||||
return io.readWork(first)[0].toUpperCase();
|
||||
}
|
||||
});
|
||||
const third = incr.work({
|
||||
label: "third compute",
|
||||
wait: first,
|
||||
async run (io) {
|
||||
await setTimeout(1000);
|
||||
return io.readWork(first)[1] * 1000;
|
||||
}
|
||||
});
|
||||
return incr.work({
|
||||
label: "last compute",
|
||||
wait: [second, third],
|
||||
async run (io) {
|
||||
await setTimeout(1000);
|
||||
return {
|
||||
second: io.readWork(second),
|
||||
third: io.readWork(third),
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
const { value: first } = await incr.compile(compilation);
|
||||
const { value: second } = await incr.compile(compilation);
|
||||
ASSERT(first === second);
|
||||
incr.forceInvalidate(file1.path);
|
||||
const { value: third } = await incr.compile(compilation);
|
||||
ASSERT(first !== third);
|
||||
ASSERT(first[0] === third[0]);
|
||||
});
|
||||
|
||||
import * as incr from "./incremental2.ts";
|
||||
import { beforeEach, test } from "node:test";
|
||||
import { tmpFile } from "#sitegen/testing";import { setTimeout } from "node:timers/promises";
|
||||
|
||||
File diff suppressed because it is too large
Load diff
|
|
@ -69,7 +69,7 @@ export function etagMatches(etag: string, ifNoneMatch: string) {
|
|||
return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
|
||||
}
|
||||
|
||||
function subarrayAsset([start, end]: View) {
|
||||
function subarrayAsset([start, end]: BufferView) {
|
||||
return assets!.buf.subarray(start, end);
|
||||
}
|
||||
|
||||
|
|
@ -115,6 +115,6 @@ process.on("message", (msg: any) => {
|
|||
import * as fs from "#sitegen/fs";
|
||||
import type { Context, Next } from "hono";
|
||||
import type { StatusCode } from "hono/utils/http-status";
|
||||
import type { BuiltAsset, BuiltAssetMap, View } from "../incremental.ts";
|
||||
import type { BuiltAsset, BuiltAssetMap, BufferView } from "../incremental.ts";
|
||||
import { Buffer } from "node:buffer";
|
||||
import * as path from "node:path";
|
||||
|
|
|
|||
11
framework/lib/testing.ts
Normal file
11
framework/lib/testing.ts
Normal file
|
|
@ -0,0 +1,11 @@
|
|||
export function tmpFile(basename: string) {
|
||||
const file = path.join(import.meta.dirname, '../../.clover/testing', basename);
|
||||
return {
|
||||
path: file,
|
||||
read: fs.readFile.bind(fs, file),
|
||||
write: fs.writeMkdir.bind(fs, file),
|
||||
};
|
||||
}
|
||||
|
||||
import * as path from 'node:path';
|
||||
import * as fs from './fs.ts';
|
||||
46
framework/marko.ts
Normal file
46
framework/marko.ts
Normal file
|
|
@ -0,0 +1,46 @@
|
|||
console.log("MARKO");
|
||||
export interface MarkoCacheEntry {
|
||||
src: string;
|
||||
scannedClientRefs: string[];
|
||||
}
|
||||
|
||||
export const markoCache = new Map<string, MarkoCacheEntry>();
|
||||
|
||||
export function loadMarko(module: NodeJS.Module, filepath: string) {
|
||||
let cache = markoCache.get(filepath);
|
||||
console.log({ filepath, has: !!cache })
|
||||
if (!cache) {
|
||||
let src = fs.readFileSync(filepath, "utf8");
|
||||
// A non-standard thing here is Clover Sitegen implements
|
||||
// its own client side scripting stuff, so it overrides
|
||||
// bare client import statements to it's own usage.
|
||||
const scannedClientRefs = new Set<string>();
|
||||
if (src.match(/^\s*client\s+import\s+["']/m)) {
|
||||
src = src.replace(
|
||||
/^\s*client\s+import\s+("[^"]+"|'[^']+')[^\n]+/m,
|
||||
(_, src) => {
|
||||
const ref = JSON.parse(`"${src.slice(1, -1)}"`);
|
||||
const resolved = hot.resolveClientRef(filepath, ref);
|
||||
scannedClientRefs.add(resolved);
|
||||
return `<CloverScriptInclude=${
|
||||
JSON.stringify(hot.getScriptId(resolved))
|
||||
} />`;
|
||||
},
|
||||
) + '\nimport { addScript as CloverScriptInclude } from "#sitegen";\n';
|
||||
}
|
||||
|
||||
src = marko.compileSync(src, filepath).code;
|
||||
src = src.replace("marko/debug/html", "#ssr/marko");
|
||||
cache = { src, scannedClientRefs: Array.from(scannedClientRefs) };
|
||||
markoCache.set(filepath, cache);
|
||||
}
|
||||
|
||||
const { src, scannedClientRefs } = cache;
|
||||
return hot.loadEsbuildCode(module, filepath, src, {
|
||||
scannedClientRefs,
|
||||
});
|
||||
}
|
||||
|
||||
import * as marko from "@marko/compiler";
|
||||
import * as hot from "./hot.ts";
|
||||
import * as fs from "#sitegen/fs";
|
||||
|
|
@ -2,102 +2,67 @@
|
|||
|
||||
const debounceMilliseconds = 25;
|
||||
|
||||
let subprocess: child_process.ChildProcess | null = null;
|
||||
process.on("beforeExit", () => {
|
||||
subprocess?.removeListener("close", onSubprocessClose);
|
||||
});
|
||||
|
||||
let watch: Watch;
|
||||
|
||||
export async function main() {
|
||||
let subprocess: child_process.ChildProcess | null = null;
|
||||
|
||||
// Catch up state by running a main build.
|
||||
const { incr } = await generate.main();
|
||||
// ...and watch the files that cause invals.
|
||||
const watch = new Watch(rebuild);
|
||||
watch.add(...incr.invals.keys());
|
||||
statusLine();
|
||||
// ... and then serve it!
|
||||
serve();
|
||||
await incr.restore();
|
||||
watch = new Watch(rebuild);
|
||||
rebuild([]);
|
||||
}
|
||||
|
||||
function serve() {
|
||||
function serve() {
|
||||
if (subprocess) {
|
||||
subprocess.removeListener("close", onSubprocessClose);
|
||||
subprocess.kill();
|
||||
}
|
||||
subprocess = child_process.fork(".clover/out/server.js", [
|
||||
subprocess = child_process.fork(".clover/o/backend.js", [
|
||||
"--development",
|
||||
], {
|
||||
stdio: "inherit",
|
||||
});
|
||||
subprocess.on("close", onSubprocessClose);
|
||||
}
|
||||
}
|
||||
|
||||
function onSubprocessClose(code: number | null, signal: string | null) {
|
||||
function onSubprocessClose(code: number | null, signal: string | null) {
|
||||
subprocess = null;
|
||||
const status = code != null ? `code ${code}` : `signal ${signal}`;
|
||||
console.error(`Backend process exited with ${status}`);
|
||||
}
|
||||
}
|
||||
|
||||
process.on("beforeExit", () => {
|
||||
subprocess?.removeListener("close", onSubprocessClose);
|
||||
});
|
||||
|
||||
function rebuild(files: string[]) {
|
||||
files = files.map((file) => path.relative(hot.projectRoot, file));
|
||||
const changed: string[] = [];
|
||||
for (const file of files) {
|
||||
let mtimeMs: number | null = null;
|
||||
try {
|
||||
mtimeMs = fs.statSync(file).mtimeMs;
|
||||
} catch (err: any) {
|
||||
if (err?.code !== "ENOENT") throw err;
|
||||
}
|
||||
if (incr.updateStat(file, mtimeMs)) changed.push(file);
|
||||
}
|
||||
if (changed.length === 0) {
|
||||
console.warn("Files were modified but the 'modify' time did not change.");
|
||||
return;
|
||||
}
|
||||
withSpinner<any, Awaited<ReturnType<typeof generate.sitegen>>>({
|
||||
text: "Rebuilding",
|
||||
successText: generate.successText,
|
||||
failureText: () => "sitegen FAIL",
|
||||
}, async (spinner) => {
|
||||
console.info("---");
|
||||
console.info(
|
||||
"Updated" +
|
||||
(changed.length === 1
|
||||
? " " + changed[0]
|
||||
: changed.map((file) => "\n- " + file)),
|
||||
);
|
||||
const result = await generate.sitegen(spinner, incr);
|
||||
incr.toDisk(); // Allows picking up this state again
|
||||
for (const file of watch.files) {
|
||||
const relative = path.relative(hot.projectRoot, file);
|
||||
if (!incr.invals.has(relative)) watch.remove(file);
|
||||
}
|
||||
return result;
|
||||
}).then((result) => {
|
||||
async function rebuild(files: string[]) {
|
||||
const hasInvalidated = files.length === 0
|
||||
|| (await Promise.all(files.map(incr.invalidate))).some(Boolean);
|
||||
if (!hasInvalidated) return;
|
||||
incr.compile(generate.generate).then(({
|
||||
watchFiles,
|
||||
newOutputs,
|
||||
newAssets
|
||||
}) => {
|
||||
const removeWatch = [...watch.files].filter(x => !watchFiles.has(x))
|
||||
for (const file of removeWatch) watch.remove(file);
|
||||
watch.add(...watchFiles);
|
||||
// Restart the server if it was changed or not running.
|
||||
if (
|
||||
!subprocess ||
|
||||
result.inserted.some(({ kind }) => kind === "backendReplace")
|
||||
) {
|
||||
if (!subprocess || newOutputs.includes("backend.js")) {
|
||||
serve();
|
||||
} else if (
|
||||
subprocess &&
|
||||
result.inserted.some(({ kind }) => kind === "asset")
|
||||
) {
|
||||
} else if (subprocess && newAssets) {
|
||||
subprocess.send({ type: "clover.assets.reload" });
|
||||
}
|
||||
return result;
|
||||
}).catch((err) => {
|
||||
console.error(util.inspect(err));
|
||||
}).finally(statusLine);
|
||||
}
|
||||
}
|
||||
|
||||
function statusLine() {
|
||||
function statusLine() {
|
||||
console.info(
|
||||
`Watching ${incr.invals.size} files \x1b[36m[last change: ${
|
||||
new Date().toLocaleTimeString()
|
||||
}]\x1b[39m`,
|
||||
`Watching ${watch.files.size} files `
|
||||
+ `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
class Watch {
|
||||
|
|
@ -174,11 +139,21 @@ class Watch {
|
|||
for (const w of this.watchers) w.close();
|
||||
}
|
||||
|
||||
#getFiles(absPath: string, event: fs.WatchEventType) {
|
||||
const files = [];
|
||||
if (this.files.has(absPath)) files.push(absPath);
|
||||
if (event === 'rename') {
|
||||
const dir = path.dirname(absPath);
|
||||
if (this.files.has(dir)) files.push(dir);
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
#handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) {
|
||||
if (!subPath) return;
|
||||
const file = path.join(root, subPath);
|
||||
if (!this.files.has(file)) return;
|
||||
this.stale.add(file);
|
||||
const files = this.#getFiles(path.join(root, subPath), event);
|
||||
if (files.length === 0) return;
|
||||
for(const file of files) this.stale.add(file);
|
||||
const { debounce } = this;
|
||||
if (debounce !== null) clearTimeout(debounce);
|
||||
this.debounce = setTimeout(() => {
|
||||
|
|
@ -192,6 +167,7 @@ class Watch {
|
|||
import * as fs from "node:fs";
|
||||
import { withSpinner } from "@paperclover/console/Spinner";
|
||||
import * as generate from "./generate.ts";
|
||||
import * as incr from "./incremental.ts";
|
||||
import * as path from "node:path";
|
||||
import * as util from "node:util";
|
||||
import * as hot from "./hot.ts";
|
||||
|
|
|
|||
126
package-lock.json
generated
126
package-lock.json
generated
|
|
@ -15,8 +15,10 @@
|
|||
"hls.js": "^1.6.5",
|
||||
"hono": "^4.7.11",
|
||||
"marko": "^6.0.20",
|
||||
"msgpackr": "^1.11.5",
|
||||
"puppeteer": "^24.10.1",
|
||||
"sharp": "^0.34.2",
|
||||
"source-map-support": "^0.5.21",
|
||||
"unique-names-generator": "^4.7.1",
|
||||
"vscode-oniguruma": "^2.0.1",
|
||||
"vscode-textmate": "^9.2.0"
|
||||
|
|
@ -1478,6 +1480,84 @@
|
|||
"url": "https://opencollective.com/unified"
|
||||
}
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-arm64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-arm64/-/msgpackr-extract-darwin-arm64-3.0.3.tgz",
|
||||
"integrity": "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-darwin-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-darwin-x64/-/msgpackr-extract-darwin-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"darwin"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm/-/msgpackr-extract-linux-arm-3.0.3.tgz",
|
||||
"integrity": "sha512-fg0uy/dG/nZEXfYilKoRe7yALaNmHoYeIoJuJ7KJ+YyU2bvY8vPv27f7UKhGRpY6euFYqEVhxCFZgAUNQBM3nw==",
|
||||
"cpu": [
|
||||
"arm"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-arm64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-arm64/-/msgpackr-extract-linux-arm64-3.0.3.tgz",
|
||||
"integrity": "sha512-YxQL+ax0XqBJDZiKimS2XQaf+2wDGVa1enVRGzEvLLVFeqa5kx2bWbtcSXgsxjQB7nRqqIGFIcLteF/sHeVtQg==",
|
||||
"cpu": [
|
||||
"arm64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-linux-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-linux-x64/-/msgpackr-extract-linux-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-cvwNfbP07pKUfq1uH+S6KJ7dT9K8WOE4ZiAcsrSes+UY55E/0jLYc+vq+DO7jlmqRb5zAggExKm0H7O/CBaesg==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"linux"
|
||||
]
|
||||
},
|
||||
"node_modules/@msgpackr-extract/msgpackr-extract-win32-x64": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/@msgpackr-extract/msgpackr-extract-win32-x64/-/msgpackr-extract-win32-x64-3.0.3.tgz",
|
||||
"integrity": "sha512-x0fWaQtYp4E6sktbsdAqnehxDgEc/VwM7uLsRCYWaiGu0ykYdZPiS8zCWdnjHwyiumousxfBm4SO31eXqwEZhQ==",
|
||||
"cpu": [
|
||||
"x64"
|
||||
],
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"os": [
|
||||
"win32"
|
||||
]
|
||||
},
|
||||
"node_modules/@paperclover/console": {
|
||||
"resolved": "git+https://git.paperclover.net/clo/console.git#1a6ac2b79fdd8a21a1c57d25723975872bc07e3e",
|
||||
"dependencies": {
|
||||
|
|
@ -3765,6 +3845,37 @@
|
|||
"integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==",
|
||||
"license": "MIT"
|
||||
},
|
||||
"node_modules/msgpackr": {
|
||||
"version": "1.11.5",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.11.5.tgz",
|
||||
"integrity": "sha512-UjkUHN0yqp9RWKy0Lplhh+wlpdt9oQBYgULZOiFhV3VclSF1JnSQWZ5r9gORQlNYaUKQoR8itv7g7z1xDDuACA==",
|
||||
"license": "MIT",
|
||||
"optionalDependencies": {
|
||||
"msgpackr-extract": "^3.0.2"
|
||||
}
|
||||
},
|
||||
"node_modules/msgpackr-extract": {
|
||||
"version": "3.0.3",
|
||||
"resolved": "https://registry.npmjs.org/msgpackr-extract/-/msgpackr-extract-3.0.3.tgz",
|
||||
"integrity": "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"node-gyp-build-optional-packages": "5.2.2"
|
||||
},
|
||||
"bin": {
|
||||
"download-msgpackr-prebuilds": "bin/download-prebuilds.js"
|
||||
},
|
||||
"optionalDependencies": {
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3",
|
||||
"@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3"
|
||||
}
|
||||
},
|
||||
"node_modules/netmask": {
|
||||
"version": "2.0.2",
|
||||
"resolved": "https://registry.npmjs.org/netmask/-/netmask-2.0.2.tgz",
|
||||
|
|
@ -3774,6 +3885,21 @@
|
|||
"node": ">= 0.4.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp-build-optional-packages": {
|
||||
"version": "5.2.2",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build-optional-packages/-/node-gyp-build-optional-packages-5.2.2.tgz",
|
||||
"integrity": "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw==",
|
||||
"license": "MIT",
|
||||
"optional": true,
|
||||
"dependencies": {
|
||||
"detect-libc": "^2.0.1"
|
||||
},
|
||||
"bin": {
|
||||
"node-gyp-build-optional-packages": "bin.js",
|
||||
"node-gyp-build-optional-packages-optional": "optional.js",
|
||||
"node-gyp-build-optional-packages-test": "build-test.js"
|
||||
}
|
||||
},
|
||||
"node_modules/node-releases": {
|
||||
"version": "2.0.19",
|
||||
"resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
|
||||
|
|
|
|||
|
|
@ -11,8 +11,10 @@
|
|||
"hls.js": "^1.6.5",
|
||||
"hono": "^4.7.11",
|
||||
"marko": "^6.0.20",
|
||||
"msgpackr": "^1.11.5",
|
||||
"puppeteer": "^24.10.1",
|
||||
"sharp": "^0.34.2",
|
||||
"source-map-support": "^0.5.21",
|
||||
"unique-names-generator": "^4.7.1",
|
||||
"vscode-oniguruma": "^2.0.1",
|
||||
"vscode-textmate": "^9.2.0"
|
||||
|
|
|
|||
|
|
@ -17,9 +17,9 @@ export async function main() {
|
|||
const start = performance.now();
|
||||
const timerSpinner = new Spinner({
|
||||
text: () =>
|
||||
`paper clover's scan3 [${
|
||||
((performance.now() - start) / 1000).toFixed(1)
|
||||
}s]`,
|
||||
`paper clover's scan3 [${((performance.now() - start) / 1000).toFixed(
|
||||
1,
|
||||
)}s]`,
|
||||
fps: 10,
|
||||
});
|
||||
using _endTimerSpinner = { [Symbol.dispose]: () => timerSpinner.stop() };
|
||||
|
|
@ -38,20 +38,23 @@ export async function main() {
|
|||
qList.addMany(items.map((subPath) => path.join(absPath, subPath)));
|
||||
|
||||
if (mediaFile) {
|
||||
const deleted = mediaFile.getChildren()
|
||||
const deleted = mediaFile
|
||||
.getChildren()
|
||||
.filter((child) => !items.includes(child.basename))
|
||||
.flatMap((child) =>
|
||||
child.kind === MediaFileKind.directory
|
||||
? child.getRecursiveFileChildren()
|
||||
: child
|
||||
: child,
|
||||
);
|
||||
|
||||
qMeta.addMany(deleted.map((mediaFile) => ({
|
||||
qMeta.addMany(
|
||||
deleted.map((mediaFile) => ({
|
||||
absPath: path.join(root, mediaFile.path),
|
||||
publicPath: mediaFile.path,
|
||||
stat: null,
|
||||
mediaFile,
|
||||
})));
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
return;
|
||||
|
|
@ -96,13 +99,13 @@ export async function main() {
|
|||
if (
|
||||
mediaFile &&
|
||||
mediaFile.date.getTime() < stat.mtime.getTime() &&
|
||||
(Date.now() - stat.mtime.getTime()) < monthMilliseconds
|
||||
Date.now() - stat.mtime.getTime() < monthMilliseconds
|
||||
) {
|
||||
date = mediaFile.date;
|
||||
console.warn(
|
||||
`M-time on ${publicPath} was likely corrupted. ${
|
||||
formatDate(mediaFile.date)
|
||||
} -> ${formatDate(stat.mtime)}`,
|
||||
`M-time on ${publicPath} was likely corrupted. ${formatDate(
|
||||
mediaFile.date,
|
||||
)} -> ${formatDate(stat.mtime)}`,
|
||||
);
|
||||
}
|
||||
mediaFile = MediaFile.createFile({
|
||||
|
|
@ -129,7 +132,10 @@ export async function main() {
|
|||
await processor.run({ absPath, stat, mediaFile, spin });
|
||||
mediaFile.setProcessed(mediaFile.processed | (1 << (16 + index)));
|
||||
for (const dependantJob of after) {
|
||||
ASSERT(dependantJob.needs > 0, `dependantJob.needs > 0, ${dependantJob.needs}`);
|
||||
ASSERT(
|
||||
dependantJob.needs > 0,
|
||||
`dependantJob.needs > 0, ${dependantJob.needs}`,
|
||||
);
|
||||
dependantJob.needs -= 1;
|
||||
if (dependantJob.needs == 0) qProcess.add(dependantJob);
|
||||
}
|
||||
|
|
@ -149,25 +155,27 @@ export async function main() {
|
|||
}));
|
||||
}
|
||||
|
||||
async function queueProcessors(
|
||||
{ absPath, stat, mediaFile }: Omit<ProcessFileArgs, "spin">,
|
||||
) {
|
||||
async function queueProcessors({
|
||||
absPath,
|
||||
stat,
|
||||
mediaFile,
|
||||
}: Omit<ProcessFileArgs, "spin">) {
|
||||
const ext = mediaFile.extensionNonEmpty.toLowerCase();
|
||||
let possible = processors.filter((p) =>
|
||||
p.include ? p.include.has(ext) : !p.exclude?.has(ext)
|
||||
p.include ? p.include.has(ext) : !p.exclude?.has(ext),
|
||||
);
|
||||
if (possible.length === 0) return;
|
||||
|
||||
const hash = possible.reduce((a, b) => a ^ b.hash, 0) | 1;
|
||||
ASSERT(hash <= 0xFFFF, `${hash.toString(16)} has no bits above 16 set`);
|
||||
ASSERT(hash <= 0xffff, `${hash.toString(16)} has no bits above 16 set`);
|
||||
let processed = mediaFile.processed;
|
||||
|
||||
// If the hash has changed, migrate the bitfield over.
|
||||
// This also runs when the processor hash is in it's initial 0 state.
|
||||
const order = decodeProcessors(mediaFile.processors);
|
||||
if ((processed & 0xFFFF) !== hash) {
|
||||
const previous = order.filter((_, i) =>
|
||||
(processed & (1 << (16 + i))) !== 0
|
||||
if ((processed & 0xffff) !== hash) {
|
||||
const previous = order.filter(
|
||||
(_, i) => (processed & (1 << (16 + i))) !== 0,
|
||||
);
|
||||
processed = hash;
|
||||
for (const { id, hash } of previous) {
|
||||
|
|
@ -182,13 +190,13 @@ export async function main() {
|
|||
}
|
||||
mediaFile.setProcessors(
|
||||
processed,
|
||||
possible.map((p) =>
|
||||
p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xFF)
|
||||
).join(";"),
|
||||
possible
|
||||
.map((p) => p.id + String.fromCharCode(p.hash >> 8, p.hash & 0xff))
|
||||
.join(";"),
|
||||
);
|
||||
} else {
|
||||
possible = order.map(({ id }) =>
|
||||
UNWRAP(possible.find((p) => p.id === id))
|
||||
UNWRAP(possible.find((p) => p.id === id)),
|
||||
);
|
||||
}
|
||||
|
||||
|
|
@ -225,8 +233,9 @@ export async function main() {
|
|||
|
||||
async function runUndoProcessors(mediaFile: MediaFile) {
|
||||
const { processed } = mediaFile;
|
||||
const previous = decodeProcessors(mediaFile.processors)
|
||||
.filter((_, i) => (processed & (1 << (16 + i))) !== 0);
|
||||
const previous = decodeProcessors(mediaFile.processors).filter(
|
||||
(_, i) => (processed & (1 << (16 + i))) !== 0,
|
||||
);
|
||||
for (const { id } of previous) {
|
||||
const p = processors.find((p) => p.id === id);
|
||||
if (!p) continue;
|
||||
|
|
@ -244,22 +253,23 @@ export async function main() {
|
|||
await qProcess.done();
|
||||
|
||||
// Update directory metadata
|
||||
const dirs = MediaFile.getDirectoriesToReindex()
|
||||
.sort((a, b) => b.path.length - a.path.length);
|
||||
const dirs = MediaFile.getDirectoriesToReindex().sort(
|
||||
(a, b) => b.path.length - a.path.length,
|
||||
);
|
||||
for (const dir of dirs) {
|
||||
const children = dir.getChildren();
|
||||
|
||||
// readme.txt
|
||||
const readmeContent = children.find((x) =>
|
||||
x.basename === "readme.txt"
|
||||
)?.contents ?? "";
|
||||
const readmeContent =
|
||||
children.find((x) => x.basename === "readme.txt")?.contents ?? "";
|
||||
|
||||
// dirsort
|
||||
let dirsort: string[] | null = null;
|
||||
const dirSortRaw =
|
||||
children.find((x) => x.basename === ".dirsort")?.contents ?? "";
|
||||
if (dirSortRaw) {
|
||||
dirsort = dirSortRaw.split("\n")
|
||||
dirsort = dirSortRaw
|
||||
.split("\n")
|
||||
.map((x) => x.trim())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
|
@ -284,7 +294,8 @@ export async function main() {
|
|||
}
|
||||
}
|
||||
|
||||
const dirHash = crypto.createHash("sha1")
|
||||
const dirHash = crypto
|
||||
.createHash("sha1")
|
||||
.update(dir.path + allHashes)
|
||||
.digest("hex");
|
||||
|
||||
|
|
@ -323,19 +334,21 @@ export async function main() {
|
|||
|
||||
console.info(
|
||||
"Updated file viewer index in \x1b[1m" +
|
||||
((performance.now() - start) / 1000).toFixed(1) + "s\x1b[0m",
|
||||
((performance.now() - start) / 1000).toFixed(1) +
|
||||
"s\x1b[0m",
|
||||
);
|
||||
|
||||
MediaFile.db.prepare("VACUUM").run();
|
||||
const { duration, count } = MediaFile.db.prepare<
|
||||
[],
|
||||
{ count: number; duration: number }
|
||||
>(`
|
||||
const { duration, count } = MediaFile.db
|
||||
.prepare<[], { count: number; duration: number }>(
|
||||
`
|
||||
select
|
||||
count(*) as count,
|
||||
sum(duration) as duration
|
||||
from media_files
|
||||
`).getNonNull();
|
||||
`,
|
||||
)
|
||||
.getNonNull();
|
||||
|
||||
console.info();
|
||||
console.info(
|
||||
|
|
@ -365,7 +378,7 @@ const execFile: typeof execFileRaw = ((
|
|||
) =>
|
||||
execFileRaw(...args).catch((e: any) => {
|
||||
if (e?.message?.startsWith?.("Command failed")) {
|
||||
if (e.code > (2 ** 31)) e.code |= 0;
|
||||
if (e.code > 2 ** 31) e.code |= 0;
|
||||
const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
|
||||
e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
|
||||
}
|
||||
|
|
@ -374,11 +387,7 @@ const execFile: typeof execFileRaw = ((
|
|||
const ffprobeBin = testProgram("ffprobe", "--help");
|
||||
const ffmpegBin = testProgram("ffmpeg", "--help");
|
||||
|
||||
const ffmpegOptions = [
|
||||
"-hide_banner",
|
||||
"-loglevel",
|
||||
"warning",
|
||||
];
|
||||
const ffmpegOptions = ["-hide_banner", "-loglevel", "warning"];
|
||||
|
||||
const procDuration: Process = {
|
||||
name: "calculate duration",
|
||||
|
|
@ -496,13 +505,10 @@ const procImageSubsets: Process = {
|
|||
for (const size of targetSizes) {
|
||||
const { w, h } = resizeDimensions(width, height, size);
|
||||
for (const { ext, args } of transcodeRules.imagePresets) {
|
||||
spin.text = baseStatus +
|
||||
` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
|
||||
spin.text = baseStatus + ` (${w}x${h}, ${ext.slice(1).toUpperCase()})`;
|
||||
|
||||
stack.use(
|
||||
await produceAsset(
|
||||
`${mediaFile.hash}/${size}${ext}`,
|
||||
async (out) => {
|
||||
await produceAsset(`${mediaFile.hash}/${size}${ext}`, async (out) => {
|
||||
await fs.mkdir(path.dirname(out));
|
||||
await fs.rm(out, { force: true });
|
||||
await execFile(ffmpegBin!, [
|
||||
|
|
@ -515,8 +521,7 @@ const procImageSubsets: Process = {
|
|||
out,
|
||||
]);
|
||||
return [out];
|
||||
},
|
||||
),
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
|
@ -561,21 +566,17 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
|
|||
if (config.encoder && typeof config.encoder.videoSrc === "string") {
|
||||
const { videoSrc, audioSrc, rate } = config.encoder;
|
||||
inputArgs = [
|
||||
...rate ? ["-r", String(rate)] : [],
|
||||
...(rate ? ["-r", String(rate)] : []),
|
||||
"-i",
|
||||
videoSrc,
|
||||
...audioSrc ? ["-i", audioSrc] : [],
|
||||
...(audioSrc ? ["-i", audioSrc] : []),
|
||||
];
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err?.code !== "ENOENT") throw err;
|
||||
}
|
||||
|
||||
const args = transcodeRules.getVideoArgs(
|
||||
preset,
|
||||
base,
|
||||
inputArgs,
|
||||
);
|
||||
const args = transcodeRules.getVideoArgs(preset, base, inputArgs);
|
||||
try {
|
||||
const fakeProgress = new Progress({ text: spin.text, spinner: null });
|
||||
fakeProgress.stop();
|
||||
|
|
@ -612,7 +613,9 @@ const procVideos = transcodeRules.videoFormats.map<Process>((preset) => ({
|
|||
const procCompression = [
|
||||
{ name: "gzip", fn: () => zlib.createGzip({ level: 9 }) },
|
||||
{ name: "zstd", fn: () => zlib.createZstdCompress() },
|
||||
].map(({ name, fn }) => ({
|
||||
].map(
|
||||
({ name, fn }) =>
|
||||
({
|
||||
name: `compress ${name}`,
|
||||
exclude: rules.extsPreCompressed,
|
||||
async run({ absPath, mediaFile }) {
|
||||
|
|
@ -627,7 +630,8 @@ const procCompression = [
|
|||
return [base];
|
||||
});
|
||||
},
|
||||
} satisfies Process as Process));
|
||||
}) satisfies Process as Process,
|
||||
);
|
||||
|
||||
const processors = [
|
||||
procDimensions,
|
||||
|
|
@ -637,16 +641,15 @@ const processors = [
|
|||
procImageSubsets,
|
||||
...procVideos,
|
||||
...procCompression,
|
||||
]
|
||||
.map((process, id, all) => {
|
||||
const strIndex = (id: number) =>
|
||||
String.fromCharCode("a".charCodeAt(0) + id);
|
||||
].map((process, id, all) => {
|
||||
const strIndex = (id: number) => String.fromCharCode("a".charCodeAt(0) + id);
|
||||
return {
|
||||
...process as Process,
|
||||
...(process as Process),
|
||||
id: strIndex(id),
|
||||
// Create a unique key.
|
||||
hash: new Uint16Array(
|
||||
crypto.createHash("sha1")
|
||||
crypto
|
||||
.createHash("sha1")
|
||||
.update(
|
||||
process.run.toString() +
|
||||
(process.version ? String(process.version) : ""),
|
||||
|
|
@ -660,7 +663,7 @@ const processors = [
|
|||
return strIndex(index);
|
||||
}),
|
||||
};
|
||||
});
|
||||
});
|
||||
|
||||
function resizeDimensions(w: number, h: number, desiredWidth: number) {
|
||||
ASSERT(desiredWidth < w, `${desiredWidth} < ${w}`);
|
||||
|
|
@ -676,10 +679,7 @@ async function produceAsset(
|
|||
if (asset.refs === 1) {
|
||||
const paths = await builder(path.join(workDir, key));
|
||||
asset.addFiles(
|
||||
paths.map((file) =>
|
||||
path.relative(workDir, file)
|
||||
.replaceAll("\\", "/")
|
||||
),
|
||||
paths.map((file) => path.relative(workDir, file).replaceAll("\\", "/")),
|
||||
);
|
||||
}
|
||||
return {
|
||||
|
|
@ -719,7 +719,7 @@ interface ProcessJob {
|
|||
absPath: string;
|
||||
stat: fs.Stats;
|
||||
mediaFile: MediaFile;
|
||||
processor: typeof processors[0];
|
||||
processor: (typeof processors)[0];
|
||||
index: number;
|
||||
after: ProcessJob[];
|
||||
needs: number;
|
||||
|
|
|
|||
|
|
@ -4,7 +4,9 @@
|
|||
font-weight: 400 750;
|
||||
font-style: normal;
|
||||
font-display: swap;
|
||||
font-variation-settings: "CASL" 0.25, "MONO" 0;
|
||||
font-variation-settings:
|
||||
"CASL" 0.25,
|
||||
"MONO" 0;
|
||||
font-style: oblique -15deg 0deg;
|
||||
unicode-range: U+0020-007E;
|
||||
}
|
||||
|
|
@ -14,7 +16,9 @@
|
|||
font-weight: 400 800;
|
||||
font-style: normal;
|
||||
font-display: swap;
|
||||
font-variation-settings: "CASL" 0.25, "MONO" 1;
|
||||
font-variation-settings:
|
||||
"CASL" 0.25,
|
||||
"MONO" 1;
|
||||
font-style: oblique -15deg 0deg;
|
||||
unicode-range: U+0020-007E;
|
||||
}
|
||||
|
|
@ -24,21 +28,13 @@
|
|||
font-weight: 400 800;
|
||||
font-style: normal;
|
||||
font-display: swap;
|
||||
font-variation-settings: "CASL" 0.25, "MONO" 1;
|
||||
font-variation-settings:
|
||||
"CASL" 0.25,
|
||||
"MONO" 1;
|
||||
font-style: oblique -15deg 0deg;
|
||||
unicode-range:
|
||||
U+00C0-00FF,
|
||||
U+00A9,
|
||||
U+2190-2193,
|
||||
U+2018,
|
||||
U+2019,
|
||||
U+201C,
|
||||
U+201D,
|
||||
U+2022,
|
||||
U+00A0-00A8,
|
||||
U+00AA-00BF,
|
||||
U+2194-2199,
|
||||
U+0100-017F;
|
||||
U+00C0-00FF, U+00A9, U+2190-2193, U+2018, U+2019, U+201C, U+201D, U+2022,
|
||||
U+00A0-00A8, U+00AA-00BF, U+2194-2199, U+0100-017F;
|
||||
}
|
||||
|
||||
*,
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ export const meta: Meta = {
|
|||
<main>
|
||||
<div>
|
||||
<h2>posts</h2>
|
||||
<p>song: <span>in the summer</span> (coming soon, 2025-07-12)</p>
|
||||
<p>song: <a href="/in-the-summer">in the summer</a> (2025-01-01)</p>
|
||||
<p>song: <a href="/waterfalls">waterfalls</a> (2025-01-01)</p>
|
||||
<h2>things</h2>
|
||||
<p><a href="/q+a">questions and answers</a></p>
|
||||
|
|
|
|||
Loading…
Reference in a new issue