From b6a172a160ee26dcaf72195ca15126c59e015d58 Mon Sep 17 00:00:00 2001 From: chloe caruso Date: Sat, 2 Aug 2025 20:10:32 -0400 Subject: [PATCH] finish it working --- framework/bundle.ts | 6 +- framework/hot.ts | 15 +++- framework/incremental.ts | 110 +++++++++++++++------------ framework/watch.ts | 158 +++++++++++++++++---------------------- src/global.css | 2 +- src/pages/index.marko | 2 +- 6 files changed, 144 insertions(+), 149 deletions(-) diff --git a/framework/bundle.ts b/framework/bundle.ts index bd650b7..3d8f859 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -173,7 +173,7 @@ export async function bundleServerJavaScript( }, ]; - const { metafile, outputFiles } = await esbuild.build({ + const { metafile, outputFiles, errors, warnings } = await esbuild.build({ bundle: true, chunkNames: "c.[hash]", entryNames: path.basename(entry, path.extname(entry)), @@ -194,7 +194,7 @@ export async function bundleServerJavaScript( jsxDev: false, define: { MIME_INLINE_DATA: JSON.stringify(mime.rawEntriesText), - CLOVER_SERVER_ENTRY: JSON.stringify(entry), + 'globalThis.CLOVER_SERVER_ENTRY': JSON.stringify(entry), }, external: Object.keys(pkg.dependencies) .filter((x) => !x.startsWith("@paperclover")), @@ -208,7 +208,7 @@ export async function bundleServerJavaScript( } | null = null; for (const output of outputFiles) { const basename = output.path.replace(/^.*?!(?:\/|\\)/, ""); - const key = "out!" + basename.replaceAll("\\", "/"); + const key = "out!/" + basename.replaceAll("\\", "/"); // If this contains the generated "$views" file, then // mark this file as the one for replacement. Because // `splitting` is `true`, esbuild will not emit this diff --git a/framework/hot.ts b/framework/hot.ts index b5bc817..bd1dc91 100644 --- a/framework/hot.ts +++ b/framework/hot.ts @@ -64,7 +64,8 @@ Module.prototype._compile = function ( const stat = fs.statSync(filename); const cssImportsMaybe: string[] = []; const imports: string[] = []; - for (const { filename: file, cloverClientRefs } of this.children) { + for (const childModule of this.children) { + const { filename: file, cloverClientRefs } = childModule; if (file.endsWith(".css")) cssImportsMaybe.push(file); else { const child = fileStats.get(file); @@ -72,6 +73,7 @@ Module.prototype._compile = function ( const { cssImportsRecursive } = child; if (cssImportsRecursive) cssImportsMaybe.push(...cssImportsRecursive); imports.push(file); + (childModule.cloverImporters ??= []).push(this); if (cloverClientRefs && cloverClientRefs.length > 0) { (this.cloverClientRefs ??= []) .push(...cloverClientRefs); @@ -193,9 +195,13 @@ export function reloadRecursive(filepath: string) { export function unload(filepath: string) { filepath = path.resolve(filepath); - const existing = cache[filepath]; - if (existing) delete cache[filepath]; - fileStats.delete(filepath); + const module = cache[filepath]; + if (!module) return; + delete cache[filepath]; + lazyMarko?.markoCache.delete(filepath) + for (const importer of module.cloverImporters ?? []) { + unload(importer.filename); + } } function deleteRecursiveInner(id: string, module: any) { @@ -294,6 +300,7 @@ declare global { interface Module { cloverClientRefs?: string[]; cloverSourceCode?: string; + cloverImporters?: Module[], _compile( this: NodeJS.Module, diff --git a/framework/incremental.ts b/framework/incremental.ts index 8152145..c2d7dc3 100644 --- a/framework/incremental.ts +++ b/framework/incremental.ts @@ -30,10 +30,8 @@ type Job = (io: Io, input: I) => Promise; export function work(job: Job): Ref; export function work(job: Job, input: I): Ref; export function work(job: Job, input: I = null as I): Ref { - const keySource = [ - JSON.stringify(util.getCallSites(2)[1]), - util.inspect(input), - ].join(":"); + const source = JSON.stringify(util.getCallSites(2)[1]); + const keySource = [source, util.inspect(input)].join(":"); const key = crypto.createHash("sha1").update(keySource).digest("base64url"); ASSERT(running); ASSERT( @@ -61,6 +59,7 @@ export function work(job: Job, input: I = null as I): Ref { affects: [], reads, writes, + debug: source, }); for (const add of reads.files) { const { affects } = UNWRAP(files.get(add)); @@ -107,11 +106,17 @@ export async function compile(compiler: () => Promise) { timerSpinner.text = "incremental flush"; await flush(start); timerSpinner.stop(); - seenWorks.clear(); - newKeys = 0; - return { value }; + return { + value, + watchFiles: new Set(files.keys()), + newOutputs: Array.from(seenWrites).filter(x => x.startsWith('f:')).map(x => x.slice(2)), + newAssets: !Array.from(seenWrites).some(x => x.startsWith('a:')), + }; } finally { running = false; + newKeys = 0; + seenWrites.clear(); + seenWorks.clear(); } } @@ -178,13 +183,7 @@ export async function restore() { await deserialize(buffer); } -export function forceInvalidate(file: string) { - const resolved = toAbs(file); - const key = toRel(resolved); - forceInvalidateEntry(UNWRAP(files.get(key), `Untracked file '${file}'`)); -} - -export function forceInvalidateEntry(entry: { affects: string[] }) { +function forceInvalidate(entry: { affects: string[] }) { const queue = [...entry.affects]; let key; while ((key = queue.shift())) { @@ -194,8 +193,9 @@ export function forceInvalidateEntry(entry: { affects: string[] }) { } function deleteWork(key: string) { - console.info({ key }); - const { reads, affects, writes: w } = UNWRAP(works.get(key)); + const work = works.get(key); + if (!work) return []; + const { reads, affects, writes: w } = work; for (const remove of reads.files) { const { affects } = UNWRAP(files.get(remove)); ASSERT(affects.includes(key)); @@ -325,43 +325,54 @@ async function deserialize(buffer: Buffer) { work, }); } else { - forceInvalidateEntry({ affects: [work] }); + forceInvalidate({ affects: [work] }); } } for (const [hash, raw, gzip, zstd] of assetEntries) { assets.set(hash, { raw, gzip, zstd }); } - await Promise.all(Array.from(files, async ([k, file]) => { - try { - if (file.type === "d") { - const contents = file.contents = await fs.readdir(k); - contents.sort(); - const contentHash = crypto - .createHash("sha1") - .update(contents.join("\0")) - .digest("base64url"); - if (file.contentHash !== contentHash) { - file.contentHash = contentHash; - throw new Error(); - } - } else if (file.type === 'f') { - const lastModified = await fs.stat(k) - .then(x => Math.floor(x.mtimeMs), () => 0); - if (file.lastModified !== lastModified) { - file.lastModified = lastModified; - throw new Error(); - } - } else { - file.type satisfies 'null'; - const stat = await fs.stat(k).catch(() => null); - if (stat) throw new Error(); + await Promise.all(Array.from(files, ([key, file]) => invalidateEntry(key, file))); +} + +export async function invalidate(filePath: string): Promise { + const key = toRel(toAbs(filePath)); + const file = UNWRAP(files.get(key), `Untracked file '${key}'`) + return invalidateEntry(key, file) +} + +export async function invalidateEntry(key: string, file: TrackedFile): Promise { + try { + if (file.type === "d") { + const contents = file.contents = await fs.readdir(key); + contents.sort(); + const contentHash = crypto + .createHash("sha1") + .update(contents.join("\0")) + .digest("base64url"); + if (file.contentHash !== contentHash) { + file.contentHash = contentHash; + throw new Error(); } - } catch (e) { - forceInvalidateEntry(file); - if (file.type === 'null') files.delete(k); + } else if (file.type === 'f') { + const lastModified = await fs.stat(key) + .then(x => Math.floor(x.mtimeMs), () => 0); + if (file.lastModified !== lastModified) { + file.lastModified = lastModified; + throw new Error(); + } + } else { + file.type satisfies 'null'; + const stat = await fs.stat(key).catch(() => null); + if (stat) throw new Error(); } - })); + return false; + } catch (e) { + forceInvalidate(file); + hot.unload(toAbs(key)); + if (file.type === 'null') files.delete(key); + return true; + } } export function getAssetManifest() { @@ -377,8 +388,8 @@ export function getAssetManifest() { ); return [key, { raw: writer.write(raw, "raw:" + hash), - gzip: writer.write(gzip, "raw:" + hash), - zstd: writer.write(zstd, "raw:" + hash), + gzip: writer.write(gzip, "gzip:" + hash), + zstd: writer.write(zstd, "zstd:" + hash), headers, }] as const; }) @@ -577,7 +588,7 @@ export function validateSerializable(value: unknown, key: string) { } else if (value && typeof value === "object") { if (Array.isArray(value)) { value.forEach((item, i) => validateSerializable(item, `${key}[${i}]`)); - } else if (Object.getPrototypeOf(value) === Object.prototype) { + } else if (Object.getPrototypeOf(value) === Object.prototype || Buffer.isBuffer(value)) { Object.entries(value).forEach(([k, v]) => validateSerializable(v, `${key}.${k}`) ); @@ -626,6 +637,7 @@ interface Asset { zstd: Buffer; } interface Work { + debug?: string; value: T; reads: Reads; writes: Writes; @@ -637,7 +649,7 @@ type TrackedFile = } & ( | { type: "f"; lastModified: number } - | { type: "d"; contentHash: string; contents: string[] | null } + | { type: "d"; contentHash: string; contents: string[] } | { type: "null"; } ); export interface BuiltAssetMap { diff --git a/framework/watch.ts b/framework/watch.ts index 14a39b4..4fcf1f0 100644 --- a/framework/watch.ts +++ b/framework/watch.ts @@ -2,102 +2,67 @@ const debounceMilliseconds = 25; +let subprocess: child_process.ChildProcess | null = null; +process.on("beforeExit", () => { + subprocess?.removeListener("close", onSubprocessClose); +}); + +let watch: Watch; + export async function main() { - let subprocess: child_process.ChildProcess | null = null; - // Catch up state by running a main build. - const { incr } = await generate.main(); - // ...and watch the files that cause invals. - const watch = new Watch(rebuild); - watch.add(...incr.invals.keys()); - statusLine(); - // ... and then serve it! - serve(); + await incr.restore(); + watch = new Watch(rebuild); + rebuild([]); +} - function serve() { - if (subprocess) { - subprocess.removeListener("close", onSubprocessClose); - subprocess.kill(); - } - subprocess = child_process.fork(".clover/out/server.js", [ - "--development", - ], { - stdio: "inherit", - }); - subprocess.on("close", onSubprocessClose); +function serve() { + if (subprocess) { + subprocess.removeListener("close", onSubprocessClose); + subprocess.kill(); } - - function onSubprocessClose(code: number | null, signal: string | null) { - subprocess = null; - const status = code != null ? `code ${code}` : `signal ${signal}`; - console.error(`Backend process exited with ${status}`); - } - - process.on("beforeExit", () => { - subprocess?.removeListener("close", onSubprocessClose); + subprocess = child_process.fork(".clover/o/backend.js", [ + "--development", + ], { + stdio: "inherit", }); + subprocess.on("close", onSubprocessClose); +} - function rebuild(files: string[]) { - files = files.map((file) => path.relative(hot.projectRoot, file)); - const changed: string[] = []; - for (const file of files) { - let mtimeMs: number | null = null; - try { - mtimeMs = fs.statSync(file).mtimeMs; - } catch (err: any) { - if (err?.code !== "ENOENT") throw err; - } - if (incr.updateStat(file, mtimeMs)) changed.push(file); - } - if (changed.length === 0) { - console.warn("Files were modified but the 'modify' time did not change."); - return; - } - withSpinner>>({ - text: "Rebuilding", - successText: generate.successText, - failureText: () => "sitegen FAIL", - }, async (spinner) => { - console.info("---"); - console.info( - "Updated" + - (changed.length === 1 - ? " " + changed[0] - : changed.map((file) => "\n- " + file)), - ); - const result = await generate.sitegen(spinner, incr); - incr.toDisk(); // Allows picking up this state again - for (const file of watch.files) { - const relative = path.relative(hot.projectRoot, file); - if (!incr.invals.has(relative)) watch.remove(file); - } - return result; - }).then((result) => { - // Restart the server if it was changed or not running. - if ( - !subprocess || - result.inserted.some(({ kind }) => kind === "backendReplace") - ) { - serve(); - } else if ( - subprocess && - result.inserted.some(({ kind }) => kind === "asset") - ) { - subprocess.send({ type: "clover.assets.reload" }); - } - return result; - }).catch((err) => { - console.error(util.inspect(err)); - }).finally(statusLine); - } +function onSubprocessClose(code: number | null, signal: string | null) { + subprocess = null; + const status = code != null ? `code ${code}` : `signal ${signal}`; + console.error(`Backend process exited with ${status}`); +} - function statusLine() { - console.info( - `Watching ${incr.invals.size} files \x1b[36m[last change: ${ - new Date().toLocaleTimeString() - }]\x1b[39m`, - ); +function rebuild(files: string[]) { + for (const file of files) { + incr.invalidate(file); } + incr.compile(generate.generate).then(({ + watchFiles, + newOutputs, + newAssets + }) => { + const removeWatch = [...watch.files].filter(x => !watchFiles.has(x)) + for (const file of removeWatch) watch.remove(file); + watch.add(...watchFiles); + // Restart the server if it was changed or not running. + if (!subprocess || newOutputs.includes("backend.js")) { + serve(); + } else if (subprocess && newAssets) { + subprocess.send({ type: "clover.assets.reload" }); + } + }).catch((err) => { + console.error(util.inspect(err)); + }).finally(statusLine); +} + +function statusLine() { + console.info( + `Watching ${watch.files.size} files ` + + `\x1b[36m[last change: ${new Date().toLocaleTimeString()}]\x1b[39m`, + ); } class Watch { @@ -174,11 +139,21 @@ class Watch { for (const w of this.watchers) w.close(); } + #getFiles(absPath: string, event: fs.WatchEventType) { + const files = []; + if (this.files.has(absPath)) files.push(absPath); + if (event === 'rename') { + const dir = path.dirname(absPath); + if (this.files.has(dir)) files.push(dir); + } + return files; + } + #handleEvent(root: string, event: fs.WatchEventType, subPath: string | null) { if (!subPath) return; - const file = path.join(root, subPath); - if (!this.files.has(file)) return; - this.stale.add(file); + const files = this.#getFiles(path.join(root, subPath), event); + if (files.length === 0) return; + for(const file of files) this.stale.add(file); const { debounce } = this; if (debounce !== null) clearTimeout(debounce); this.debounce = setTimeout(() => { @@ -192,6 +167,7 @@ class Watch { import * as fs from "node:fs"; import { withSpinner } from "@paperclover/console/Spinner"; import * as generate from "./generate.ts"; +import * as incr from "./incremental.ts"; import * as path from "node:path"; import * as util from "node:util"; import * as hot from "./hot.ts"; diff --git a/src/global.css b/src/global.css index 27d24e3..0e29c27 100644 --- a/src/global.css +++ b/src/global.css @@ -59,7 +59,7 @@ main { } h1 { - font-size: 2.5em; + font-size: 2em; } h1, diff --git a/src/pages/index.marko b/src/pages/index.marko index 50d72b1..047da17 100644 --- a/src/pages/index.marko +++ b/src/pages/index.marko @@ -29,7 +29,7 @@ export const meta: Meta = {

posts

-

song: in the summer (coming soon, 2025-07-12)

+

song: in the summer (2025-01-01)

song: waterfalls (2025-01-01)

things

questions and answers