feat: font subsetting

flake.nix

@@ -7,8 +7,12 @@
     with inputs.nixpkgs.legacyPackages.${system}; {
       devShells.default = pkgs.mkShell {
         buildInputs = [
+          # clover sitegen v3
           pkgs.nodejs_24 # runtime
           pkgs.deno # formatter
+          pkgs.python3 # for font subsetting
+
+          # paperclover.net
           (pkgs.ffmpeg.override {
             withOpus = true;
             withSvtav1 = true;

framework/backend/entry-node.ts

@@ -9,7 +9,7 @@ const server = serve({
   fetch: app.default.fetch,
   port: Number(process.env.PORT ?? 3000),
 }, ({ address, port }) => {
-  if (address === "::") address = "::1";
+  if (address === "::") address = "localhost";
   console.info(url.format({
     protocol,
     hostname: address,

framework/bundle.ts

@@ -94,7 +94,6 @@ export async function bundleClientJavaScript(
     const chunk = route.startsWith("/js/c.");
     if (!chunk) {
       const key = hot.getScriptId(toAbs(UNWRAP(entryPoint)));
-      console.log(route, key);
       route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
       scripts[key] = text;
     }

framework/font.ts

@@ -0,0 +1,192 @@
+export async function buildFonts(fonts: sg.Font[]) {
+  if (fonts.length === 0) return;
+
+  const temp = path.resolve(".clover/font");
+  const venv = path.join(temp, "venv");
+  const bin = path.join(venv, "bin");
+
+  if (!fs.existsSync(venv)) {
+    await subprocess.exec("python", ["-m", "venv", venv]);
+  }
+  if (!fs.existsSync(path.join(bin, "pyftsubset"))) {
+    await subprocess.exec(path.join(bin, "pip"), [
+      "install",
+      "fonttools==4.38.0",
+      "brotli==1.0.7",
+    ]);
+  }
+
+  const instances = new async.OnceMap<void>();
+  const subsets = new async.OnceMap<void>();
+  async function makeInstance(input: string, vars: sg.FontVars) {
+    const args = Object.entries(vars).map((x) => {
+      const lower = x[0].toLowerCase();
+      const k = lower === "wght" || lower === "slnt" ? lower : x[0];
+      const v = Array.isArray(x[1]) ? x[1].join(":") : x[1];
+      return `${k}=${v}`;
+    }).sort();
+    const hash = crypto
+      .createHash("sha256")
+      .update(`${input}${args.join(" ")}`)
+      .digest("hex")
+      .slice(0, 16)
+      .toLowerCase();
+    const outfile = path.join(temp, `${hash}${path.extname(input)}`);
+    await instances.get(outfile, async () => {
+      await fs.rm(outfile, { force: true });
+      await subprocess.exec(path.join(bin, "fonttools"), [
+        "varLib.instancer",
+        input,
+        ...args,
+        `--output=${outfile}`,
+      ]);
+      ASSERT(fs.existsSync(outfile));
+    });
+    return outfile;
+  }
+
+  await Promise.all(fonts.map((font) =>
+    incr.work(async (io, font) => {
+      const baseFile = await fetchFont(font.name, font.sources);
+      await Promise.all(font.subsets.map(async (subset) => {
+        let file = baseFile;
+        if (subset.vars) {
+          file = await makeInstance(baseFile, subset.vars);
+        }
+        const unicodes = fontRangesToString(
+          Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],
+        );
+        const hash = crypto
+          .createHash("sha256")
+          .update(`${file}${unicodes}`)
+          .digest("hex")
+          .slice(0, 16)
+          .toLowerCase();
+        const woff = path.join(temp, hash + ".woff2");
+        await subprocess.exec(path.join(bin, "pyftsubset"), [
+          file,
+          "--flavor=woff2",
+          `--output-file=${woff}`,
+          ...subset.layoutFeatures
+            ? [`--layout-features=${subset.layoutFeatures.join(",")}`]
+            : [],
+          `--unicodes=${unicodes}`,
+        ]);
+        await io.writeAsset({
+          pathname: subset.asset.replace("[ext]", "woff2"),
+          buffer: await fs.readFile(woff),
+        });
+      }));
+    }, font)
+  ));
+}
+
+export async function fetchFont(name: string, sources: string[]) {
+  const errs = [];
+
+  for (const source of sources) {
+    const cacheName = path.join("", name + path.extname(source));
+    if (fs.existsSync(cacheName)) return cacheName;
+
+    if (source.startsWith("https://")) {
+      const response = await fetch(source);
+      if (response.ok) {
+        await fs.writeMkdir(
+          cacheName,
+          Buffer.from(await response.arrayBuffer()),
+        );
+      } else {
+        errs.push(
+          new Error(
+            `Fetching from ${source} failed: ${response.status} ${response.statusText}`,
+          ),
+        );
+        continue;
+      }
+    }
+
+    if (path.isAbsolute(source)) {
+      if (fs.existsSync(source)) return source;
+      errs.push(new Error(`Font not available at absolute path ${source}`));
+    } else if (!source.includes("/") && !source.includes("\\")) {
+      const home = os.homedir();
+      const bases = process.platform === "win32"
+        ? [
+          "\\Windows\\Fonts",
+          path.win32.join(home, "AppData\\Local\\Microsoft\\Windows\\Fonts"),
+        ]
+        : process.platform === "darwin"
+        ? [
+          "/Library/Fonts",
+          path.posix.join(home, "Library/Fonts"),
+        ]
+        : [
+          "/usr/share/fonts",
+          "/usr/local/share/fonts",
+          path.posix.join(home, ".local/share/fonts"),
+        ];
+      for (const base of bases) {
+        const found = fs.readDirRecOptionalSync(base)
+          .find((file) => path.basename(file) === source);
+        if (found) {
+          return path.join(base, found);
+        }
+      }
+      errs.push(
+        new Error(
+          `Font file ${source} not found in the following directories: ${
+            bases.join(", ")
+          }`,
+        ),
+      );
+    }
+  }
+
+  throw new AggregateError(errs, `Failed to fetch the ${name} font`);
+}
+
+function fontRangesToString(ranges: sg.FontRange[]): string {
+  const segments: Array<{ start: number; end: number }> = [];
+
+  ranges.forEach((range) => {
+    if (typeof range === "string") {
+      for (let i = 0; i < range.length; i++) {
+        const cp = range.codePointAt(i) || 0;
+        segments.push({ start: cp, end: cp });
+      }
+    } else if (typeof range === "number") {
+      segments.push({ start: range, end: range });
+    } else {
+      segments.push({ start: range.start, end: range.end });
+    }
+  });
+
+  segments.sort((a, b) => a.start - b.start);
+
+  const merged: Array<{ start: number; end: number }> = [];
+  for (const seg of segments) {
+    const last = merged[merged.length - 1];
+    if (last && seg.start <= last.end + 1) {
+      last.end = Math.max(last.end, seg.end);
+    } else {
+      merged.push({ ...seg });
+    }
+  }
+
+  return merged.map(({ start, end }) =>
+    start === end
+      ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}`
+      : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${
+        end.toString(16).toUpperCase().padStart(4, "0")
+      }`
+  ).join(",");
+}
+
+import * as fs from "#sitegen/fs";
+import * as os from "node:os";
+import * as path from "node:path";
+import * as sg from "#sitegen";
+import * as subprocess from "#sitegen/subprocess";
+import * as incr from "./incremental.ts";
+import * as crypto from "node:crypto";
+import * as async from "#sitegen/async";

framework/generate.ts

@@ -22,6 +22,9 @@ export async function generate() {
 
   // TODO: loadMarkoCache
 
+  // -- start font work --
+  const builtFonts = fonts.buildFonts(siteConfig.fonts);
+
   // -- perform build-time rendering --
   const builtPages = pages.map((item) => incr.work(preparePage, item));
   const builtViews = views.map((item) => incr.work(prepareView, item));
@@ -70,7 +73,12 @@ export async function generate() {
     assembleAndWritePage(page, styleMap, scriptMap)
   );
 
-  await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);
+  await Promise.all([
+    builtBackend,
+    builtStaticFiles,
+    ...pAssemblePages,
+    builtFonts,
+  ]);
 }
 
 export async function readManifest(io: Io) {
@@ -80,6 +88,7 @@ export async function readManifest(io: Io) {
       root: toRel(section.root),
     })),
     backends: cfg.backends.map(toRel),
+    fonts: cfg.fonts,
   };
 }
 
@@ -324,14 +333,17 @@ export async function assembleAndWritePage(
 
 export type PageOrView = PreparedPage | PreparedView;
 
+import * as path from "node:path";
+
+import * as fs from "#sitegen/fs";
+import * as meta from "#sitegen/meta";
+import * as render from "#engine/render";
 import * as sg from "#sitegen";
-import * as incr from "./incremental.ts";
+import type { FileItem } from "#sitegen";
-import { Io } from "./incremental.ts";
+
 import * as bundle from "./bundle.ts";
 import * as css from "./css.ts";
-import * as render from "#engine/render";
+import * as fonts from "./font.ts";
 import * as hot from "./hot.ts";
-import * as fs from "#sitegen/fs";
+import * as incr from "./incremental.ts";
-import type { FileItem } from "#sitegen";
+import { Io } from "./incremental.ts";
-import * as path from "node:path";
-import * as meta from "#sitegen/meta";

framework/lib/assets.ts

@@ -2,6 +2,7 @@
 // This module implements decoding and serving of the asset blobs,
 // but also implements patching of dynamic assets. The `Manifest`
 // is generated by `incremental.ts`
+const debug = console.scoped("assets");
 const root = import.meta.dirname;
 let current: Loaded | null = null;
 
@@ -36,6 +37,7 @@ interface Loaded {
 export interface DynamicEntry extends AssetBase {
   buffer: Buffer;
 }
+type CacheMode = "auto" | "long-term" | "temporary" | "no-cache";
 
 export async function reload() {
   const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));
@@ -59,7 +61,7 @@ export async function reload() {
 export async function middleware(c: Context, next: Next) {
   if (!current) current = await reload();
   const asset = current.map[c.req.path];
-  if (asset) return assetInner(c, asset, 200);
+  if (asset) return assetInner(c, c.req.path, asset, 200);
   return next();
 }
 
@@ -68,16 +70,27 @@ export async function notFound(c: Context) {
   let pathname = c.req.path;
   do {
     const asset = current.map[pathname + "/404"];
-    if (asset) return assetInner(c, asset, 404);
+    if (asset) return assetInner(c, pathname + "/404", asset, 404);
     pathname = pathname.slice(0, pathname.lastIndexOf("/"));
   } while (pathname);
   const asset = current.map["/404"];
-  if (asset) return assetInner(c, asset, 404);
+  if (asset) return assetInner(c, "/404", asset, 404);
   return c.text("the 'Not Found' page was not found", 404);
 }
 
-export async function serveAsset(c: Context, id: Key, status: StatusCode) {
+export async function serveAsset(
-  return assetInner(c, (current ?? (await reload())).map[id], status);
+  c: Context,
+  id: Key,
+  status: StatusCode = 200,
+  cache: CacheMode = 'auto',
+) {
+  return assetInner(
+    c,
+    id,
+    (current ?? (await reload())).map[id],
+    status,
+    cache,
+  );
 }
 
 /** @deprecated */
@@ -89,21 +102,43 @@ export function etagMatches(etag: string, ifNoneMatch: string) {
   return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
 }
 
-function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) {
+function assetInner(
+  c: Context,
+  key: string,
+  asset: Manifest[Key],
+  status: StatusCode,
+  cache: CacheMode = "auto",
+) {
   ASSERT(current);
   if (asset.type === 0) {
-    return respondWithBufferAndViews(c, current.static, asset, status);
+    return respondWithBufferAndViews(
+      c,
+      key,
+      current.static,
+      asset,
+      status,
+      cache,
+    );
   } else {
     const entry = UNWRAP(current.dynamic.get(asset.id));
-    return respondWithBufferAndViews(c, entry.buffer, entry, status);
+    return respondWithBufferAndViews(
+      c,
+      key,
+      entry.buffer,
+      entry,
+      status,
+      cache,
+    );
   }
 }
 
 function respondWithBufferAndViews(
   c: Context,
+  key: string,
   buffer: Buffer,
   asset: AssetBase,
   status: StatusCode,
+  cache: CacheMode,
 ) {
   const ifNoneMatch = c.req.header("If-None-Match");
   if (ifNoneMatch) {
@@ -136,6 +171,29 @@ function respondWithBufferAndViews(
   } else {
     body = buffer.subarray(...asset.raw);
   }
+  debug(
+    `${key} encoding=${headers["Content-Encoding"] ?? "raw"} status=${status}`,
+  );
+  if (!Object.keys(headers).some((x) => x.toLowerCase() === "cache-control")) {
+    if (cache === "auto") {
+      if (status < 200 || status >= 300) {
+        cache = 'no-cache';
+      } else if (headers['content-type']?.includes('text/html')) {
+        cache = 'temporary';
+      } else {
+        cache = 'long-term';
+      }
+    }
+    if (cache === "no-cache") {
+      headers["Cache-Control"] = "no-store";
+    } else if (cache === "long-term") {
+      headers["Cache-Control"] =
+        "public, max-age=7200, stale-while-revalidate=7200, immutable";
+    } else if (cache === "temporary") {
+      headers["Cache-Control"] =
+        "public, max-age=7200, stale-while-revalidate=7200";
+    }
+  }
   return (c.res = new Response(body, { headers, status }));
 }
 
@@ -232,3 +290,4 @@ import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";
 import * as crypto from "node:crypto";
 import * as zlib from "node:zlib";
 import * as util from "node:util";
+import * as console from "@paperclover/console";

framework/lib/error.ts

@@ -0,0 +1,47 @@
+declare global {
+  interface Error {
+    /* Extra fields on errors are extremely helpful for debugging. */
+    [metadata: string]: unknown;
+  }
+}
+
+/** Retrieve an error message from any value */
+export function message(error: unknown): string {
+  const message = (error as { message: unknown })?.message ?? error;
+  try {
+    return typeof message === "string" ? message : JSON.stringify(message);
+  } catch {}
+  try {
+    return String(message);
+  } catch {}
+  return `Could not stringify error message ${typeof message}`;
+}
+
+/** Retrieve an error-like object from any value. Useful to check fields. */
+export function obj(error: unknown): { message: string } & Partial<Error> {
+  if (error instanceof Error) return error;
+  return {
+    message: message(error),
+    ...(typeof error === "object" && error),
+  };
+}
+
+/* Node.js error codes are strings */
+export function code(error: unknown): NodeErrorCode | null {
+  const code = (error as { code: unknown })?.code;
+  return typeof code === "string" ? code : null;
+}
+
+/** Attach extra fields and throw */
+export function rethrowWithMetadata(
+  err: unknown,
+  meta: Record<string, unknown>,
+): never {
+  const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/
+  Object.assign(error, meta);
+  throw error;
+}
+
+export type NodeErrorCode =
+  | keyof typeof import("node:os").constants.errno
+  | (string & {});

framework/lib/meta.ts

@@ -4,6 +4,9 @@ export interface Meta {
   openGraph?: OpenGraph;
   alternates?: Alternates;
 }
+export interface Template extends Omit<Meta, "title"> {
+  titleTemplate?: (title: string) => string,
+}
 export interface OpenGraph {
   title?: string;
   description?: string | undefined;
@@ -19,6 +22,6 @@ export interface AlternateType {
   title: string;
 }
 export function renderMeta({ title }: Meta): string {
-  return `<title>${esc(title)}</title>`;
+  return `<title>${esc(title)}</title><link rel="icon" type="image/x-icon" href="/favicon.ico">`;
 }
-import { escapeHtml as esc } from "#engine/render";
+import { escapeHtmlContent as esc } from "#engine/render";

framework/lib/mime.txt

@@ -90,6 +90,7 @@
 .v       text/x-v
 .wav     audio/x-wav
 .wmv     video/x-ms-wmv
+.woff2   font/woff2
 .xls     application/vnd.ms-excel
 .xlsx    application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
 .xml     application/xml

framework/lib/sitegen.ts

@@ -31,6 +31,24 @@ export interface FileItem {
 export interface Section {
   root: string;
 }
+export interface Font {
+  name: string;
+  /**
+   * Specify either font name, file path, or URL to fetch it.
+   * Private fonts do not have a URL and will fail to build if missing.
+   */
+  sources: string[];
+  subsets: Array<{
+    vars?: Record<string, FontVariableAxis>;
+    layoutFeatures?: string[];
+    unicodes: FontRange | FontRange[];
+    /** Include [ext] to autofill 'woff2' */
+    asset: string;
+  }>;
+}
+export type FontVars = Record<string, FontVariableAxis>;
+export type FontVariableAxis = number | [min: number, max: number];
+export type FontRange = string | number | { start: number, end: number };
 
 export const userData = render.userData<SitegenRender>(() => {
   throw new Error("This function can only be used in a page (static or view)");

framework/lib/sqlite.ts

@@ -11,9 +11,9 @@ export function getDb(file: string) {
   let db = map.get(file);
   if (db) return db;
   const fileWithExt = file.includes(".") ? file : file + ".sqlite";
-  db = new WrappedDatabase(
+  const dir = process.env.CLOVER_DB ?? ".clover";
-    path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),
+  fs.mkdirSync(dir);
-  );
+  db = new WrappedDatabase(path.join(dir, fileWithExt));
   map.set(file, db);
   return db;
 }
@@ -21,7 +21,7 @@ export function getDb(file: string) {
 export class WrappedDatabase {
   file: string;
   node: DatabaseSync;
-  stmts: Stmt[];
+  stmts: Stmt[] = [];
   stmtTableMigrate: WeakRef<StatementSync> | null = null;
 
   constructor(file: string) {
@@ -153,3 +153,4 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
 
 import { DatabaseSync, StatementSync } from "node:sqlite";
 import * as path from "node:path";
+import * as fs from "#sitegen/fs";

framework/lib/subprocess.ts

@@ -0,0 +1,15 @@
+const execFileRaw = util.promisify(child_process.execFile);
+export const exec: typeof execFileRaw = ((
+  ...args: Parameters<typeof execFileRaw>
+) =>
+  execFileRaw(...args).catch((e: any) => {
+    if (e?.message?.startsWith?.("Command failed")) {
+      if (e.code > 2 ** 31) e.code |= 0;
+      const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
+      e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
+    }
+    throw e;
+  })) as any;
+
+import * as util from 'node:util';
+import * as child_process from 'node:child_process';

framework/watch.ts

@@ -25,6 +25,7 @@ function serve() {
     "--development",
   ], {
     stdio: "inherit",
+    execArgv: ['--no-warnings'],
   });
   subprocess.on("close", onSubprocessClose);
 }

readme.md

@@ -11,23 +11,21 @@ that assist building websites. these tools power <https://paperclover.net>.
   - Different languages can be used at the same time. Supports `async function`
     components, `<Suspense />`, and custom extensions.
 - **Incremental static site generator and build system.**
-  - Build entire production site at start, incremental updates when pages
+  - Build both development and production sites on startup start. Incremental
-    change; Build system state survives coding sessions.
+    generator rebuilds changed pages; Build system state survives coding sessions.
-  - The only difference in development and production mode is hidden source-maps
+  - Multiple backend support.
-    and stripped `console.debug` calls. The site you see locally is the same
-    site you see deployed.
   - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
     checks when the files change. For example, changing a component re-tests
     only pages that use that component and re-lints only the changed file.
 - **Integrated libraries for building complex, content heavy web sites.**
   - Static asset serving with ETag and build-time compression.
-  - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`)
+  - Dynamicly rendered pages with static client. (`import "#sitegen/view"`)
   - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
   - TODO: Meta and Open Graph generation. (`export const meta`)
-  - TODO: Font subsetting tools to reduce bytes downloaded by fonts.
+  - Font subsetting to reduce page bandwidth.
-- **Built on the battle-tested Node.js runtime.**
+- **Built on the stable and battle-tested Node.js runtime.**
 
-None of these tools are complex or revolutionary. Rather, this project is the
+none of these tools are complex or revolutionary. rather, this project is the
 sum of many years of experience on managing content heavy websites, and an
 example on how other over-complicate other frameworks.
 
@@ -35,9 +33,11 @@ example on how other over-complicate other frameworks.
 
 Included is `src`, which contains `paperclover.net`. Website highlights:
 
+- TODO: flashy homepage.
 - [Question/Answer board, custom markdown parser and components][q+a].
 - [File viewer with fast ui/ux + optimized media streaming][file].
 - [Personal, friends-only blog with password protection][friends].
+- TODO: digital garden styled blog.
 
 [q+a]: https://paperclover.net/q+a
 [file]: https://paperclover.net/file
@@ -51,15 +51,21 @@ minimum system requirements:
 - random access memory.
 - windows 7 or later, macos, or other operating system.
 
+required software:
+
+- node.js v24
+- python v3
+
 my development machine, for example, is Dell Inspiron 7348 with Core i7
 
     npm install
 
-    # production generation
+    # build site using 'run.js' to enable runtime plugins
     node run generate
-    node .clover/o/backend
+    # the built site runs in regular node.js
+    node --enable-source-maps .clover/o/backend
 
-    # "development" watch mode
+    # watch-rebuild mode
     node run watch
 
 for unix systems, the provided `flake.nix` can be used with `nix develop` to
@@ -67,7 +73,7 @@ open a shell with all needed system dependencies.
 
 ## Deployment
 
-There are two primary server components to be deployed: the web server and the
+there are two primary server components to be deployed: the web server and the
 sourth of truth server. The latter is a singleton that runs on Clover's NAS,
 which holds the full contents of the file storage. The web server pulls data
 from the source of truth and renders web pages, and can be duplicated to
@@ -79,7 +85,8 @@ Deployment of the source of truth can be done with Docker Compose:
       backend:
         container_name: backend
         build:
-          # this uses loopback to hit the self-hosted git server
+          # this uses loopback to hit the self-hosted git server,
+          # docker will cache the image to not re-fetch on reboot.
           context: http://127.0.0.1:3000/clo/sitegen.git
           dockerfile: src/source-of-truth.dockerfile
         environment:
@@ -96,10 +103,23 @@ Deployment of the source of truth can be done with Docker Compose:
           - /mnt/storage1/clover/Documents/Config/paperclover:/data
           - /mnt/storage1/clover/Published:/published
 
-Due to caching, one may need to manually purge images via
+Due to caching, images may need to be purged via `docker image rm {image} -f`
-`docker image rm ix-clover-backend -f` when an update is desired
+when an update is desired. Some docker GUIs support force pulls, some are buggy.
 
-TODO: deployment instructions for a web node
+The web server performs rendering. A `Dockerfile` for it is present in
+`src/web.dockerfile` but it is currently unused. Deployments are done by
+building the project locally, and then using `rsync` to copy files.
+
+    node run generate
+    rsync .clover/o "$REMOTE_USER:~/paperclover" --exclude=.clover --exclude=.env \
+      --delete-after --progress --human-readable
+    ssh "$REMOTE_USER" /bin/bash << EOF
+        set -e
+        cd ~/paperclover
+        npm ci
+        pm2 restart site
+        echo "-> https://paperclover.net"
+    EOF
 
 ## Contributions
 

run.js

@@ -3,7 +3,6 @@
 import * as util from "node:util";
 import * as zlib from "node:zlib";
 import * as url from "node:url";
-import * as module from "node:module";
 import process from "node:process";
 
 // Disable experimental warnings (Type Stripping, etc)
@@ -28,8 +27,8 @@ try {
   const brand = process.versions.bun
     ? `bun ${process.versions.bun}`
     : process.versions.deno
-    ? `deno ${process.versions.deno}`
+      ? `deno ${process.versions.deno}`
-    : null;
+      : null;
   const brandText = brand ? ` (${brand})` : "";
   globalThis.console.error(
     `sitegen depends on a node.js v24. your runtime is missing features.\n` +
@@ -75,16 +74,26 @@ if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
     else if (mod.default?.fetch) {
       const protocol = "http";
       const { serve } = hot.load("@hono/node-server");
-      serve({
+      serve(
-        fetch: mod.default.fetch,
+        {
-      }, ({ address, port }) => {
+          fetch: mod.default.fetch,
-        if (address === "::") address = "::1";
+        },
-        console.info(url.format({
+        ({ address, port }) => {
-          protocol,
+          if (address === "::") address = "::1";
-          hostname: address,
+          console.info(
-          port,
+            url.format({
-        }));
+              protocol,
-      });
+              hostname: address,
+              port,
+            }),
+          );
+        },
+      );
+    } else {
+      console.warn(
+        hot.load("path").relative('.', found),
+        'does not export a "main" function',
+      );
     }
   } catch (e) {
     console.error(util.inspect(e));

src/admin.ts

@@ -3,7 +3,11 @@ const cookieAge = 60 * 60 * 24 * 365; // 1 year
 let lastKnownToken: string | null = null;
 function compareToken(token: string) {
   if (token === lastKnownToken) return true;
-  lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
+  try {
+    lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
+  } catch {
+    return false;
+  }
   return token === lastKnownToken;
 }
 

src/backend.ts

@@ -15,6 +15,20 @@ app.route("", require("./file-viewer/backend.tsx").app);
 // Asset middleware has least precedence
 app.use(assets.middleware);
 
+// Asset Aliases
+app.get(
+  "/apple-touch-icon-precomposed.png",
+  (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
+);
+app.get(
+  "/apple-touch-icon-57x57.png",
+  (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
+);
+app.get(
+  "/apple-touch-icon-57x57-precomposed.png",
+  (c) => assets.serveAsset(c, "/apple-touch-icon.png"),
+);
+
 // Handlers
 app.notFound(assets.notFound);
 

src/file-viewer/bin/scan3.ts

@@ -10,7 +10,6 @@
 // This is the third iteration of the scanner, hence its name "scan3";
 // Remember that any software you want to be maintainable and high
 // quality cannot be written with AI.
-const root = path.resolve("/Volumes/clover/Published");
 const workDir = path.resolve(".clover/derived");
 const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY);
 
@@ -404,18 +403,6 @@ interface Process {
   undo?(mediaFile: MediaFile): Promise<void>;
 }
 
-const execFileRaw = util.promisify(child_process.execFile);
-const execFile: typeof execFileRaw = ((
-  ...args: Parameters<typeof execFileRaw>
-) =>
-  execFileRaw(...args).catch((e: any) => {
-    if (e?.message?.startsWith?.("Command failed")) {
-      if (e.code > 2 ** 31) e.code |= 0;
-      const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
-      e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
-    }
-    throw e;
-  })) as any;
 const ffprobeBin = testProgram("ffprobe", "--help");
 const ffmpegBin = testProgram("ffmpeg", "--help");
 
@@ -426,7 +413,7 @@ const procDuration: Process = {
   enable: ffprobeBin !== null,
   include: rules.extsDuration,
   async run({ absPath, mediaFile }) {
-    const { stdout } = await execFile(ffprobeBin!, [
+    const { stdout } = await subprocess.exec(ffprobeBin!, [
       "-v",
       "error",
       "-show_entries",
@@ -793,11 +780,11 @@ import { Progress } from "@paperclover/console/Progress";
 import { Spinner } from "@paperclover/console/Spinner";
 import * as async from "#sitegen/async";
 import * as fs from "#sitegen/fs";
+import * as subprocess from "#sitegen/subprocess";
 
 import * as path from "node:path";
 import * as zlib from "node:zlib";
 import * as child_process from "node:child_process";
-import * as util from "node:util";
 import * as crypto from "node:crypto";
 import * as stream from "node:stream";
 
@@ -814,3 +801,4 @@ import * as highlight from "@/file-viewer/highlight.ts";
 import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
 import * as rsync from "@/file-viewer/rsync.ts";
 import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
+import { rawFileRoot as root } from "../paths.ts";

src/file-viewer/cache.ts

@@ -12,9 +12,9 @@ import { escapeUri } from "./format.ts";
 declare const Deno: any;
 
 const sourceOfTruth = "https://nas.paperclover.net:43250";
-const caCert = fs.readFileSync("src/file-viewer/cert.pem");
+// const caCert = fs.readFileSync("src/file-viewer/cert.pem");
 
-const diskCacheRoot = path.join(import.meta.dirname, "../.clover/filecache/");
+const diskCacheRoot = path.join(import.meta.dirname, ".filecache/");
 const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB
 const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB
 const loadInProgress = new Map<
@@ -338,7 +338,7 @@ const agent: any = typeof Bun !== "undefined"
     // Bun has two non-standard fetch extensions
     decompress: false,
     tls: {
-      ca: caCert,
+      // ca: caCert,
     },
   }
   // TODO: https://github.com/denoland/deno/issues/12291
@@ -351,7 +351,7 @@ const agent: any = typeof Bun !== "undefined"
   // }
   // Node.js supports node:http
   : new Agent({
-    ca: caCert,
+    // ca: caCert,
   });
 
 function fetchFileNode(pathname: string): Promise<ReadableStream> {

src/file-viewer/paths.ts

@@ -0,0 +1,12 @@
+export const nasRoot = process.platform === "win32"
+  ? "\\\\zenith\\clover"
+  : process.platform === "darwin"
+  ? "/Volumes/clover"
+  : "/media/clover";
+
+export const rawFileRoot = process.env.CLOVER_FILE_RAW ??
+  path.join(nasRoot, "Published");
+export const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
+  path.join(nasRoot, "Documents/Config/paperclover/derived");
+
+import * as path from "node:path";

src/site.ts

@@ -1,11 +1,11 @@
-// This file defines the different "Sections" of the website. The sections act
-// as separate codebases, but hosted as one. This allows me to have
-// sub-projects like the file viewer in 'file', or the question answer system
-// in 'q+a'. Each section can define configuration, pages, backend routes, and
-// contain other files.
 const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths);
 
-export const siteSections: Section[] = [
+// Different sections of the website are split into their own folders. Acting as
+// as separate codebases, they are hosted as one. This allows me to have
+// sub-projects like the file viewer in 'file/', and the question answer system
+// in 'q+a', but maintain clear boundaries. Each section can define
+// configuration, pages, backend routes, and contain other files.
+export const siteSections: sg.Section[] = [
   { root: join(".") },
   { root: join("q+a/") },
   { root: join("file-viewer/") },
@@ -14,10 +14,96 @@ export const siteSections: Section[] = [
   // { root: join("fiction/"), pageBase: "/fiction" },
 ];
 
+// All backends are bundled. The backend named "backend" is run by "node run watch"
 export const backends: string[] = [
   join("backend.ts"),
   join("source-of-truth.ts"),
 ];
 
+// Font subsets reduce bandwidth and protect against proprietary font theft.
+const fontRoot = path.join(nasRoot, 'Documents/Font');
+const ascii = { start: 0x20, end: 0x7E };
+const nonAscii: sg.FontRange[] = [
+  { start: 0xC0, end: 0xFF },
+  { start: 0x2190, end: 0x2193 },
+  { start: 0xA0, end: 0xA8 },
+  { start: 0xAA, end: 0xBF },
+  { start: 0x2194, end: 0x2199 },
+  { start: 0x100, end: 0x17F },
+  0xA9,
+  0x2018,
+  0x2019,
+  0x201C,
+  0x201D,
+  0x2022,
+];
+const recursiveVars: sg.FontVars = {
+  WGHT: [400, 750],
+  SLNT: [-15, 0],
+  CASL: 0.25,
+  CRSV: 0.5,
+  MONO: 0,
+};
+const recursiveVarsMono = { ...recursiveVars, MONO: 1 };
+const recursiveVarsQuestion: sg.FontVars = {
+  ...recursiveVars,
+  MONO: [0, 1],
+  WGHT: [400, 1000],
+};
+const layoutFeatures = ["numr", "dnom", "frac"];
+export const fonts: sg.Font[] = [
+  {
+    name: "Recursive",
+    sources: [
+      "Recursive_VF_1.085.ttf",
+      path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
+      "https://paperclover.net/file/_unlisted/Recursive_VF_1.085.ttf",
+    ],
+    subsets: [
+      {
+        asset: "/recultramin.[ext]",
+        layoutFeatures,
+        vars: recursiveVars,
+        unicodes: ascii,
+      },
+      {
+        asset: "/recmono.[ext]",
+        vars: recursiveVarsMono,
+        unicodes: ascii,
+      },
+      {
+        asset: "/recqa.[ext]",
+        vars: recursiveVarsQuestion,
+        unicodes: ascii,
+      },
+      {
+        asset: "/recexotic.[ext]",
+        vars: recursiveVarsQuestion,
+        unicodes: nonAscii,
+      },
+    ],
+  },
+  {
+    name: "AT Name Sans Display Hairline",
+    sources: [
+      "ATNameSansDisplay-Hairline.woff2",
+      path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
+    ],
+    subsets: [
+      {
+        asset: "/cydn_header.[ext]",
+        layoutFeatures,
+        unicodes: "cotlyedon",
+      },
+    ],
+  },
+];
+
+export async function main() {
+  await font.buildFonts(fonts);
+}
+
 import * as path from "node:path";
-import type { Section } from "#sitegen";
+import * as font from "../framework/font.ts";
+import type * as sg from "#sitegen";
+import { nasRoot } from "./file-viewer/paths.ts";

src/source-of-truth.ts

@@ -22,17 +22,6 @@ export default app;
 
 const token = UNWRAP(process.env.CLOVER_SOT_KEY);
 
-const nasRoot = process.platform === "win32"
-  ? "\\\\zenith\\clover"
-  : process.platform === "darwin"
-  ? "/Volumes/clover"
-  : "/media/clover";
-
-const rawFileRoot = process.env.CLOVER_FILE_RAW ??
-  path.join(nasRoot, "Published");
-const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
-  path.join(nasRoot, "Documents/Config/paperclover/derived");
-
 if (!fs.existsSync(rawFileRoot)) {
   throw new Error(`${rawFileRoot} does not exist`);
 }
@@ -125,3 +114,4 @@ import * as fsCallbacks from "node:fs";
 import * as util from "node:util";
 import * as stream from "node:stream";
 import * as mime from "#sitegen/mime";
+import { derivedFileRoot, rawFileRoot } from "./file-viewer/paths.ts";

src/web.dockerfile

@@ -0,0 +1,19 @@
+from node:24 as builder
+
+run apt install -y git
+
+workdir /tmp/builder
+copy package*.json ./
+run npm ci
+copy . ./
+run node run generate
+run npm prune --production && cp -r .clover/o /app && cp -r node_modules /app/
+
+from node:24 as runtime
+
+workdir /app
+copy --from=builder /app/ ./
+
+env PORT=80
+
+cmd ["node", "--enable-source-maps", "/app/backend.js"]