clo/sitegen
1
0
Fork 0
Code Issues 20 Pull requests Projects Releases Packages Wiki Activity Actions

feat: font subsetting

Browse source
This commit is contained in:
clover caruso 2025-08-14 20:35:33 -07:00
parent cb12824da4
commit c9d24a4fdd
25 changed files with 685 additions and 191 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
flake.nix
View file

@ -7,8 +7,12 @@
with inputs.nixpkgs.legacyPackages.${system}; {
devShells.default = pkgs.mkShell {
buildInputs = [
# clover sitegen v3
pkgs.nodejs_24 # runtime
pkgs.deno # formatter
pkgs.python3 # for font subsetting
# paperclover.net
(pkgs.ffmpeg.override {
withOpus = true;
withSvtav1 = true;

2
framework/backend/entry-node.ts
View file

@ -9,7 +9,7 @@ const server = serve({
fetch: app.default.fetch,
port: Number(process.env.PORT ?? 3000),
}, ({ address, port }) => {
if (address === "::") address = "::1";
if (address === "::") address = "localhost";
console.info(url.format({
protocol,
hostname: address,

1
framework/bundle.ts
View file

@ -94,7 +94,6 @@ export async function bundleClientJavaScript(
const chunk = route.startsWith("/js/c.");
if (!chunk) {
const key = hot.getScriptId(toAbs(UNWRAP(entryPoint)));
console.log(route, key);
route = "/js/" + key.replace(/\.client\.tsx?/, ".js");
scripts[key] = text;
}

192
framework/font.ts Normal file
View file

@ -0,0 +1,192 @@
export async function buildFonts(fonts: sg.Font[]) {
if (fonts.length === 0) return;
const temp = path.resolve(".clover/font");
const venv = path.join(temp, "venv");
const bin = path.join(venv, "bin");
if (!fs.existsSync(venv)) {
await subprocess.exec("python", ["-m", "venv", venv]);
}
if (!fs.existsSync(path.join(bin, "pyftsubset"))) {
await subprocess.exec(path.join(bin, "pip"), [
"install",
"fonttools==4.38.0",
"brotli==1.0.7",
]);
}
const instances = new async.OnceMap<void>();
const subsets = new async.OnceMap<void>();
async function makeInstance(input: string, vars: sg.FontVars) {
const args = Object.entries(vars).map((x) => {
const lower = x[0].toLowerCase();
const k = lower === "wght" || lower === "slnt" ? lower : x[0];
const v = Array.isArray(x[1]) ? x[1].join(":") : x[1];
return `${k}=${v}`;
}).sort();
const hash = crypto
.createHash("sha256")
.update(`${input}${args.join(" ")}`)
.digest("hex")
.slice(0, 16)
.toLowerCase();
const outfile = path.join(temp, `${hash}${path.extname(input)}`);
await instances.get(outfile, async () => {
await fs.rm(outfile, { force: true });
await subprocess.exec(path.join(bin, "fonttools"), [
"varLib.instancer",
input,
...args,
`--output=${outfile}`,
]);
ASSERT(fs.existsSync(outfile));
});
return outfile;
}
await Promise.all(fonts.map((font) =>
incr.work(async (io, font) => {
const baseFile = await fetchFont(font.name, font.sources);
await Promise.all(font.subsets.map(async (subset) => {
let file = baseFile;
if (subset.vars) {
file = await makeInstance(baseFile, subset.vars);
}
const unicodes = fontRangesToString(
Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes],
);
const hash = crypto
.createHash("sha256")
.update(`${file}${unicodes}`)
.digest("hex")
.slice(0, 16)
.toLowerCase();
const woff = path.join(temp, hash + ".woff2");
await subprocess.exec(path.join(bin, "pyftsubset"), [
file,
"--flavor=woff2",
`--output-file=${woff}`,
...subset.layoutFeatures
? [`--layout-features=${subset.layoutFeatures.join(",")}`]
: [],
`--unicodes=${unicodes}`,
]);
await io.writeAsset({
pathname: subset.asset.replace("[ext]", "woff2"),
buffer: await fs.readFile(woff),
});
}));
}, font)
));
}
export async function fetchFont(name: string, sources: string[]) {
const errs = [];
for (const source of sources) {
const cacheName = path.join("", name + path.extname(source));
if (fs.existsSync(cacheName)) return cacheName;
if (source.startsWith("https://")) {
const response = await fetch(source);
if (response.ok) {
await fs.writeMkdir(
cacheName,
Buffer.from(await response.arrayBuffer()),
);
} else {
errs.push(
new Error(
`Fetching from ${source} failed: ${response.status} ${response.statusText}`,
),
);
continue;
}
}
if (path.isAbsolute(source)) {
if (fs.existsSync(source)) return source;
errs.push(new Error(`Font not available at absolute path ${source}`));
} else if (!source.includes("/") && !source.includes("\\")) {
const home = os.homedir();
const bases = process.platform === "win32"
? [
"\\Windows\\Fonts",
path.win32.join(home, "AppData\\Local\\Microsoft\\Windows\\Fonts"),
]
: process.platform === "darwin"
? [
"/Library/Fonts",
path.posix.join(home, "Library/Fonts"),
]
: [
"/usr/share/fonts",
"/usr/local/share/fonts",
path.posix.join(home, ".local/share/fonts"),
];
for (const base of bases) {
const found = fs.readDirRecOptionalSync(base)
.find((file) => path.basename(file) === source);
if (found) {
return path.join(base, found);
}
}
errs.push(
new Error(
`Font file ${source} not found in the following directories: ${
bases.join(", ")
}`,
),
);
}
}
throw new AggregateError(errs, `Failed to fetch the ${name} font`);
}
function fontRangesToString(ranges: sg.FontRange[]): string {
const segments: Array<{ start: number; end: number }> = [];
ranges.forEach((range) => {
if (typeof range === "string") {
for (let i = 0; i < range.length; i++) {
const cp = range.codePointAt(i) || 0;
segments.push({ start: cp, end: cp });
}
} else if (typeof range === "number") {
segments.push({ start: range, end: range });
} else {
segments.push({ start: range.start, end: range.end });
}
});
segments.sort((a, b) => a.start - b.start);
const merged: Array<{ start: number; end: number }> = [];
for (const seg of segments) {
const last = merged[merged.length - 1];
if (last && seg.start <= last.end + 1) {
last.end = Math.max(last.end, seg.end);
} else {
merged.push({ ...seg });
}
}
return merged.map(({ start, end }) =>
start === end
? `U+${start.toString(16).toUpperCase().padStart(4, "0")}`
: `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${
end.toString(16).toUpperCase().padStart(4, "0")
}`
).join(",");
}
import * as fs from "#sitegen/fs";
import * as os from "node:os";
import * as path from "node:path";
import * as sg from "#sitegen";
import * as subprocess from "#sitegen/subprocess";
import * as incr from "./incremental.ts";
import * as crypto from "node:crypto";
import * as async from "#sitegen/async";

28
framework/generate.ts
View file

@ -22,6 +22,9 @@ export async function generate() {
// TODO: loadMarkoCache
// -- start font work --
const builtFonts = fonts.buildFonts(siteConfig.fonts);
// -- perform build-time rendering --
const builtPages = pages.map((item) => incr.work(preparePage, item));
const builtViews = views.map((item) => incr.work(prepareView, item));
@ -70,7 +73,12 @@ export async function generate() {
assembleAndWritePage(page, styleMap, scriptMap)
);
await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);
await Promise.all([
builtBackend,
builtStaticFiles,
...pAssemblePages,
builtFonts,
]);
}
export async function readManifest(io: Io) {
@ -80,6 +88,7 @@ export async function readManifest(io: Io) {
root: toRel(section.root),
})),
backends: cfg.backends.map(toRel),
fonts: cfg.fonts,
};
}
@ -324,14 +333,17 @@ export async function assembleAndWritePage(
export type PageOrView = PreparedPage | PreparedView;
import * as path from "node:path";
import * as fs from "#sitegen/fs";
import * as meta from "#sitegen/meta";
import * as render from "#engine/render";
import * as sg from "#sitegen";
import * as incr from "./incremental.ts";
import { Io } from "./incremental.ts";
import type { FileItem } from "#sitegen";
import * as bundle from "./bundle.ts";
import * as css from "./css.ts";
import * as render from "#engine/render";
import * as fonts from "./font.ts";
import * as hot from "./hot.ts";
import * as fs from "#sitegen/fs";
import type { FileItem } from "#sitegen";
import * as path from "node:path";
import * as meta from "#sitegen/meta";
import * as incr from "./incremental.ts";
import { Io } from "./incremental.ts";

75
framework/lib/assets.ts
View file

@ -2,6 +2,7 @@
// This module implements decoding and serving of the asset blobs,
// but also implements patching of dynamic assets. The `Manifest`
// is generated by `incremental.ts`
const debug = console.scoped("assets");
const root = import.meta.dirname;
let current: Loaded | null = null;
@ -36,6 +37,7 @@ interface Loaded {
export interface DynamicEntry extends AssetBase {
buffer: Buffer;
}
type CacheMode = "auto" | "long-term" | "temporary" | "no-cache";
export async function reload() {
const map = await fs.readJson<Manifest>(path.join(root, "asset.json"));
@ -59,7 +61,7 @@ export async function reload() {
export async function middleware(c: Context, next: Next) {
if (!current) current = await reload();
const asset = current.map[c.req.path];
if (asset) return assetInner(c, asset, 200);
if (asset) return assetInner(c, c.req.path, asset, 200);
return next();
}
@ -68,16 +70,27 @@ export async function notFound(c: Context) {
let pathname = c.req.path;
do {
const asset = current.map[pathname + "/404"];
if (asset) return assetInner(c, asset, 404);
if (asset) return assetInner(c, pathname + "/404", asset, 404);
pathname = pathname.slice(0, pathname.lastIndexOf("/"));
} while (pathname);
const asset = current.map["/404"];
if (asset) return assetInner(c, asset, 404);
if (asset) return assetInner(c, "/404", asset, 404);
return c.text("the 'Not Found' page was not found", 404);
}
export async function serveAsset(c: Context, id: Key, status: StatusCode) {
return assetInner(c, (current ?? (await reload())).map[id], status);
export async function serveAsset(
c: Context,
id: Key,
status: StatusCode = 200,
cache: CacheMode = 'auto',
) {
return assetInner(
c,
id,
(current ?? (await reload())).map[id],
status,
cache,
);
}
/** @deprecated */
@ -89,21 +102,43 @@ export function etagMatches(etag: string, ifNoneMatch: string) {
return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1;
}
function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) {
function assetInner(
c: Context,
key: string,
asset: Manifest[Key],
status: StatusCode,
cache: CacheMode = "auto",
) {
ASSERT(current);
if (asset.type === 0) {
return respondWithBufferAndViews(c, current.static, asset, status);
return respondWithBufferAndViews(
c,
key,
current.static,
asset,
status,
cache,
);
} else {
const entry = UNWRAP(current.dynamic.get(asset.id));
return respondWithBufferAndViews(c, entry.buffer, entry, status);
return respondWithBufferAndViews(
c,
key,
entry.buffer,
entry,
status,
cache,
);
}
}
function respondWithBufferAndViews(
c: Context,
key: string,
buffer: Buffer,
asset: AssetBase,
status: StatusCode,
cache: CacheMode,
) {
const ifNoneMatch = c.req.header("If-None-Match");
if (ifNoneMatch) {
@ -136,6 +171,29 @@ function respondWithBufferAndViews(
} else {
body = buffer.subarray(...asset.raw);
}
debug(
`${key} encoding=${headers["Content-Encoding"] ?? "raw"} status=${status}`,
);
if (!Object.keys(headers).some((x) => x.toLowerCase() === "cache-control")) {
if (cache === "auto") {
if (status < 200 || status >= 300) {
cache = 'no-cache';
} else if (headers['content-type']?.includes('text/html')) {
cache = 'temporary';
} else {
cache = 'long-term';
}
}
if (cache === "no-cache") {
headers["Cache-Control"] = "no-store";
} else if (cache === "long-term") {
headers["Cache-Control"] =
"public, max-age=7200, stale-while-revalidate=7200, immutable";
} else if (cache === "temporary") {
headers["Cache-Control"] =
"public, max-age=7200, stale-while-revalidate=7200";
}
}
return (c.res = new Response(body, { headers, status }));
}
@ -232,3 +290,4 @@ import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts";
import * as crypto from "node:crypto";
import * as zlib from "node:zlib";
import * as util from "node:util";
import * as console from "@paperclover/console";

47
framework/lib/error.ts Normal file
View file

@ -0,0 +1,47 @@
declare global {
interface Error {
/* Extra fields on errors are extremely helpful for debugging. */
[metadata: string]: unknown;
}
}
/** Retrieve an error message from any value */
export function message(error: unknown): string {
const message = (error as { message: unknown })?.message ?? error;
try {
return typeof message === "string" ? message : JSON.stringify(message);
} catch {}
try {
return String(message);
} catch {}
return `Could not stringify error message ${typeof message}`;
}
/** Retrieve an error-like object from any value. Useful to check fields. */
export function obj(error: unknown): { message: string } & Partial<Error> {
if (error instanceof Error) return error;
return {
message: message(error),
...(typeof error === "object" && error),
};
}
/* Node.js error codes are strings */
export function code(error: unknown): NodeErrorCode | null {
const code = (error as { code: unknown })?.code;
return typeof code === "string" ? code : null;
}
/** Attach extra fields and throw */
export function rethrowWithMetadata(
err: unknown,
meta: Record<string, unknown>,
): never {
const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/
Object.assign(error, meta);
throw error;
}
export type NodeErrorCode =
| keyof typeof import("node:os").constants.errno
| (string & {});

7
framework/lib/meta.ts
View file

@ -4,6 +4,9 @@ export interface Meta {
openGraph?: OpenGraph;
alternates?: Alternates;
}
export interface Template extends Omit<Meta, "title"> {
titleTemplate?: (title: string) => string,
}
export interface OpenGraph {
title?: string;
description?: string | undefined;
@ -19,6 +22,6 @@ export interface AlternateType {
title: string;
}
export function renderMeta({ title }: Meta): string {
return `<title>${esc(title)}</title>`;
return `<title>${esc(title)}</title><link rel="icon" type="image/x-icon" href="/favicon.ico">`;
}
import { escapeHtml as esc } from "#engine/render";
import { escapeHtmlContent as esc } from "#engine/render";

199
framework/lib/mime.txt
View file

@ -1,99 +1,100 @@
# media types
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types
.aac audio/x-aac
.aif audio/x-aiff
.aifc audio/x-aiff
.aiff audio/x-aiff
.asm text/x-asm
.avi video/x-msvideo
.bat application/x-msdownload
.c text/x-c
.chat text/x-clover-chatlog
.class application/java-vm
.cmd application/x-msdownload
.com application/x-msdownload
.conf text/plain
.cpp text/x-c
.css text/css
.csv text/csv
.cxx text/x-c
.def text/plain
.diff text/plain
.dll application/x-msdownload
.dmg application/octet-stream
.doc application/msword
.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document
.epub application/epub+zip
.exe application/x-msdownload
.flv video/x-flv
.fbx application/fbx
.gz application/x-gzip
.h text/x-c
.h264 video/h264
.hh text/x-c
.htm text/html;charset=utf-8
.html text/html;charset=utf-8
.ico image/x-icon
.ics text/calendar
.in text/plain
.jar application/java-archive
.java text/x-java-source
.jpeg image/jpeg
.jpg image/jpeg
.jpgv video/jpeg
.jxl image/jxl
.js application/javascript
.json application/json
.latex application/x-latex
.list text/plain
.log text/plain
.m4a audio/mp4
.man text/troff
.mid audio/midi
.midi audio/midi
.mov video/quicktime
.mp3 audio/mpeg
.mp4 video/mp4
.msh model/mesh
.msi application/x-msdownload
.obj application/octet-stream
.ogg audio/ogg
.otf application/x-font-otf
.pdf application/pdf
.png image/png
.ppt application/vnd.ms-powerpoint
.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation
.psd image/vnd.adobe.photoshop
.py text/x-python
.rar application/x-rar-compressed
.rss application/rss+xml
.rtf application/rtf
.rtx text/richtext
.s text/x-asm
.pem application/x-pem-file"
.ser application/java-serialized-object
.sh application/x-sh
.sig application/pgp-signature
.silo model/mesh
.svg image/svg+xml
.t text/troff
.tar application/x-tar
.text text/plain
.tgz application/x-gzip
.tif image/tiff
.tiff image/tiff
.torrent application/x-bittorrent
.ttc application/x-font-ttf
.ttf application/x-font-ttf
.txt text/plain
.urls text/uri-list
.v text/x-v
.wav audio/x-wav
.wmv video/x-ms-wmv
.xls application/vnd.ms-excel
.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
.xml application/xml
.xps application/vnd.ms-xpsdocument
# special cased based on file name
rss.xml application/rss+xml
# media types
# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types
.aac audio/x-aac
.aif audio/x-aiff
.aifc audio/x-aiff
.aiff audio/x-aiff
.asm text/x-asm
.avi video/x-msvideo
.bat application/x-msdownload
.c text/x-c
.chat text/x-clover-chatlog
.class application/java-vm
.cmd application/x-msdownload
.com application/x-msdownload
.conf text/plain
.cpp text/x-c
.css text/css
.csv text/csv
.cxx text/x-c
.def text/plain
.diff text/plain
.dll application/x-msdownload
.dmg application/octet-stream
.doc application/msword
.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document
.epub application/epub+zip
.exe application/x-msdownload
.flv video/x-flv
.fbx application/fbx
.gz application/x-gzip
.h text/x-c
.h264 video/h264
.hh text/x-c
.htm text/html;charset=utf-8
.html text/html;charset=utf-8
.ico image/x-icon
.ics text/calendar
.in text/plain
.jar application/java-archive
.java text/x-java-source
.jpeg image/jpeg
.jpg image/jpeg
.jpgv video/jpeg
.jxl image/jxl
.js application/javascript
.json application/json
.latex application/x-latex
.list text/plain
.log text/plain
.m4a audio/mp4
.man text/troff
.mid audio/midi
.midi audio/midi
.mov video/quicktime
.mp3 audio/mpeg
.mp4 video/mp4
.msh model/mesh
.msi application/x-msdownload
.obj application/octet-stream
.ogg audio/ogg
.otf application/x-font-otf
.pdf application/pdf
.png image/png
.ppt application/vnd.ms-powerpoint
.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation
.psd image/vnd.adobe.photoshop
.py text/x-python
.rar application/x-rar-compressed
.rss application/rss+xml
.rtf application/rtf
.rtx text/richtext
.s text/x-asm
.pem application/x-pem-file"
.ser application/java-serialized-object
.sh application/x-sh
.sig application/pgp-signature
.silo model/mesh
.svg image/svg+xml
.t text/troff
.tar application/x-tar
.text text/plain
.tgz application/x-gzip
.tif image/tiff
.tiff image/tiff
.torrent application/x-bittorrent
.ttc application/x-font-ttf
.ttf application/x-font-ttf
.txt text/plain
.urls text/uri-list
.v text/x-v
.wav audio/x-wav
.wmv video/x-ms-wmv
.woff2 font/woff2
.xls application/vnd.ms-excel
.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet
.xml application/xml
.xps application/vnd.ms-xpsdocument
# special cased based on file name
rss.xml application/rss+xml

18
framework/lib/sitegen.ts
View file

@ -31,6 +31,24 @@ export interface FileItem {
export interface Section {
root: string;
}
export interface Font {
name: string;
/**
* Specify either font name, file path, or URL to fetch it.
* Private fonts do not have a URL and will fail to build if missing.
*/
sources: string[];
subsets: Array<{
vars?: Record<string, FontVariableAxis>;
layoutFeatures?: string[];
unicodes: FontRange | FontRange[];
/** Include [ext] to autofill 'woff2' */
asset: string;
}>;
}
export type FontVars = Record<string, FontVariableAxis>;
export type FontVariableAxis = number | [min: number, max: number];
export type FontRange = string | number | { start: number, end: number };
export const userData = render.userData<SitegenRender>(() => {
throw new Error("This function can only be used in a page (static or view)");

9
framework/lib/sqlite.ts
View file

@ -11,9 +11,9 @@ export function getDb(file: string) {
let db = map.get(file);
if (db) return db;
const fileWithExt = file.includes(".") ? file : file + ".sqlite";
db = new WrappedDatabase(
path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt),
);
const dir = process.env.CLOVER_DB ?? ".clover";
fs.mkdirSync(dir);
db = new WrappedDatabase(path.join(dir, fileWithExt));
map.set(file, db);
return db;
}
@ -21,7 +21,7 @@ export function getDb(file: string) {
export class WrappedDatabase {
file: string;
node: DatabaseSync;
stmts: Stmt[];
stmts: Stmt[] = [];
stmtTableMigrate: WeakRef<StatementSync> | null = null;
constructor(file: string) {
@ -153,3 +153,4 @@ export class Stmt<Args extends unknown[] = unknown[], Row = unknown> {
import { DatabaseSync, StatementSync } from "node:sqlite";
import * as path from "node:path";
import * as fs from "#sitegen/fs";

15
framework/lib/subprocess.ts Normal file
View file

@ -0,0 +1,15 @@
const execFileRaw = util.promisify(child_process.execFile);
export const exec: typeof execFileRaw = ((
...args: Parameters<typeof execFileRaw>
) =>
execFileRaw(...args).catch((e: any) => {
if (e?.message?.startsWith?.("Command failed")) {
if (e.code > 2 ** 31) e.code |= 0;
const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
}
throw e;
})) as any;
import * as util from 'node:util';
import * as child_process from 'node:child_process';

1
framework/watch.ts
View file

@ -25,6 +25,7 @@ function serve() {
"--development",
], {
stdio: "inherit",
execArgv: ['--no-warnings'],
});
subprocess.on("close", onSubprocessClose);
}

54
readme.md
View file

@ -11,23 +11,21 @@ that assist building websites. these tools power <https://paperclover.net>.
- Different languages can be used at the same time. Supports `async function`
components, `<Suspense />`, and custom extensions.
- **Incremental static site generator and build system.**
- Build entire production site at start, incremental updates when pages
change; Build system state survives coding sessions.
- The only difference in development and production mode is hidden source-maps
and stripped `console.debug` calls. The site you see locally is the same
site you see deployed.
- Build both development and production sites on startup start. Incremental
generator rebuilds changed pages; Build system state survives coding sessions.
- Multiple backend support.
- (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs
checks when the files change. For example, changing a component re-tests
only pages that use that component and re-lints only the changed file.
- **Integrated libraries for building complex, content heavy web sites.**
- Static asset serving with ETag and build-time compression.
- Dynamicly rendered pages with static client. (`#import "#sitegen/view"`)
- Dynamicly rendered pages with static client. (`import "#sitegen/view"`)
- Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`)
- TODO: Meta and Open Graph generation. (`export const meta`)
- TODO: Font subsetting tools to reduce bytes downloaded by fonts.
- **Built on the battle-tested Node.js runtime.**
- Font subsetting to reduce page bandwidth.
- **Built on the stable and battle-tested Node.js runtime.**
None of these tools are complex or revolutionary. Rather, this project is the
none of these tools are complex or revolutionary. rather, this project is the
sum of many years of experience on managing content heavy websites, and an
example on how other over-complicate other frameworks.
@ -35,9 +33,11 @@ example on how other over-complicate other frameworks.
Included is `src`, which contains `paperclover.net`. Website highlights:
- TODO: flashy homepage.
- [Question/Answer board, custom markdown parser and components][q+a].
- [File viewer with fast ui/ux + optimized media streaming][file].
- [Personal, friends-only blog with password protection][friends].
- TODO: digital garden styled blog.
[q+a]: https://paperclover.net/q+a
[file]: https://paperclover.net/file
@ -51,15 +51,21 @@ minimum system requirements:
- random access memory.
- windows 7 or later, macos, or other operating system.
required software:
- node.js v24
- python v3
my development machine, for example, is Dell Inspiron 7348 with Core i7
npm install
# production generation
# build site using 'run.js' to enable runtime plugins
node run generate
node .clover/o/backend
# the built site runs in regular node.js
node --enable-source-maps .clover/o/backend
# "development" watch mode
# watch-rebuild mode
node run watch
for unix systems, the provided `flake.nix` can be used with `nix develop` to
@ -67,7 +73,7 @@ open a shell with all needed system dependencies.
## Deployment
There are two primary server components to be deployed: the web server and the
there are two primary server components to be deployed: the web server and the
sourth of truth server. The latter is a singleton that runs on Clover's NAS,
which holds the full contents of the file storage. The web server pulls data
from the source of truth and renders web pages, and can be duplicated to
@ -79,7 +85,8 @@ Deployment of the source of truth can be done with Docker Compose:
backend:
container_name: backend
build:
# this uses loopback to hit the self-hosted git server
# this uses loopback to hit the self-hosted git server,
# docker will cache the image to not re-fetch on reboot.
context: http://127.0.0.1:3000/clo/sitegen.git
dockerfile: src/source-of-truth.dockerfile
environment:
@ -96,10 +103,23 @@ Deployment of the source of truth can be done with Docker Compose:
- /mnt/storage1/clover/Documents/Config/paperclover:/data
- /mnt/storage1/clover/Published:/published
Due to caching, one may need to manually purge images via
`docker image rm ix-clover-backend -f` when an update is desired
Due to caching, images may need to be purged via `docker image rm {image} -f`
when an update is desired. Some docker GUIs support force pulls, some are buggy.
TODO: deployment instructions for a web node
The web server performs rendering. A `Dockerfile` for it is present in
`src/web.dockerfile` but it is currently unused. Deployments are done by
building the project locally, and then using `rsync` to copy files.
node run generate
rsync .clover/o "$REMOTE_USER:~/paperclover" --exclude=.clover --exclude=.env \
--delete-after --progress --human-readable
ssh "$REMOTE_USER" /bin/bash << EOF
set -e
cd ~/paperclover
npm ci
pm2 restart site
echo "-> https://paperclover.net"
EOF
## Contributions

35
run.js
View file

@ -3,7 +3,6 @@
import * as util from "node:util";
import * as zlib from "node:zlib";
import * as url from "node:url";
import * as module from "node:module";
import process from "node:process";
// Disable experimental warnings (Type Stripping, etc)
@ -28,8 +27,8 @@ try {
const brand = process.versions.bun
? `bun ${process.versions.bun}`
: process.versions.deno
? `deno ${process.versions.deno}`
: null;
? `deno ${process.versions.deno}`
: null;
const brandText = brand ? ` (${brand})` : "";
globalThis.console.error(
`sitegen depends on a node.js v24. your runtime is missing features.\n` +
@ -75,16 +74,26 @@ if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) {
else if (mod.default?.fetch) {
const protocol = "http";
const { serve } = hot.load("@hono/node-server");
serve({
fetch: mod.default.fetch,
}, ({ address, port }) => {
if (address === "::") address = "::1";
console.info(url.format({
protocol,
hostname: address,
port,
}));
});
serve(
{
fetch: mod.default.fetch,
},
({ address, port }) => {
if (address === "::") address = "::1";
console.info(
url.format({
protocol,
hostname: address,
port,
}),
);
},
);
} else {
console.warn(
hot.load("path").relative('.', found),
'does not export a "main" function',
);
}
} catch (e) {
console.error(util.inspect(e));

6
src/admin.ts
View file

@ -3,7 +3,11 @@ const cookieAge = 60 * 60 * 24 * 365; // 1 year
let lastKnownToken: string | null = null;
function compareToken(token: string) {
if (token === lastKnownToken) return true;
lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
try {
lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim();
} catch {
return false;
}
return token === lastKnownToken;
}

14
src/backend.ts
View file

@ -15,6 +15,20 @@ app.route("", require("./file-viewer/backend.tsx").app);
// Asset middleware has least precedence
app.use(assets.middleware);
// Asset Aliases
app.get(
"/apple-touch-icon-precomposed.png",
(c) => assets.serveAsset(c, "/apple-touch-icon.png"),
);
app.get(
"/apple-touch-icon-57x57.png",
(c) => assets.serveAsset(c, "/apple-touch-icon.png"),
);
app.get(
"/apple-touch-icon-57x57-precomposed.png",
(c) => assets.serveAsset(c, "/apple-touch-icon.png"),
);
// Handlers
app.notFound(assets.notFound);

18
src/file-viewer/bin/scan3.ts
View file

@ -10,7 +10,6 @@
// This is the third iteration of the scanner, hence its name "scan3";
// Remember that any software you want to be maintainable and high
// quality cannot be written with AI.
const root = path.resolve("/Volumes/clover/Published");
const workDir = path.resolve(".clover/derived");
const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY);
@ -404,18 +403,6 @@ interface Process {
undo?(mediaFile: MediaFile): Promise<void>;
}
const execFileRaw = util.promisify(child_process.execFile);
const execFile: typeof execFileRaw = ((
...args: Parameters<typeof execFileRaw>
) =>
execFileRaw(...args).catch((e: any) => {
if (e?.message?.startsWith?.("Command failed")) {
if (e.code > 2 ** 31) e.code |= 0;
const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`;
e.message = `${e.cmd.split(" ")[0]} failed with ${code}`;
}
throw e;
})) as any;
const ffprobeBin = testProgram("ffprobe", "--help");
const ffmpegBin = testProgram("ffmpeg", "--help");
@ -426,7 +413,7 @@ const procDuration: Process = {
enable: ffprobeBin !== null,
include: rules.extsDuration,
async run({ absPath, mediaFile }) {
const { stdout } = await execFile(ffprobeBin!, [
const { stdout } = await subprocess.exec(ffprobeBin!, [
"-v",
"error",
"-show_entries",
@ -793,11 +780,11 @@ import { Progress } from "@paperclover/console/Progress";
import { Spinner } from "@paperclover/console/Spinner";
import * as async from "#sitegen/async";
import * as fs from "#sitegen/fs";
import * as subprocess from "#sitegen/subprocess";
import * as path from "node:path";
import * as zlib from "node:zlib";
import * as child_process from "node:child_process";
import * as util from "node:util";
import * as crypto from "node:crypto";
import * as stream from "node:stream";
@ -814,3 +801,4 @@ import * as highlight from "@/file-viewer/highlight.ts";
import * as ffmpeg from "@/file-viewer/ffmpeg.ts";
import * as rsync from "@/file-viewer/rsync.ts";
import * as transcodeRules from "@/file-viewer/transcode-rules.ts";
import { rawFileRoot as root } from "../paths.ts";

8
src/file-viewer/cache.ts
View file

@ -12,9 +12,9 @@ import { escapeUri } from "./format.ts";
declare const Deno: any;
const sourceOfTruth = "https://nas.paperclover.net:43250";
const caCert = fs.readFileSync("src/file-viewer/cert.pem");
// const caCert = fs.readFileSync("src/file-viewer/cert.pem");
const diskCacheRoot = path.join(import.meta.dirname, "../.clover/filecache/");
const diskCacheRoot = path.join(import.meta.dirname, ".filecache/");
const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB
const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB
const loadInProgress = new Map<
@ -338,7 +338,7 @@ const agent: any = typeof Bun !== "undefined"
// Bun has two non-standard fetch extensions
decompress: false,
tls: {
ca: caCert,
// ca: caCert,
},
}
// TODO: https://github.com/denoland/deno/issues/12291
@ -351,7 +351,7 @@ const agent: any = typeof Bun !== "undefined"
// }
// Node.js supports node:http
: new Agent({
ca: caCert,
// ca: caCert,
});
function fetchFileNode(pathname: string): Promise<ReadableStream> {

12
src/file-viewer/paths.ts Normal file
View file

@ -0,0 +1,12 @@
export const nasRoot = process.platform === "win32"
? "\\\\zenith\\clover"
: process.platform === "darwin"
? "/Volumes/clover"
: "/media/clover";
export const rawFileRoot = process.env.CLOVER_FILE_RAW ??
path.join(nasRoot, "Published");
export const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
path.join(nasRoot, "Documents/Config/paperclover/derived");
import * as path from "node:path";

100
src/site.ts
View file

@ -1,11 +1,11 @@
// This file defines the different "Sections" of the website. The sections act
// as separate codebases, but hosted as one. This allows me to have
// sub-projects like the file viewer in 'file', or the question answer system
// in 'q+a'. Each section can define configuration, pages, backend routes, and
// contain other files.
const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths);
export const siteSections: Section[] = [
// Different sections of the website are split into their own folders. Acting as
// as separate codebases, they are hosted as one. This allows me to have
// sub-projects like the file viewer in 'file/', and the question answer system
// in 'q+a', but maintain clear boundaries. Each section can define
// configuration, pages, backend routes, and contain other files.
export const siteSections: sg.Section[] = [
{ root: join(".") },
{ root: join("q+a/") },
{ root: join("file-viewer/") },
@ -14,10 +14,96 @@ export const siteSections: Section[] = [
// { root: join("fiction/"), pageBase: "/fiction" },
];
// All backends are bundled. The backend named "backend" is run by "node run watch"
export const backends: string[] = [
join("backend.ts"),
join("source-of-truth.ts"),
];
// Font subsets reduce bandwidth and protect against proprietary font theft.
const fontRoot = path.join(nasRoot, 'Documents/Font');
const ascii = { start: 0x20, end: 0x7E };
const nonAscii: sg.FontRange[] = [
{ start: 0xC0, end: 0xFF },
{ start: 0x2190, end: 0x2193 },
{ start: 0xA0, end: 0xA8 },
{ start: 0xAA, end: 0xBF },
{ start: 0x2194, end: 0x2199 },
{ start: 0x100, end: 0x17F },
0xA9,
0x2018,
0x2019,
0x201C,
0x201D,
0x2022,
];
const recursiveVars: sg.FontVars = {
WGHT: [400, 750],
SLNT: [-15, 0],
CASL: 0.25,
CRSV: 0.5,
MONO: 0,
};
const recursiveVarsMono = { ...recursiveVars, MONO: 1 };
const recursiveVarsQuestion: sg.FontVars = {
...recursiveVars,
MONO: [0, 1],
WGHT: [400, 1000],
};
const layoutFeatures = ["numr", "dnom", "frac"];
export const fonts: sg.Font[] = [
{
name: "Recursive",
sources: [
"Recursive_VF_1.085.ttf",
path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
"https://paperclover.net/file/_unlisted/Recursive_VF_1.085.ttf",
],
subsets: [
{
asset: "/recultramin.[ext]",
layoutFeatures,
vars: recursiveVars,
unicodes: ascii,
},
{
asset: "/recmono.[ext]",
vars: recursiveVarsMono,
unicodes: ascii,
},
{
asset: "/recqa.[ext]",
vars: recursiveVarsQuestion,
unicodes: ascii,
},
{
asset: "/recexotic.[ext]",
vars: recursiveVarsQuestion,
unicodes: nonAscii,
},
],
},
{
name: "AT Name Sans Display Hairline",
sources: [
"ATNameSansDisplay-Hairline.woff2",
path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"),
],
subsets: [
{
asset: "/cydn_header.[ext]",
layoutFeatures,
unicodes: "cotlyedon",
},
],
},
];
export async function main() {
await font.buildFonts(fonts);
}
import * as path from "node:path";
import type { Section } from "#sitegen";
import * as font from "../framework/font.ts";
import type * as sg from "#sitegen";
import { nasRoot } from "./file-viewer/paths.ts";

12
src/source-of-truth.ts
View file

@ -22,17 +22,6 @@ export default app;
const token = UNWRAP(process.env.CLOVER_SOT_KEY);
const nasRoot = process.platform === "win32"
? "\\\\zenith\\clover"
: process.platform === "darwin"
? "/Volumes/clover"
: "/media/clover";
const rawFileRoot = process.env.CLOVER_FILE_RAW ??
path.join(nasRoot, "Published");
const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
path.join(nasRoot, "Documents/Config/paperclover/derived");
if (!fs.existsSync(rawFileRoot)) {
throw new Error(`${rawFileRoot} does not exist`);
}
@ -125,3 +114,4 @@ import * as fsCallbacks from "node:fs";
import * as util from "node:util";
import * as stream from "node:stream";
import * as mime from "#sitegen/mime";
import { derivedFileRoot, rawFileRoot } from "./file-viewer/paths.ts";

BIN
src/static/apple-touch-icon.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 72 KiB

BIN
src/static/favicon.ico Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 66 KiB

19
src/web.dockerfile Normal file
View file

@ -0,0 +1,19 @@
from node:24 as builder
run apt install -y git
workdir /tmp/builder
copy package*.json ./
run npm ci
copy . ./
run node run generate
run npm prune --production && cp -r .clover/o /app && cp -r node_modules /app/
from node:24 as runtime
workdir /app
copy --from=builder /app/ ./
env PORT=80
cmd ["node", "--enable-source-maps", "/app/backend.js"]