diff --git a/flake.nix b/flake.nix index e2f36c1..4b73e17 100644 --- a/flake.nix +++ b/flake.nix @@ -7,8 +7,12 @@ with inputs.nixpkgs.legacyPackages.${system}; { devShells.default = pkgs.mkShell { buildInputs = [ + # clover sitegen v3 pkgs.nodejs_24 # runtime pkgs.deno # formatter + pkgs.python3 # for font subsetting + + # paperclover.net (pkgs.ffmpeg.override { withOpus = true; withSvtav1 = true; diff --git a/framework/backend/entry-node.ts b/framework/backend/entry-node.ts index 4f4a249..3529272 100644 --- a/framework/backend/entry-node.ts +++ b/framework/backend/entry-node.ts @@ -9,7 +9,7 @@ const server = serve({ fetch: app.default.fetch, port: Number(process.env.PORT ?? 3000), }, ({ address, port }) => { - if (address === "::") address = "::1"; + if (address === "::") address = "localhost"; console.info(url.format({ protocol, hostname: address, diff --git a/framework/bundle.ts b/framework/bundle.ts index 3a4f3a1..01c9e02 100644 --- a/framework/bundle.ts +++ b/framework/bundle.ts @@ -94,7 +94,6 @@ export async function bundleClientJavaScript( const chunk = route.startsWith("/js/c."); if (!chunk) { const key = hot.getScriptId(toAbs(UNWRAP(entryPoint))); - console.log(route, key); route = "/js/" + key.replace(/\.client\.tsx?/, ".js"); scripts[key] = text; } diff --git a/framework/font.ts b/framework/font.ts new file mode 100644 index 0000000..7645a05 --- /dev/null +++ b/framework/font.ts @@ -0,0 +1,192 @@ +export async function buildFonts(fonts: sg.Font[]) { + if (fonts.length === 0) return; + + const temp = path.resolve(".clover/font"); + const venv = path.join(temp, "venv"); + const bin = path.join(venv, "bin"); + + if (!fs.existsSync(venv)) { + await subprocess.exec("python", ["-m", "venv", venv]); + } + if (!fs.existsSync(path.join(bin, "pyftsubset"))) { + await subprocess.exec(path.join(bin, "pip"), [ + "install", + "fonttools==4.38.0", + "brotli==1.0.7", + ]); + } + + const instances = new async.OnceMap(); + const subsets = new async.OnceMap(); + async function makeInstance(input: string, vars: sg.FontVars) { + const args = Object.entries(vars).map((x) => { + const lower = x[0].toLowerCase(); + const k = lower === "wght" || lower === "slnt" ? lower : x[0]; + const v = Array.isArray(x[1]) ? x[1].join(":") : x[1]; + return `${k}=${v}`; + }).sort(); + const hash = crypto + .createHash("sha256") + .update(`${input}${args.join(" ")}`) + .digest("hex") + .slice(0, 16) + .toLowerCase(); + const outfile = path.join(temp, `${hash}${path.extname(input)}`); + await instances.get(outfile, async () => { + await fs.rm(outfile, { force: true }); + await subprocess.exec(path.join(bin, "fonttools"), [ + "varLib.instancer", + input, + ...args, + `--output=${outfile}`, + ]); + ASSERT(fs.existsSync(outfile)); + }); + return outfile; + } + + await Promise.all(fonts.map((font) => + incr.work(async (io, font) => { + const baseFile = await fetchFont(font.name, font.sources); + await Promise.all(font.subsets.map(async (subset) => { + let file = baseFile; + if (subset.vars) { + file = await makeInstance(baseFile, subset.vars); + } + const unicodes = fontRangesToString( + Array.isArray(subset.unicodes) ? subset.unicodes : [subset.unicodes], + ); + const hash = crypto + .createHash("sha256") + .update(`${file}${unicodes}`) + .digest("hex") + .slice(0, 16) + .toLowerCase(); + const woff = path.join(temp, hash + ".woff2"); + await subprocess.exec(path.join(bin, "pyftsubset"), [ + file, + "--flavor=woff2", + `--output-file=${woff}`, + ...subset.layoutFeatures + ? [`--layout-features=${subset.layoutFeatures.join(",")}`] + : [], + `--unicodes=${unicodes}`, + ]); + await io.writeAsset({ + pathname: subset.asset.replace("[ext]", "woff2"), + buffer: await fs.readFile(woff), + }); + })); + }, font) + )); +} + +export async function fetchFont(name: string, sources: string[]) { + const errs = []; + + for (const source of sources) { + const cacheName = path.join("", name + path.extname(source)); + if (fs.existsSync(cacheName)) return cacheName; + + if (source.startsWith("https://")) { + const response = await fetch(source); + if (response.ok) { + await fs.writeMkdir( + cacheName, + Buffer.from(await response.arrayBuffer()), + ); + } else { + errs.push( + new Error( + `Fetching from ${source} failed: ${response.status} ${response.statusText}`, + ), + ); + continue; + } + } + + if (path.isAbsolute(source)) { + if (fs.existsSync(source)) return source; + errs.push(new Error(`Font not available at absolute path ${source}`)); + } else if (!source.includes("/") && !source.includes("\\")) { + const home = os.homedir(); + const bases = process.platform === "win32" + ? [ + "\\Windows\\Fonts", + path.win32.join(home, "AppData\\Local\\Microsoft\\Windows\\Fonts"), + ] + : process.platform === "darwin" + ? [ + "/Library/Fonts", + path.posix.join(home, "Library/Fonts"), + ] + : [ + "/usr/share/fonts", + "/usr/local/share/fonts", + path.posix.join(home, ".local/share/fonts"), + ]; + for (const base of bases) { + const found = fs.readDirRecOptionalSync(base) + .find((file) => path.basename(file) === source); + if (found) { + return path.join(base, found); + } + } + errs.push( + new Error( + `Font file ${source} not found in the following directories: ${ + bases.join(", ") + }`, + ), + ); + } + } + + throw new AggregateError(errs, `Failed to fetch the ${name} font`); +} + +function fontRangesToString(ranges: sg.FontRange[]): string { + const segments: Array<{ start: number; end: number }> = []; + + ranges.forEach((range) => { + if (typeof range === "string") { + for (let i = 0; i < range.length; i++) { + const cp = range.codePointAt(i) || 0; + segments.push({ start: cp, end: cp }); + } + } else if (typeof range === "number") { + segments.push({ start: range, end: range }); + } else { + segments.push({ start: range.start, end: range.end }); + } + }); + + segments.sort((a, b) => a.start - b.start); + + const merged: Array<{ start: number; end: number }> = []; + for (const seg of segments) { + const last = merged[merged.length - 1]; + if (last && seg.start <= last.end + 1) { + last.end = Math.max(last.end, seg.end); + } else { + merged.push({ ...seg }); + } + } + + return merged.map(({ start, end }) => + start === end + ? `U+${start.toString(16).toUpperCase().padStart(4, "0")}` + : `U+${start.toString(16).toUpperCase().padStart(4, "0")}-${ + end.toString(16).toUpperCase().padStart(4, "0") + }` + ).join(","); +} + +import * as fs from "#sitegen/fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import * as sg from "#sitegen"; +import * as subprocess from "#sitegen/subprocess"; +import * as incr from "./incremental.ts"; +import * as crypto from "node:crypto"; +import * as async from "#sitegen/async"; diff --git a/framework/generate.ts b/framework/generate.ts index 31ca14c..90fb65a 100644 --- a/framework/generate.ts +++ b/framework/generate.ts @@ -22,6 +22,9 @@ export async function generate() { // TODO: loadMarkoCache + // -- start font work -- + const builtFonts = fonts.buildFonts(siteConfig.fonts); + // -- perform build-time rendering -- const builtPages = pages.map((item) => incr.work(preparePage, item)); const builtViews = views.map((item) => incr.work(prepareView, item)); @@ -70,7 +73,12 @@ export async function generate() { assembleAndWritePage(page, styleMap, scriptMap) ); - await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]); + await Promise.all([ + builtBackend, + builtStaticFiles, + ...pAssemblePages, + builtFonts, + ]); } export async function readManifest(io: Io) { @@ -80,6 +88,7 @@ export async function readManifest(io: Io) { root: toRel(section.root), })), backends: cfg.backends.map(toRel), + fonts: cfg.fonts, }; } @@ -324,14 +333,17 @@ export async function assembleAndWritePage( export type PageOrView = PreparedPage | PreparedView; +import * as path from "node:path"; + +import * as fs from "#sitegen/fs"; +import * as meta from "#sitegen/meta"; +import * as render from "#engine/render"; import * as sg from "#sitegen"; -import * as incr from "./incremental.ts"; -import { Io } from "./incremental.ts"; +import type { FileItem } from "#sitegen"; + import * as bundle from "./bundle.ts"; import * as css from "./css.ts"; -import * as render from "#engine/render"; +import * as fonts from "./font.ts"; import * as hot from "./hot.ts"; -import * as fs from "#sitegen/fs"; -import type { FileItem } from "#sitegen"; -import * as path from "node:path"; -import * as meta from "#sitegen/meta"; +import * as incr from "./incremental.ts"; +import { Io } from "./incremental.ts"; diff --git a/framework/lib/assets.ts b/framework/lib/assets.ts index ac77d49..fa5338d 100644 --- a/framework/lib/assets.ts +++ b/framework/lib/assets.ts @@ -2,6 +2,7 @@ // This module implements decoding and serving of the asset blobs, // but also implements patching of dynamic assets. The `Manifest` // is generated by `incremental.ts` +const debug = console.scoped("assets"); const root = import.meta.dirname; let current: Loaded | null = null; @@ -36,6 +37,7 @@ interface Loaded { export interface DynamicEntry extends AssetBase { buffer: Buffer; } +type CacheMode = "auto" | "long-term" | "temporary" | "no-cache"; export async function reload() { const map = await fs.readJson(path.join(root, "asset.json")); @@ -59,7 +61,7 @@ export async function reload() { export async function middleware(c: Context, next: Next) { if (!current) current = await reload(); const asset = current.map[c.req.path]; - if (asset) return assetInner(c, asset, 200); + if (asset) return assetInner(c, c.req.path, asset, 200); return next(); } @@ -68,16 +70,27 @@ export async function notFound(c: Context) { let pathname = c.req.path; do { const asset = current.map[pathname + "/404"]; - if (asset) return assetInner(c, asset, 404); + if (asset) return assetInner(c, pathname + "/404", asset, 404); pathname = pathname.slice(0, pathname.lastIndexOf("/")); } while (pathname); const asset = current.map["/404"]; - if (asset) return assetInner(c, asset, 404); + if (asset) return assetInner(c, "/404", asset, 404); return c.text("the 'Not Found' page was not found", 404); } -export async function serveAsset(c: Context, id: Key, status: StatusCode) { - return assetInner(c, (current ?? (await reload())).map[id], status); +export async function serveAsset( + c: Context, + id: Key, + status: StatusCode = 200, + cache: CacheMode = 'auto', +) { + return assetInner( + c, + id, + (current ?? (await reload())).map[id], + status, + cache, + ); } /** @deprecated */ @@ -89,21 +102,43 @@ export function etagMatches(etag: string, ifNoneMatch: string) { return ifNoneMatch === etag || ifNoneMatch.split(/,\s*/).indexOf(etag) > -1; } -function assetInner(c: Context, asset: Manifest[Key], status: StatusCode) { +function assetInner( + c: Context, + key: string, + asset: Manifest[Key], + status: StatusCode, + cache: CacheMode = "auto", +) { ASSERT(current); if (asset.type === 0) { - return respondWithBufferAndViews(c, current.static, asset, status); + return respondWithBufferAndViews( + c, + key, + current.static, + asset, + status, + cache, + ); } else { const entry = UNWRAP(current.dynamic.get(asset.id)); - return respondWithBufferAndViews(c, entry.buffer, entry, status); + return respondWithBufferAndViews( + c, + key, + entry.buffer, + entry, + status, + cache, + ); } } function respondWithBufferAndViews( c: Context, + key: string, buffer: Buffer, asset: AssetBase, status: StatusCode, + cache: CacheMode, ) { const ifNoneMatch = c.req.header("If-None-Match"); if (ifNoneMatch) { @@ -136,6 +171,29 @@ function respondWithBufferAndViews( } else { body = buffer.subarray(...asset.raw); } + debug( + `${key} encoding=${headers["Content-Encoding"] ?? "raw"} status=${status}`, + ); + if (!Object.keys(headers).some((x) => x.toLowerCase() === "cache-control")) { + if (cache === "auto") { + if (status < 200 || status >= 300) { + cache = 'no-cache'; + } else if (headers['content-type']?.includes('text/html')) { + cache = 'temporary'; + } else { + cache = 'long-term'; + } + } + if (cache === "no-cache") { + headers["Cache-Control"] = "no-store"; + } else if (cache === "long-term") { + headers["Cache-Control"] = + "public, max-age=7200, stale-while-revalidate=7200, immutable"; + } else if (cache === "temporary") { + headers["Cache-Control"] = + "public, max-age=7200, stale-while-revalidate=7200"; + } + } return (c.res = new Response(body, { headers, status })); } @@ -232,3 +290,4 @@ import type { AssetKey as Key } from "../../.clover/ts/asset.d.ts"; import * as crypto from "node:crypto"; import * as zlib from "node:zlib"; import * as util from "node:util"; +import * as console from "@paperclover/console"; diff --git a/framework/lib/error.ts b/framework/lib/error.ts new file mode 100644 index 0000000..8162491 --- /dev/null +++ b/framework/lib/error.ts @@ -0,0 +1,47 @@ +declare global { + interface Error { + /* Extra fields on errors are extremely helpful for debugging. */ + [metadata: string]: unknown; + } +} + +/** Retrieve an error message from any value */ +export function message(error: unknown): string { + const message = (error as { message: unknown })?.message ?? error; + try { + return typeof message === "string" ? message : JSON.stringify(message); + } catch {} + try { + return String(message); + } catch {} + return `Could not stringify error message ${typeof message}`; +} + +/** Retrieve an error-like object from any value. Useful to check fields. */ +export function obj(error: unknown): { message: string } & Partial { + if (error instanceof Error) return error; + return { + message: message(error), + ...(typeof error === "object" && error), + }; +} + +/* Node.js error codes are strings */ +export function code(error: unknown): NodeErrorCode | null { + const code = (error as { code: unknown })?.code; + return typeof code === "string" ? code : null; +} + +/** Attach extra fields and throw */ +export function rethrowWithMetadata( + err: unknown, + meta: Record, +): never { + const error = err && typeof err === "object" ? err : new Error(message(err)); // no stack trace :/ + Object.assign(error, meta); + throw error; +} + +export type NodeErrorCode = + | keyof typeof import("node:os").constants.errno + | (string & {}); diff --git a/framework/lib/meta.ts b/framework/lib/meta.ts index 511b2a9..0e2c272 100644 --- a/framework/lib/meta.ts +++ b/framework/lib/meta.ts @@ -4,6 +4,9 @@ export interface Meta { openGraph?: OpenGraph; alternates?: Alternates; } +export interface Template extends Omit { + titleTemplate?: (title: string) => string, +} export interface OpenGraph { title?: string; description?: string | undefined; @@ -19,6 +22,6 @@ export interface AlternateType { title: string; } export function renderMeta({ title }: Meta): string { - return `${esc(title)}`; + return `${esc(title)}`; } -import { escapeHtml as esc } from "#engine/render"; +import { escapeHtmlContent as esc } from "#engine/render"; diff --git a/framework/lib/mime.txt b/framework/lib/mime.txt index 7cf5993..ad6220b 100644 --- a/framework/lib/mime.txt +++ b/framework/lib/mime.txt @@ -1,99 +1,100 @@ -# media types -# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types -.aac audio/x-aac -.aif audio/x-aiff -.aifc audio/x-aiff -.aiff audio/x-aiff -.asm text/x-asm -.avi video/x-msvideo -.bat application/x-msdownload -.c text/x-c -.chat text/x-clover-chatlog -.class application/java-vm -.cmd application/x-msdownload -.com application/x-msdownload -.conf text/plain -.cpp text/x-c -.css text/css -.csv text/csv -.cxx text/x-c -.def text/plain -.diff text/plain -.dll application/x-msdownload -.dmg application/octet-stream -.doc application/msword -.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document -.epub application/epub+zip -.exe application/x-msdownload -.flv video/x-flv -.fbx application/fbx -.gz application/x-gzip -.h text/x-c -.h264 video/h264 -.hh text/x-c -.htm text/html;charset=utf-8 -.html text/html;charset=utf-8 -.ico image/x-icon -.ics text/calendar -.in text/plain -.jar application/java-archive -.java text/x-java-source -.jpeg image/jpeg -.jpg image/jpeg -.jpgv video/jpeg -.jxl image/jxl -.js application/javascript -.json application/json -.latex application/x-latex -.list text/plain -.log text/plain -.m4a audio/mp4 -.man text/troff -.mid audio/midi -.midi audio/midi -.mov video/quicktime -.mp3 audio/mpeg -.mp4 video/mp4 -.msh model/mesh -.msi application/x-msdownload -.obj application/octet-stream -.ogg audio/ogg -.otf application/x-font-otf -.pdf application/pdf -.png image/png -.ppt application/vnd.ms-powerpoint -.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation -.psd image/vnd.adobe.photoshop -.py text/x-python -.rar application/x-rar-compressed -.rss application/rss+xml -.rtf application/rtf -.rtx text/richtext -.s text/x-asm -.pem application/x-pem-file" -.ser application/java-serialized-object -.sh application/x-sh -.sig application/pgp-signature -.silo model/mesh -.svg image/svg+xml -.t text/troff -.tar application/x-tar -.text text/plain -.tgz application/x-gzip -.tif image/tiff -.tiff image/tiff -.torrent application/x-bittorrent -.ttc application/x-font-ttf -.ttf application/x-font-ttf -.txt text/plain -.urls text/uri-list -.v text/x-v -.wav audio/x-wav -.wmv video/x-ms-wmv -.xls application/vnd.ms-excel -.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet -.xml application/xml -.xps application/vnd.ms-xpsdocument - -# special cased based on file name -rss.xml application/rss+xml +# media types +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Guides/MIME_types +.aac audio/x-aac +.aif audio/x-aiff +.aifc audio/x-aiff +.aiff audio/x-aiff +.asm text/x-asm +.avi video/x-msvideo +.bat application/x-msdownload +.c text/x-c +.chat text/x-clover-chatlog +.class application/java-vm +.cmd application/x-msdownload +.com application/x-msdownload +.conf text/plain +.cpp text/x-c +.css text/css +.csv text/csv +.cxx text/x-c +.def text/plain +.diff text/plain +.dll application/x-msdownload +.dmg application/octet-stream +.doc application/msword +.docx application/vnd.openxmlformats-officedocument.wordprocessingml.document +.epub application/epub+zip +.exe application/x-msdownload +.flv video/x-flv +.fbx application/fbx +.gz application/x-gzip +.h text/x-c +.h264 video/h264 +.hh text/x-c +.htm text/html;charset=utf-8 +.html text/html;charset=utf-8 +.ico image/x-icon +.ics text/calendar +.in text/plain +.jar application/java-archive +.java text/x-java-source +.jpeg image/jpeg +.jpg image/jpeg +.jpgv video/jpeg +.jxl image/jxl +.js application/javascript +.json application/json +.latex application/x-latex +.list text/plain +.log text/plain +.m4a audio/mp4 +.man text/troff +.mid audio/midi +.midi audio/midi +.mov video/quicktime +.mp3 audio/mpeg +.mp4 video/mp4 +.msh model/mesh +.msi application/x-msdownload +.obj application/octet-stream +.ogg audio/ogg +.otf application/x-font-otf +.pdf application/pdf +.png image/png +.ppt application/vnd.ms-powerpoint +.pptx application/vnd.openxmlformats-officedocument.presentationml.presentation +.psd image/vnd.adobe.photoshop +.py text/x-python +.rar application/x-rar-compressed +.rss application/rss+xml +.rtf application/rtf +.rtx text/richtext +.s text/x-asm +.pem application/x-pem-file" +.ser application/java-serialized-object +.sh application/x-sh +.sig application/pgp-signature +.silo model/mesh +.svg image/svg+xml +.t text/troff +.tar application/x-tar +.text text/plain +.tgz application/x-gzip +.tif image/tiff +.tiff image/tiff +.torrent application/x-bittorrent +.ttc application/x-font-ttf +.ttf application/x-font-ttf +.txt text/plain +.urls text/uri-list +.v text/x-v +.wav audio/x-wav +.wmv video/x-ms-wmv +.woff2 font/woff2 +.xls application/vnd.ms-excel +.xlsx application/vnd.openxmlformats-officedocument.spreadsheetml.sheet +.xml application/xml +.xps application/vnd.ms-xpsdocument + +# special cased based on file name +rss.xml application/rss+xml diff --git a/framework/lib/sitegen.ts b/framework/lib/sitegen.ts index af99dca..e51e5fc 100644 --- a/framework/lib/sitegen.ts +++ b/framework/lib/sitegen.ts @@ -31,6 +31,24 @@ export interface FileItem { export interface Section { root: string; } +export interface Font { + name: string; + /** + * Specify either font name, file path, or URL to fetch it. + * Private fonts do not have a URL and will fail to build if missing. + */ + sources: string[]; + subsets: Array<{ + vars?: Record; + layoutFeatures?: string[]; + unicodes: FontRange | FontRange[]; + /** Include [ext] to autofill 'woff2' */ + asset: string; + }>; +} +export type FontVars = Record; +export type FontVariableAxis = number | [min: number, max: number]; +export type FontRange = string | number | { start: number, end: number }; export const userData = render.userData(() => { throw new Error("This function can only be used in a page (static or view)"); diff --git a/framework/lib/sqlite.ts b/framework/lib/sqlite.ts index 477ab9e..9b49b2f 100644 --- a/framework/lib/sqlite.ts +++ b/framework/lib/sqlite.ts @@ -11,9 +11,9 @@ export function getDb(file: string) { let db = map.get(file); if (db) return db; const fileWithExt = file.includes(".") ? file : file + ".sqlite"; - db = new WrappedDatabase( - path.join(process.env.CLOVER_DB ?? ".clover", fileWithExt), - ); + const dir = process.env.CLOVER_DB ?? ".clover"; + fs.mkdirSync(dir); + db = new WrappedDatabase(path.join(dir, fileWithExt)); map.set(file, db); return db; } @@ -21,7 +21,7 @@ export function getDb(file: string) { export class WrappedDatabase { file: string; node: DatabaseSync; - stmts: Stmt[]; + stmts: Stmt[] = []; stmtTableMigrate: WeakRef | null = null; constructor(file: string) { @@ -153,3 +153,4 @@ export class Stmt { import { DatabaseSync, StatementSync } from "node:sqlite"; import * as path from "node:path"; +import * as fs from "#sitegen/fs"; diff --git a/framework/lib/subprocess.ts b/framework/lib/subprocess.ts new file mode 100644 index 0000000..2d183de --- /dev/null +++ b/framework/lib/subprocess.ts @@ -0,0 +1,15 @@ +const execFileRaw = util.promisify(child_process.execFile); +export const exec: typeof execFileRaw = (( + ...args: Parameters +) => + execFileRaw(...args).catch((e: any) => { + if (e?.message?.startsWith?.("Command failed")) { + if (e.code > 2 ** 31) e.code |= 0; + const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; + e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; + } + throw e; + })) as any; + +import * as util from 'node:util'; +import * as child_process from 'node:child_process'; diff --git a/framework/watch.ts b/framework/watch.ts index 9fe2157..453dbce 100644 --- a/framework/watch.ts +++ b/framework/watch.ts @@ -25,6 +25,7 @@ function serve() { "--development", ], { stdio: "inherit", + execArgv: ['--no-warnings'], }); subprocess.on("close", onSubprocessClose); } diff --git a/readme.md b/readme.md index cecf111..8301190 100644 --- a/readme.md +++ b/readme.md @@ -11,23 +11,21 @@ that assist building websites. these tools power . - Different languages can be used at the same time. Supports `async function` components, ``, and custom extensions. - **Incremental static site generator and build system.** - - Build entire production site at start, incremental updates when pages - change; Build system state survives coding sessions. - - The only difference in development and production mode is hidden source-maps - and stripped `console.debug` calls. The site you see locally is the same - site you see deployed. + - Build both development and production sites on startup start. Incremental + generator rebuilds changed pages; Build system state survives coding sessions. + - Multiple backend support. - (TODO) Tests, Lints, and Type-checking is run alongside, and only re-runs checks when the files change. For example, changing a component re-tests only pages that use that component and re-lints only the changed file. - **Integrated libraries for building complex, content heavy web sites.** - Static asset serving with ETag and build-time compression. - - Dynamicly rendered pages with static client. (`#import "#sitegen/view"`) + - Dynamicly rendered pages with static client. (`import "#sitegen/view"`) - Databases with a typed SQLite wrapper. (`import "#sitegen/sqlite"`) - TODO: Meta and Open Graph generation. (`export const meta`) - - TODO: Font subsetting tools to reduce bytes downloaded by fonts. -- **Built on the battle-tested Node.js runtime.** + - Font subsetting to reduce page bandwidth. +- **Built on the stable and battle-tested Node.js runtime.** -None of these tools are complex or revolutionary. Rather, this project is the +none of these tools are complex or revolutionary. rather, this project is the sum of many years of experience on managing content heavy websites, and an example on how other over-complicate other frameworks. @@ -35,9 +33,11 @@ example on how other over-complicate other frameworks. Included is `src`, which contains `paperclover.net`. Website highlights: +- TODO: flashy homepage. - [Question/Answer board, custom markdown parser and components][q+a]. - [File viewer with fast ui/ux + optimized media streaming][file]. - [Personal, friends-only blog with password protection][friends]. +- TODO: digital garden styled blog. [q+a]: https://paperclover.net/q+a [file]: https://paperclover.net/file @@ -51,15 +51,21 @@ minimum system requirements: - random access memory. - windows 7 or later, macos, or other operating system. +required software: + +- node.js v24 +- python v3 + my development machine, for example, is Dell Inspiron 7348 with Core i7 npm install - # production generation + # build site using 'run.js' to enable runtime plugins node run generate - node .clover/o/backend + # the built site runs in regular node.js + node --enable-source-maps .clover/o/backend - # "development" watch mode + # watch-rebuild mode node run watch for unix systems, the provided `flake.nix` can be used with `nix develop` to @@ -67,7 +73,7 @@ open a shell with all needed system dependencies. ## Deployment -There are two primary server components to be deployed: the web server and the +there are two primary server components to be deployed: the web server and the sourth of truth server. The latter is a singleton that runs on Clover's NAS, which holds the full contents of the file storage. The web server pulls data from the source of truth and renders web pages, and can be duplicated to @@ -79,7 +85,8 @@ Deployment of the source of truth can be done with Docker Compose: backend: container_name: backend build: - # this uses loopback to hit the self-hosted git server + # this uses loopback to hit the self-hosted git server, + # docker will cache the image to not re-fetch on reboot. context: http://127.0.0.1:3000/clo/sitegen.git dockerfile: src/source-of-truth.dockerfile environment: @@ -96,10 +103,23 @@ Deployment of the source of truth can be done with Docker Compose: - /mnt/storage1/clover/Documents/Config/paperclover:/data - /mnt/storage1/clover/Published:/published -Due to caching, one may need to manually purge images via -`docker image rm ix-clover-backend -f` when an update is desired +Due to caching, images may need to be purged via `docker image rm {image} -f` +when an update is desired. Some docker GUIs support force pulls, some are buggy. -TODO: deployment instructions for a web node +The web server performs rendering. A `Dockerfile` for it is present in +`src/web.dockerfile` but it is currently unused. Deployments are done by +building the project locally, and then using `rsync` to copy files. + + node run generate + rsync .clover/o "$REMOTE_USER:~/paperclover" --exclude=.clover --exclude=.env \ + --delete-after --progress --human-readable + ssh "$REMOTE_USER" /bin/bash << EOF + set -e + cd ~/paperclover + npm ci + pm2 restart site + echo "-> https://paperclover.net" + EOF ## Contributions diff --git a/run.js b/run.js index ea0c1dd..8e46ce1 100644 --- a/run.js +++ b/run.js @@ -3,7 +3,6 @@ import * as util from "node:util"; import * as zlib from "node:zlib"; import * as url from "node:url"; -import * as module from "node:module"; import process from "node:process"; // Disable experimental warnings (Type Stripping, etc) @@ -28,8 +27,8 @@ try { const brand = process.versions.bun ? `bun ${process.versions.bun}` : process.versions.deno - ? `deno ${process.versions.deno}` - : null; + ? `deno ${process.versions.deno}` + : null; const brandText = brand ? ` (${brand})` : ""; globalThis.console.error( `sitegen depends on a node.js v24. your runtime is missing features.\n` + @@ -75,16 +74,26 @@ if (process.argv[1].startsWith(import.meta.filename.slice(0, -".js".length))) { else if (mod.default?.fetch) { const protocol = "http"; const { serve } = hot.load("@hono/node-server"); - serve({ - fetch: mod.default.fetch, - }, ({ address, port }) => { - if (address === "::") address = "::1"; - console.info(url.format({ - protocol, - hostname: address, - port, - })); - }); + serve( + { + fetch: mod.default.fetch, + }, + ({ address, port }) => { + if (address === "::") address = "::1"; + console.info( + url.format({ + protocol, + hostname: address, + port, + }), + ); + }, + ); + } else { + console.warn( + hot.load("path").relative('.', found), + 'does not export a "main" function', + ); } } catch (e) { console.error(util.inspect(e)); diff --git a/src/admin.ts b/src/admin.ts index dd90fda..f9ee787 100644 --- a/src/admin.ts +++ b/src/admin.ts @@ -3,7 +3,11 @@ const cookieAge = 60 * 60 * 24 * 365; // 1 year let lastKnownToken: string | null = null; function compareToken(token: string) { if (token === lastKnownToken) return true; - lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); + try { + lastKnownToken = fs.readFileSync(".clover/admin-token.txt", "utf8").trim(); + } catch { + return false; + } return token === lastKnownToken; } diff --git a/src/backend.ts b/src/backend.ts index 48b5a11..74236a9 100644 --- a/src/backend.ts +++ b/src/backend.ts @@ -15,6 +15,20 @@ app.route("", require("./file-viewer/backend.tsx").app); // Asset middleware has least precedence app.use(assets.middleware); +// Asset Aliases +app.get( + "/apple-touch-icon-precomposed.png", + (c) => assets.serveAsset(c, "/apple-touch-icon.png"), +); +app.get( + "/apple-touch-icon-57x57.png", + (c) => assets.serveAsset(c, "/apple-touch-icon.png"), +); +app.get( + "/apple-touch-icon-57x57-precomposed.png", + (c) => assets.serveAsset(c, "/apple-touch-icon.png"), +); + // Handlers app.notFound(assets.notFound); diff --git a/src/file-viewer/bin/scan3.ts b/src/file-viewer/bin/scan3.ts index 261100e..be408d0 100644 --- a/src/file-viewer/bin/scan3.ts +++ b/src/file-viewer/bin/scan3.ts @@ -10,7 +10,6 @@ // This is the third iteration of the scanner, hence its name "scan3"; // Remember that any software you want to be maintainable and high // quality cannot be written with AI. -const root = path.resolve("/Volumes/clover/Published"); const workDir = path.resolve(".clover/derived"); const sotToken = UNWRAP(process.env.CLOVER_SOT_KEY); @@ -404,18 +403,6 @@ interface Process { undo?(mediaFile: MediaFile): Promise; } -const execFileRaw = util.promisify(child_process.execFile); -const execFile: typeof execFileRaw = (( - ...args: Parameters -) => - execFileRaw(...args).catch((e: any) => { - if (e?.message?.startsWith?.("Command failed")) { - if (e.code > 2 ** 31) e.code |= 0; - const code = e.signal ? `signal ${e.signal}` : `code ${e.code}`; - e.message = `${e.cmd.split(" ")[0]} failed with ${code}`; - } - throw e; - })) as any; const ffprobeBin = testProgram("ffprobe", "--help"); const ffmpegBin = testProgram("ffmpeg", "--help"); @@ -426,7 +413,7 @@ const procDuration: Process = { enable: ffprobeBin !== null, include: rules.extsDuration, async run({ absPath, mediaFile }) { - const { stdout } = await execFile(ffprobeBin!, [ + const { stdout } = await subprocess.exec(ffprobeBin!, [ "-v", "error", "-show_entries", @@ -793,11 +780,11 @@ import { Progress } from "@paperclover/console/Progress"; import { Spinner } from "@paperclover/console/Spinner"; import * as async from "#sitegen/async"; import * as fs from "#sitegen/fs"; +import * as subprocess from "#sitegen/subprocess"; import * as path from "node:path"; import * as zlib from "node:zlib"; import * as child_process from "node:child_process"; -import * as util from "node:util"; import * as crypto from "node:crypto"; import * as stream from "node:stream"; @@ -814,3 +801,4 @@ import * as highlight from "@/file-viewer/highlight.ts"; import * as ffmpeg from "@/file-viewer/ffmpeg.ts"; import * as rsync from "@/file-viewer/rsync.ts"; import * as transcodeRules from "@/file-viewer/transcode-rules.ts"; +import { rawFileRoot as root } from "../paths.ts"; diff --git a/src/file-viewer/cache.ts b/src/file-viewer/cache.ts index fd85eeb..74121d1 100644 --- a/src/file-viewer/cache.ts +++ b/src/file-viewer/cache.ts @@ -12,9 +12,9 @@ import { escapeUri } from "./format.ts"; declare const Deno: any; const sourceOfTruth = "https://nas.paperclover.net:43250"; -const caCert = fs.readFileSync("src/file-viewer/cert.pem"); +// const caCert = fs.readFileSync("src/file-viewer/cert.pem"); -const diskCacheRoot = path.join(import.meta.dirname, "../.clover/filecache/"); +const diskCacheRoot = path.join(import.meta.dirname, ".filecache/"); const diskCacheMaxSize = 14 * 1024 * 1024 * 1024; // 14GB const ramCacheMaxSize = 1 * 1024 * 1024 * 1024; // 1.5GB const loadInProgress = new Map< @@ -338,7 +338,7 @@ const agent: any = typeof Bun !== "undefined" // Bun has two non-standard fetch extensions decompress: false, tls: { - ca: caCert, + // ca: caCert, }, } // TODO: https://github.com/denoland/deno/issues/12291 @@ -351,7 +351,7 @@ const agent: any = typeof Bun !== "undefined" // } // Node.js supports node:http : new Agent({ - ca: caCert, + // ca: caCert, }); function fetchFileNode(pathname: string): Promise { diff --git a/src/file-viewer/paths.ts b/src/file-viewer/paths.ts new file mode 100644 index 0000000..8d974d9 --- /dev/null +++ b/src/file-viewer/paths.ts @@ -0,0 +1,12 @@ +export const nasRoot = process.platform === "win32" + ? "\\\\zenith\\clover" + : process.platform === "darwin" + ? "/Volumes/clover" + : "/media/clover"; + +export const rawFileRoot = process.env.CLOVER_FILE_RAW ?? + path.join(nasRoot, "Published"); +export const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ?? + path.join(nasRoot, "Documents/Config/paperclover/derived"); + +import * as path from "node:path"; diff --git a/src/site.ts b/src/site.ts index 3c47ebc..214e8f6 100644 --- a/src/site.ts +++ b/src/site.ts @@ -1,11 +1,11 @@ -// This file defines the different "Sections" of the website. The sections act -// as separate codebases, but hosted as one. This allows me to have -// sub-projects like the file viewer in 'file', or the question answer system -// in 'q+a'. Each section can define configuration, pages, backend routes, and -// contain other files. const join = (...paths: string[]) => path.join(import.meta.dirname, ...paths); -export const siteSections: Section[] = [ +// Different sections of the website are split into their own folders. Acting as +// as separate codebases, they are hosted as one. This allows me to have +// sub-projects like the file viewer in 'file/', and the question answer system +// in 'q+a', but maintain clear boundaries. Each section can define +// configuration, pages, backend routes, and contain other files. +export const siteSections: sg.Section[] = [ { root: join(".") }, { root: join("q+a/") }, { root: join("file-viewer/") }, @@ -14,10 +14,96 @@ export const siteSections: Section[] = [ // { root: join("fiction/"), pageBase: "/fiction" }, ]; +// All backends are bundled. The backend named "backend" is run by "node run watch" export const backends: string[] = [ join("backend.ts"), join("source-of-truth.ts"), ]; +// Font subsets reduce bandwidth and protect against proprietary font theft. +const fontRoot = path.join(nasRoot, 'Documents/Font'); +const ascii = { start: 0x20, end: 0x7E }; +const nonAscii: sg.FontRange[] = [ + { start: 0xC0, end: 0xFF }, + { start: 0x2190, end: 0x2193 }, + { start: 0xA0, end: 0xA8 }, + { start: 0xAA, end: 0xBF }, + { start: 0x2194, end: 0x2199 }, + { start: 0x100, end: 0x17F }, + 0xA9, + 0x2018, + 0x2019, + 0x201C, + 0x201D, + 0x2022, +]; +const recursiveVars: sg.FontVars = { + WGHT: [400, 750], + SLNT: [-15, 0], + CASL: 0.25, + CRSV: 0.5, + MONO: 0, +}; +const recursiveVarsMono = { ...recursiveVars, MONO: 1 }; +const recursiveVarsQuestion: sg.FontVars = { + ...recursiveVars, + MONO: [0, 1], + WGHT: [400, 1000], +}; +const layoutFeatures = ["numr", "dnom", "frac"]; +export const fonts: sg.Font[] = [ + { + name: "Recursive", + sources: [ + "Recursive_VF_1.085.ttf", + path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"), + "https://paperclover.net/file/_unlisted/Recursive_VF_1.085.ttf", + ], + subsets: [ + { + asset: "/recultramin.[ext]", + layoutFeatures, + vars: recursiveVars, + unicodes: ascii, + }, + { + asset: "/recmono.[ext]", + vars: recursiveVarsMono, + unicodes: ascii, + }, + { + asset: "/recqa.[ext]", + vars: recursiveVarsQuestion, + unicodes: ascii, + }, + { + asset: "/recexotic.[ext]", + vars: recursiveVarsQuestion, + unicodes: nonAscii, + }, + ], + }, + { + name: "AT Name Sans Display Hairline", + sources: [ + "ATNameSansDisplay-Hairline.woff2", + path.join(fontRoot, "/ArrowType/Recursive/Recursive_VF_1.085.ttf"), + ], + subsets: [ + { + asset: "/cydn_header.[ext]", + layoutFeatures, + unicodes: "cotlyedon", + }, + ], + }, +]; + +export async function main() { + await font.buildFonts(fonts); +} + import * as path from "node:path"; -import type { Section } from "#sitegen"; +import * as font from "../framework/font.ts"; +import type * as sg from "#sitegen"; +import { nasRoot } from "./file-viewer/paths.ts"; diff --git a/src/source-of-truth.ts b/src/source-of-truth.ts index af2826f..1189ec0 100644 --- a/src/source-of-truth.ts +++ b/src/source-of-truth.ts @@ -22,17 +22,6 @@ export default app; const token = UNWRAP(process.env.CLOVER_SOT_KEY); -const nasRoot = process.platform === "win32" - ? "\\\\zenith\\clover" - : process.platform === "darwin" - ? "/Volumes/clover" - : "/media/clover"; - -const rawFileRoot = process.env.CLOVER_FILE_RAW ?? - path.join(nasRoot, "Published"); -const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ?? - path.join(nasRoot, "Documents/Config/paperclover/derived"); - if (!fs.existsSync(rawFileRoot)) { throw new Error(`${rawFileRoot} does not exist`); } @@ -125,3 +114,4 @@ import * as fsCallbacks from "node:fs"; import * as util from "node:util"; import * as stream from "node:stream"; import * as mime from "#sitegen/mime"; +import { derivedFileRoot, rawFileRoot } from "./file-viewer/paths.ts"; diff --git a/src/static/apple-touch-icon.png b/src/static/apple-touch-icon.png new file mode 100644 index 0000000..5f24ff6 Binary files /dev/null and b/src/static/apple-touch-icon.png differ diff --git a/src/static/favicon.ico b/src/static/favicon.ico new file mode 100644 index 0000000..738099c Binary files /dev/null and b/src/static/favicon.ico differ diff --git a/src/web.dockerfile b/src/web.dockerfile new file mode 100644 index 0000000..03c3a6c --- /dev/null +++ b/src/web.dockerfile @@ -0,0 +1,19 @@ +from node:24 as builder + +run apt install -y git + +workdir /tmp/builder +copy package*.json ./ +run npm ci +copy . ./ +run node run generate +run npm prune --production && cp -r .clover/o /app && cp -r node_modules /app/ + +from node:24 as runtime + +workdir /app +copy --from=builder /app/ ./ + +env PORT=80 + +cmd ["node", "--enable-source-maps", "/app/backend.js"]