// The "source of truth" server is the canonical storage for // paper clover's files. This is technically needed because // the VPS she uses can only store about 20gb of content, where // the contents of /file is about 48gb as of writing; only a // limited amount of data can be cached. // // What's great about this system is it also allows scaling the // website up into multiple servers, if that is ever desired. // When that happens, mutations to "q+a" will be moved here, // and the SQLite synchronization mechanism will apply to both // of those databases. // // An alternative to this would have been to use a SMB client, // but I've read that the systems used for caching don't work // like the HTTP Cache-Control header, where you can say a file // is valid for up to a certain amount of time. If we seriously // need cache busts (paper clover does not), the proper way // would be to push a message to all VPS nodes instead of // checking upstream if a file changed every time. const app = new Hono(); export default app; const token = process.env.CLOVER_SOT_KEY; const nasRoot = process.platform === "win32" ? "\\\\zenith\\clover" : process.platform === "darwin" ? "/Volumes/clover" : "/media/clover"; const rawFileRoot = process.env.CLOVER_FILE_RAW ?? path.join(nasRoot, "Published"); const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ?? path.join(nasRoot, "Documents/Config/paperclover/derived"); if (!fs.existsSync(rawFileRoot)) { throw new Error(`${rawFileRoot} does not exist`); } if (!fs.existsSync(derivedFileRoot)) { throw new Error(`${derivedFileRoot} does not exist`); } type Awaitable = T | Promise; // Re-use file descriptors if the same file is being read twice. const fds = new Map>(); app.get("/file/*", async (c) => { const fullQuery = c.req.path.slice("/file".length); const [filePath, derivedAsset, ...invalid] = fullQuery.split("$/"); if (invalid.length > 0) return c.notFound(); if (filePath.length <= 1) return c.notFound(); const permissions = FilePermissions.getByPrefix(filePath); if (permissions !== 0) { if (c.req.header("Authorization") !== token) { return c.json({ error: "invalid authorization header" }); } } const file = MediaFile.getByPath(filePath); if (!file || file.kind === MediaFileKind.directory) { return c.notFound(); } const fullPath = derivedAsset ? path.join(derivedFileRoot, file.hash, derivedAsset) : path.join(rawFileRoot, file.path); let handle: { fd: number; refs: number } | null = null; try { handle = await fds.get(fullPath) ?? null; if (!handle) { const promise = openFile(fullPath, "r") .then((fd) => ({ fd, refs: 0 })) .catch((err) => { fds.delete(fullPath); throw err; }); fds.set(file.path, promise); fds.set(file.path, handle = await promise); } handle.refs += 1; } catch (err: any) { if (err.code === "ENOENT") { return c.notFound(); } throw err; } const nodeStream = fs.createReadStream(fullPath, { fd: handle.fd, fs: { close: util.callbackify(async () => { ASSERT(handle); if ((handle.refs -= 1) <= 0) { fds.delete(fullPath); await closeFile(handle.fd); } handle = null; }), read: fsCallbacks.read, }, }); c.header("Content-Type", mime.contentTypeFor(fullPath)); if (!derivedAsset) { c.header("Content-Length", file.size.toString()); } return c.body(stream.Readable.toWeb(nodeStream) as ReadableStream); }); const openFile = util.promisify(fsCallbacks.open); const closeFile = util.promisify(fsCallbacks.close); import { Hono } from "#hono"; import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts"; import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts"; import * as path from "node:path"; import * as fs from "#sitegen/fs"; import * as fsCallbacks from "node:fs"; import * as util from "node:util"; import * as stream from "node:stream"; import * as mime from "#sitegen/mime";