chloe caruso
c5ac450f21
the asset system is reworked to support "dynamic" entries, where each
entry is a separate file on disk containing the latest generation's
headers+raw+gzip+zstd. when calling view.regenerate, it will look for
pages that had "export const regenerate" during generation, and render
those pages using the view system, but then store the results as assets
instead of sending as a response.
pages configured as regenerable are also bundled as views, using the
non-aliasing key "page:${page.id}". this cannot alias because file
paths may not contain a colon.
118 lines
4 KiB
TypeScript
118 lines
4 KiB
TypeScript
// The "source of truth" server is the canonical storage for
|
|
// paper clover's files. This is technically needed because
|
|
// the VPS she uses can only store about 20gb of content, where
|
|
// the contents of /file is about 48gb as of writing; only a
|
|
// limited amount of data can be cached.
|
|
//
|
|
// What's great about this system is it also allows scaling the
|
|
// website up into multiple servers, if that is ever desired.
|
|
// When that happens, mutations to "q+a" will be moved here,
|
|
// and the SQLite synchronization mechanism will apply to both
|
|
// of those databases.
|
|
//
|
|
// An alternative to this would have been to use a SMB client,
|
|
// but I've read that the systems used for caching don't work
|
|
// like the HTTP Cache-Control header, where you can say a file
|
|
// is valid for up to a certain amount of time. If we seriously
|
|
// need cache busts (paper clover does not), the proper way
|
|
// would be to push a message to all VPS nodes instead of
|
|
// checking upstream if a file changed every time.
|
|
const app = new Hono();
|
|
export default app;
|
|
|
|
const token = process.env.CLOVER_SOT_KEY;
|
|
|
|
const nasRoot = process.platform === "win32"
|
|
? "\\\\zenith\\clover"
|
|
: process.platform === "darwin"
|
|
? "/Volumes/clover"
|
|
: "/media/clover";
|
|
|
|
const rawFileRoot = process.env.CLOVER_FILE_RAW ??
|
|
path.join(nasRoot, "Published");
|
|
const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
|
|
path.join(nasRoot, "Documents/Config/paperclover/derived");
|
|
|
|
if (!fs.existsSync(rawFileRoot)) {
|
|
throw new Error(`${rawFileRoot} does not exist`);
|
|
}
|
|
if (!fs.existsSync(derivedFileRoot)) {
|
|
throw new Error(`${derivedFileRoot} does not exist`);
|
|
}
|
|
|
|
type Awaitable<T> = T | Promise<T>;
|
|
|
|
// Re-use file descriptors if the same file is being read twice.
|
|
const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();
|
|
|
|
app.get("/file/*", async (c) => {
|
|
const fullQuery = c.req.path.slice("/file".length);
|
|
const [filePath, derivedAsset, ...invalid] = fullQuery.split("$/");
|
|
if (invalid.length > 0) return c.notFound();
|
|
if (filePath.length <= 1) return c.notFound();
|
|
const permissions = FilePermissions.getByPrefix(filePath);
|
|
if (permissions !== 0) {
|
|
if (c.req.header("Authorization") !== token) {
|
|
return c.json({ error: "invalid authorization header" });
|
|
}
|
|
}
|
|
const file = MediaFile.getByPath(filePath);
|
|
if (!file || file.kind === MediaFileKind.directory) {
|
|
return c.notFound();
|
|
}
|
|
const fullPath = derivedAsset
|
|
? path.join(derivedFileRoot, file.hash, derivedAsset)
|
|
: path.join(rawFileRoot, file.path);
|
|
let handle: { fd: number; refs: number } | null = null;
|
|
try {
|
|
handle = await fds.get(fullPath) ?? null;
|
|
if (!handle) {
|
|
const promise = openFile(fullPath, "r")
|
|
.then((fd) => ({ fd, refs: 0 }))
|
|
.catch((err) => {
|
|
fds.delete(fullPath);
|
|
throw err;
|
|
});
|
|
fds.set(file.path, promise);
|
|
fds.set(file.path, handle = await promise);
|
|
}
|
|
handle.refs += 1;
|
|
} catch (err: any) {
|
|
if (err.code === "ENOENT") {
|
|
return c.notFound();
|
|
}
|
|
throw err;
|
|
}
|
|
const nodeStream = fs.createReadStream(fullPath, {
|
|
fd: handle.fd,
|
|
fs: {
|
|
close: util.callbackify(async () => {
|
|
ASSERT(handle);
|
|
if ((handle.refs -= 1) <= 0) {
|
|
fds.delete(fullPath);
|
|
await closeFile(handle.fd);
|
|
}
|
|
handle = null;
|
|
}),
|
|
read: fsCallbacks.read,
|
|
},
|
|
});
|
|
c.header("Content-Type", mime.contentTypeFor(fullPath));
|
|
if (!derivedAsset) {
|
|
c.header("Content-Length", file.size.toString());
|
|
}
|
|
return c.body(stream.Readable.toWeb(nodeStream) as ReadableStream);
|
|
});
|
|
|
|
const openFile = util.promisify(fsCallbacks.open);
|
|
const closeFile = util.promisify(fsCallbacks.close);
|
|
|
|
import { Hono } from "#hono";
|
|
import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
|
|
import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
|
|
import * as path from "node:path";
|
|
import * as fs from "#sitegen/fs";
|
|
import * as fsCallbacks from "node:fs";
|
|
import * as util from "node:util";
|
|
import * as stream from "node:stream";
|
|
import * as mime from "#sitegen/mime";
|