115 lines
4 KiB
TypeScript
115 lines
4 KiB
TypeScript
// The "source of truth" server is the canonical storage for
|
|
// paper clover's files. This is technically needed because
|
|
// the VPS she uses can only store about 20gb of content, where
|
|
// the contents of /file is about 48gb as of writing; only a
|
|
// limited amount of data can be cached.
|
|
//
|
|
// What's great about this system is it also allows scaling the
|
|
// website up into multiple servers, if that is ever desired.
|
|
// When that happens, mutations to "q+a" will be moved here,
|
|
// and the SQLite synchronization mechanism will apply to both
|
|
// of those databases.
|
|
//
|
|
// An alternative to this would have been to use a SMB client,
|
|
// but I've read that the systems used for caching don't work
|
|
// like the HTTP Cache-Control header, where you can say a file
|
|
// is valid for up to a certain amount of time. If we seriously
|
|
// need cache busts (paper clover does not), the proper way
|
|
// would be to push a message to all VPS nodes instead of
|
|
// checking upstream if a file changed every time.
|
|
const app = new Hono();
|
|
export default app;
|
|
|
|
const token = process.env.CLOVER_SOT_KEY;
|
|
|
|
const nasRoot = process.platform === "win32"
|
|
? "\\\\zenith\\clover"
|
|
: process.platform === "darwin"
|
|
? "/Volumes/clover"
|
|
: "/media/clover";
|
|
|
|
const rawFileRoot = process.env.CLOVER_FILE_RAW ??
|
|
path.join(nasRoot, "Published");
|
|
const derivedFileRoot = process.env.CLOVER_FILE_DERIVED ??
|
|
path.join(nasRoot, "Documents/Config/paperclover/derived");
|
|
|
|
if (!fs.existsSync(rawFileRoot)) throw new Error(`${rawFileRoot} does not exist`);
|
|
if (!fs.existsSync(derivedFileRoot)) throw new Error(`${derivedFileRoot} does not exist`);
|
|
|
|
type Awaitable<T> = T | Promise<T>;
|
|
|
|
// Re-use file descriptors if the same file is being read twice.
|
|
const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();
|
|
|
|
app.get("/file/*", async (c) => {
|
|
const fullQuery = c.req.path.slice("/file".length);
|
|
const [filePath, derivedAsset, ...invalid] = fullQuery.split("$/");
|
|
if (invalid.length > 0) return c.notFound();
|
|
if (filePath.length <= 1) return c.notFound();
|
|
const permissions = FilePermissions.getByPrefix(filePath);
|
|
if (permissions !== 0) {
|
|
if (c.req.header("Authorization") !== token) {
|
|
return c.json({ error: "invalid authorization header" });
|
|
}
|
|
}
|
|
const file = MediaFile.getByPath(filePath);
|
|
if (!file || file.kind === MediaFileKind.directory) {
|
|
return c.notFound();
|
|
}
|
|
const fullPath = derivedAsset
|
|
? path.join(derivedFileRoot, file.hash, derivedAsset)
|
|
: path.join(rawFileRoot, file.path);
|
|
let handle: { fd: number; refs: number } | null = null;
|
|
console.log("start", fullPath);
|
|
try {
|
|
handle = await fds.get(fullPath) ?? null;
|
|
if (!handle) {
|
|
const promise = openFile(fullPath, "r")
|
|
.then((fd) => ({ fd, refs: 0 }))
|
|
.catch((err) => {
|
|
fds.delete(fullPath);
|
|
throw err;
|
|
});
|
|
fds.set(file.path, promise);
|
|
fds.set(file.path, handle = await promise);
|
|
}
|
|
handle.refs += 1;
|
|
} catch (err: any) {
|
|
if (err.code === "ENOENT") {
|
|
return c.notFound();
|
|
}
|
|
throw err;
|
|
}
|
|
const nodeStream = fs.createReadStream(fullPath, {
|
|
fd: handle.fd,
|
|
fs: {
|
|
close: util.callbackify(async () => {
|
|
ASSERT(handle);
|
|
if ((handle.refs -= 1) <= 0) {
|
|
fds.delete(fullPath);
|
|
await closeFile(handle.fd);
|
|
}
|
|
handle = null;
|
|
}),
|
|
read: fsCallbacks.read,
|
|
},
|
|
});
|
|
c.header("Content-Type", mime.contentTypeFor(fullPath));
|
|
if (!derivedAsset) {
|
|
c.header("Content-Length", file.size.toString());
|
|
}
|
|
return c.body(stream.Readable.toWeb(nodeStream) as ReadableStream);
|
|
});
|
|
|
|
const openFile = util.promisify(fsCallbacks.open);
|
|
const closeFile = util.promisify(fsCallbacks.close);
|
|
|
|
import { Hono } from "#hono";
|
|
import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
|
|
import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
|
|
import * as path from "node:path";
|
|
import * as fs from "#sitegen/fs";
|
|
import * as fsCallbacks from "node:fs";
|
|
import * as util from "node:util";
|
|
import * as stream from "node:stream";
|
|
import * as mime from "#sitegen/mime";
|