sitegen/src/source-of-truth.ts

// The "source of truth" server is the canonical storage for
// paper clover's files. This is technically needed because
// the VPS she uses can only store about 20gb of content, where
// the contents of /file is about 48gb as of writing; only a
// limited amount of data can be cached.
//
// What's great about this system is it also allows scaling the
// website up into multiple servers, if that is ever desired.
// When that happens, mutations to "q+a" will be moved here,
// and the SQLite synchronization mechanism will apply to both
// of those databases.
//
// An alternative to this would have been to use a SMB client,
// but I've read that the systems used for caching don't work
// like the HTTP Cache-Control header, where you can say a file
// is valid for up to a certain amount of time. If we seriously
// need cache busts (paper clover does not), the proper way
// would be to push a message to all VPS nodes instead of
// checking upstream if a file changed every time.
const app = new Hono();
export default app;

const token = "bwaa";
const nasRoot = "/Volumes/clover";

const rawFileRoot = path.join(nasRoot, "Published");
const derivedFileRoot = path.join(
  nasRoot,
  "Documents/Config/clover_file/derived",
);

type Awaitable<T> = T | Promise<T>;

// Re-use file descriptors if the same file is being read twice.
const fds = new Map<string, Awaitable<{ fd: number; refs: number }>>();

app.get("/file/*", async (c) => {
  const fullQuery = c.req.path.slice("/file".length);
  const [filePath, derivedAsset, ...invalid] = fullQuery.split("$/");
  if (invalid.length > 0) return c.notFound();
  if (filePath.length <= 1) return c.notFound();
  const permissions = FilePermissions.getByPrefix(filePath);
  if (permissions !== 0) {
    if (c.req.header("Authorization") !== token) {
      return c.json({ error: "invalid authorization header" });
    }
  }
  const file = MediaFile.getByPath(filePath);
  if (!file || file.kind === MediaFileKind.directory) {
    return c.notFound();
  }
  const fullPath = derivedAsset
    ? path.join(derivedFileRoot, file.hash, derivedAsset)
    : path.join(rawFileRoot, file.path);
  let handle: { fd: number; refs: number } | null = null;
  console.log("start", fullPath);
  try {
    handle = await fds.get(fullPath) ?? null;
    if (!handle) {
      const promise = openFile(fullPath, "r")
        .then((fd) => ({ fd, refs: 0 }))
        .catch((err) => {
          fds.delete(fullPath);
          throw err;
        });
      fds.set(file.path, promise);
      fds.set(file.path, handle = await promise);
    }
    handle.refs += 1;
  } catch (err: any) {
    if (err.code === "ENOENT") {
      return c.notFound();
    }
    throw err;
  }
  const nodeStream = fs.createReadStream(fullPath, {
    fd: handle.fd,
    fs: {
      close: util.callbackify(async () => {
        ASSERT(handle);
        if ((handle.refs -= 1) <= 0) {
          fds.delete(fullPath);
          await closeFile(handle.fd);
        }
        handle = null;
      }),
      read: fsCallbacks.read,
    },
  });
  c.header("Content-Type", mime.contentTypeFor(fullPath));
  if (!derivedAsset)
    c.header("Content-Length", file.size.toString());
  return c.body(stream.Readable.toWeb(nodeStream) as ReadableStream);
});

const openFile = util.promisify(fsCallbacks.open);
const closeFile = util.promisify(fsCallbacks.close);

import { Hono } from "#hono";
import { MediaFile, MediaFileKind } from "@/file-viewer/models/MediaFile.ts";
import { FilePermissions } from "@/file-viewer/models/FilePermissions.ts";
import * as path from "node:path";
import * as fs from "#sitegen/fs";
import * as fsCallbacks from "node:fs";
import * as util from "node:util";
import * as stream from "node:stream";
import * as mime from "#sitegen/mime";