sitegen/framework/generate.ts

export function main() {
  return withSpinner({
    text: "Recovering State",
    successText: ({ elapsed }) =>
      "sitegen! update in " + elapsed.toFixed(1) + "s",
    failureText: () => "sitegen FAIL",
  }, sitegen);
}

async function sitegen(status: Spinner) {
  const startTime = performance.now();

  let root = path.resolve(import.meta.dirname, "../src");
  const join = (...sub: string[]) => path.join(root, ...sub);
  const incr = Incremental.fromDisk();
  await incr.statAllFiles();

  // Sitegen reviews every defined section for resources to process
  const sections: sg.Section[] =
    require(path.join(root, "site.ts")).siteSections;

  // Static files are compressed and served as-is.
  // - "{section}/static/*.png"
  let staticFiles: FileItem[] = [];
  // Pages are rendered then served as static files.
  // - "{section}/pages/*.marko"
  let pages: FileItem[] = [];
  // Views are dynamically rendered pages called via backend code.
  // - "{section}/views/*.tsx"
  let views: FileItem[] = [];
  // Public scripts are bundled for the client as static assets under "/js/[...]"
  // This is used for the file viewer's canvases.
  // Note that '.client.ts' can be placed anywhere in the file structure.
  // - "{section}/scripts/*.client.ts"
  let scripts: FileItem[] = [];

  // -- Scan for files --
  status.text = "Scanning Project";
  for (const section of sections) {
    const { root: sectionRoot } = section;
    const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
    const rootPrefix = root === sectionRoot
      ? ""
      : path.relative(root, sectionRoot) + "/";
    const kinds = [
      {
        dir: sectionPath("pages"),
        list: pages,
        prefix: "/",
        exclude: [".css", ".client.ts", ".client.tsx"],
      },
      { dir: sectionPath("static"), list: staticFiles, prefix: "/", ext: true },
      { dir: sectionPath("scripts"), list: scripts, prefix: rootPrefix },
      {
        dir: sectionPath("views"),
        list: views,
        prefix: rootPrefix,
        exclude: [".css", ".client.ts", ".client.tsx"],
      },
    ];
    for (const { dir, list, prefix, exclude = [], ext = false } of kinds) {
      const items = fs.readDirRecOptionalSync(dir);
      item: for (const subPath of items) {
        const file = path.join(dir, subPath);
        const stat = fs.statSync(file);
        if (stat.isDirectory()) continue;
        for (const e of exclude) {
          if (subPath.endsWith(e)) continue item;
        }
        const trim = ext
          ? subPath
          : subPath.slice(0, -path.extname(subPath).length).replaceAll(
            ".",
            "/",
          );
        let id = prefix + trim.replaceAll("\\", "/");
        if (prefix === "/" && id.endsWith("/index")) {
          id = id.slice(0, -"/index".length) || "/";
        }
        list.push({ id, file: file });
      }
    }
  }
  scripts = scripts.filter(({ file }) => !file.match(/\.client\.[tj]sx?/));
  const globalCssPath = join("global.css");

  // TODO: invalidate incremental resources

  // -- server side render --
  status.text = "Building";
  const cssOnce = new OnceMap<css.Output>();
  const cssQueue = new Queue<[string, string[], css.Theme], css.Output>({
    name: "Bundle",
    fn: ([, files, theme]) => css.bundleCssFiles(files, theme),
    passive: true,
    getItemText: ([id]) => id,
    maxJobs: 2,
  });
  interface RenderResult {
    body: string;
    head: string;
    css: css.Output;
    scriptFiles: string[];
    item: FileItem;
  }
  const renderResults: RenderResult[] = [];
  async function loadPageModule({ file }: FileItem) {
    require(file);
  }
  async function renderPage(item: FileItem) {
    // -- load and validate module --
    let {
      default: Page,
      meta: metadata,
      theme: pageTheme,
      layout,
    } = require(item.file);
    if (!Page) throw new Error("Page is missing a 'default' export.");
    if (!metadata) {
      throw new Error("Page is missing 'meta' export with a title.");
    }
    if (layout?.theme) pageTheme = layout.theme;
    const theme = {
      bg: "#fff",
      fg: "#050505",
      primary: "#2e7dab",
      ...pageTheme,
    };

    // -- metadata --
    const renderedMetaPromise = Promise.resolve(
      typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
    ).then((m) => meta.renderMeta(m));
    // -- css --
    const cssImports = [globalCssPath, ...hot.getCssImports(item.file)];
    const cssPromise = cssOnce.get(
      cssImports.join(":") + JSON.stringify(theme),
      () => cssQueue.add([item.id, cssImports, theme]),
    );
    // -- html --
    let page = [engine.kElement, Page, {}];
    if (layout?.default) {
      page = [engine.kElement, layout.default, { children: page }];
    }
    const bodyPromise = engine.ssrAsync(page, {
      sitegen: sg.initRender(),
    });

    const [{ text, addon }, cssBundle, renderedMeta] = await Promise.all([
      bodyPromise,
      cssPromise,
      renderedMetaPromise,
    ]);
    if (!renderedMeta.includes("<title>")) {
      throw new Error(
        "Page is missing 'meta.title'. " +
          "All pages need a title tag.",
      );
    }
    // The script content is not ready, allow another page to Render. The page
    // contents will be rebuilt at the end. This is more convenient anyways
    // because it means client scripts don't re-render the page.
    renderResults.push({
      body: text,
      head: renderedMeta,
      css: cssBundle,
      scriptFiles: Array.from(addon.sitegen.scripts),
      item: item,
    });
  }
  // This is done in two passes so that a page that throws during evaluation
  // will report "Load Render Module" instead of "Render Static Page".
  const neededPages = pages.filter((page) => incr.needsBuild("asset", page.id));
  const spinnerFormat = status.format;
  status.format = () => "";
  const moduleLoadQueue = new Queue({
    name: "Load Render Module",
    fn: loadPageModule,
    getItemText,
    maxJobs: 1,
  });
  moduleLoadQueue.addMany(neededPages);
  await moduleLoadQueue.done({ method: "stop" });
  const pageQueue = new Queue({
    name: "Render Static Page",
    fn: renderPage,
    getItemText,
    maxJobs: 2,
  });
  pageQueue.addMany(neededPages);
  await pageQueue.done({ method: "stop" });
  status.format = spinnerFormat;

  // -- bundle backend and views --
  status.text = "Bundle backend code";
  const {} = await bundle.bundleServerJavaScript(
    join("backend.ts"),
    views,
  );

  // -- bundle scripts --
  const referencedScripts = Array.from(
    new Set(renderResults.flatMap((r) => r.scriptFiles)),
  );
  const extraPublicScripts = scripts.map((entry) => entry.file);
  const uniqueCount = new Set([
    ...referencedScripts,
    ...extraPublicScripts,
  ]).size;
  status.text = `Bundle ${uniqueCount} Scripts`;
  await bundle.bundleClientJavaScript(
    referencedScripts,
    extraPublicScripts,
    incr,
  );

  // -- copy/compress static files --
  async function doStaticFile(item: FileItem) {
    const body = await fs.readFile(item.file);
    await incr.putAsset({
      sources: [item.file],
      key: item.id,
      body,
    });
  }
  const staticQueue = new Queue({
    name: "Load Static",
    fn: doStaticFile,
    getItemText,
    maxJobs: 16,
  });
  status.format = () => "";
  staticQueue.addMany(
    staticFiles.filter((file) => incr.needsBuild("asset", file.id)),
  );
  await staticQueue.done({ method: "stop" });
  status.format = spinnerFormat;

  // -- concatenate static rendered pages --
  status.text = `Concat ${renderResults.length} Pages`;
  await Promise.all(
    renderResults.map(
      async (
        { item: page, body, head, css, scriptFiles },
      ) => {
        const doc = wrapDocument({
          body,
          head,
          inlineCss: css.text,
          scripts: scriptFiles.map(
            (id) =>
              UNWRAP(
                incr.out.script.get(
                  path.basename(id).replace(/\.client\.[jt]sx?$/, ""),
                ),
              ),
          ).map((x) => `{${x}}`).join("\n"),
        });
        await incr.putAsset({
          sources: [page.file, ...css.sources],
          key: page.id,
          body: doc,
          headers: {
            "Content-Type": "text/html",
          },
        });
      },
    ),
  );
  status.format = () => "";
  status.text = ``;
  // This will wait for all compression jobs to finish, which up
  // to this point have been left as dangling promises.
  await incr.wait();

  // Flush the site to disk.
  status.format = spinnerFormat;
  status.text = `Incremental Flush`;
  incr.flush(); // Write outputs
  incr.toDisk(); // Allows picking up this state again
  return { elapsed: (performance.now() - startTime) / 1000 };
}

function getItemText({ file }: FileItem) {
  return path.relative(hot.projectSrc, file).replaceAll("\\", "/");
}

import { OnceMap, Queue } from "./queue.ts";
import { Incremental } from "./incremental.ts";
import * as bundle from "./bundle.ts";
import * as css from "./css.ts";
import * as engine from "./engine/ssr.ts";
import * as hot from "./hot.ts";
import * as fs from "#sitegen/fs";
import * as sg from "#sitegen";
import type { FileItem } from "#sitegen";
import * as path from "node:path";
import * as meta from "#sitegen/meta";
import { Spinner, withSpinner } from "@paperclover/console/Spinner";
import { wrapDocument } from "./lib/view.ts";