sitegen/framework/generate.ts

// This file contains the main site generator build process.
// By using `incr.work`'s ability to cache work between runs,
// the site generator is very fast to re-run.
//
// See `watch.ts` for a live development environment.
const { toRel, toAbs } = incr;
const globalCssPath = toAbs("src/global.css");

export async function main() {
  if (!process.argv.includes("-f")) await incr.restore();
  await incr.compile(generate);
}

export async function generate() {
  // -- read config and discover files --
  const siteConfig = await incr.work(readManifest);
  const metaTemplate: meta.Template = {
    ...siteConfig.meta,
    base: new URL(siteConfig.meta.base),
  };
  const {
    staticFiles,
    scripts,
    views,
    pages,
  } = await discoverAllFiles(siteConfig);

  // TODO: make sure that `static` and `pages` does not overlap

  // TODO: loadMarkoCache

  // -- start font work --
  const builtFonts = fonts.buildFonts(siteConfig.fonts);

  // -- perform build-time rendering --
  const builtPages = pages.map((item) =>
    incr.work(preparePage, { item, metaTemplate })
  );
  const builtViews = views.map((item) => incr.work(prepareView, item));
  const builtStaticFiles = Promise.all(
    staticFiles.map((item) =>
      incr.work(
        async (io, { id, file }) =>
          void (await io.writeAsset({
            pathname: id,
            buffer: await io.readFile(file),
          })),
        item,
      )
    ),
  );
  const routes = await Promise.all([...builtViews, ...builtPages]);

  const viewsAndDynPages: incr.Ref<PageOrView>[] = [
    ...builtViews,
    ...builtPages.filter((page) => UNWRAP(page.value).regenerate),
  ];

  // -- page resources --
  const scriptMap = incr.work(bundle.bundleClientJavaScript, {
    clientRefs: routes.flatMap((x) => x.clientRefs),
    extraPublicScripts: scripts.map((entry) => entry.file),
    dev: false,
  });
  const styleMap = prepareInlineCss(routes);

  // -- backend --
  const builtBackend = bundle.bundleServerJavaScript({
    entries: siteConfig.backends,
    platform: "node",
    styleMap,
    scriptMap,
    viewItems: viewsAndDynPages.map((ref) => {
      const { id, file, type } = UNWRAP(ref.value);
      return { id: type === "page" ? `page:${id}` : id, file };
    }),
    viewRefs: viewsAndDynPages,
    metaTemplate,
  });

  // -- assemble page assets --
  const pAssemblePages = builtPages.map((page) =>
    assembleAndWritePage(page, styleMap, scriptMap)
  );

  await Promise.all([
    builtBackend,
    builtStaticFiles,
    ...pAssemblePages,
    builtFonts,
  ]);
}

export async function readManifest(io: Io) {
  const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
  return {
    siteSections: cfg.siteSections.map((section) => ({
      root: toRel(section.root),
    })),
    backends: cfg.backends.map(toRel),
    fonts: cfg.fonts,
    meta: {
      ...cfg.meta,
      base: cfg.meta.base.toString(),
    },
  };
}

export async function discoverAllFiles(
  siteConfig: Awaited<ReturnType<typeof readManifest>>,
) {
  return (
    await Promise.all(
      siteConfig.siteSections.map(({ root: sectionRoot }) =>
        incr.work(scanSiteSection, toAbs(sectionRoot))
      ),
    )
  ).reduce((acc, next) => ({
    staticFiles: acc.staticFiles.concat(next.staticFiles),
    pages: acc.pages.concat(next.pages),
    views: acc.views.concat(next.views),
    scripts: acc.scripts.concat(next.scripts),
  }));
}

export async function scanSiteSection(io: Io, sectionRoot: string) {
  // Static files are compressed and served as-is.
  // - "{section}/static/*.png"
  let staticFiles: FileItem[] = [];
  // Pages are rendered then served as static files.
  // - "{section}/pages/*.marko"
  let pages: FileItem[] = [];
  // Views are dynamically rendered pages called via backend code.
  // - "{section}/views/*.tsx"
  let views: FileItem[] = [];
  // Public scripts are bundled for the client as static assets under "/js/[...]"
  // This is used for the file viewer's canvases.
  // Note that '.client.ts' can be placed anywhere in the file structure.
  // - "{section}/scripts/*.client.ts"
  let scripts: FileItem[] = [];

  const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
  const rootPrefix = hot.projectSrc === sectionRoot
    ? ""
    : path.relative(hot.projectSrc, sectionRoot) + "/";
  const kinds = [
    {
      dir: sectionPath("pages"),
      list: pages,
      prefix: "/",
      include: [".tsx", ".mdx", ".marko"],
      exclude: [".client.ts", ".client.tsx"],
    },
    {
      dir: sectionPath("static"),
      list: staticFiles,
      prefix: "/",
      ext: true,
    },
    {
      dir: sectionPath("scripts"),
      list: scripts,
      prefix: rootPrefix,
      include: [".client.ts", ".client.tsx"],
    },
    {
      dir: sectionPath("views"),
      list: views,
      prefix: rootPrefix,
      include: [".tsx", ".mdx", ".marko"],
      exclude: [".client.ts", ".client.tsx"],
    },
  ];
  for (const kind of kinds) {
    const {
      dir,
      list,
      prefix,
      include = [""],
      exclude = [],
      ext = false,
    } = kind;

    let items;
    try {
      items = await io.readDirRecursive(dir);
    } catch (err: any) {
      if (err.code === "ENOENT") continue;
      throw err;
    }
    for (const subPath of items) {
      const file = path.join(dir, subPath);
      const stat = fs.statSync(file);
      if (stat.isDirectory()) continue;
      if (!include.some((e) => subPath.endsWith(e))) continue;
      if (exclude.some((e) => subPath.endsWith(e))) continue;
      const trim = ext
        ? subPath
        : subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
      let id = prefix + trim.replaceAll("\\", "/");
      if (prefix === "/" && id.endsWith("/index")) {
        id = id.slice(0, -"/index".length) || "/";
      }
      list.push({ id, file: path.relative(hot.projectRoot, file) });
    }
  }

  return { staticFiles, pages, views, scripts };
}

export async function preparePage(
  io: Io,
  { item, metaTemplate }: { item: sg.FileItem; metaTemplate: meta.Template },
) {
  // -- load and validate module --
  let {
    default: Page,
    meta: metadata,
    theme: pageTheme,
    layout,
    regenerate,
  } = await io.import<sg.PageExports>(item.file);
  if (!Page) throw new Error("Page is missing a 'default' export.");
  if (!metadata) throw new Error("Page is missing 'meta' export with a title.");

  // -- css --
  if (layout?.theme) pageTheme = layout.theme;
  const theme: css.Theme = {
    ...css.defaultTheme,
    ...pageTheme,
  };
  const cssImports = Array.from(
    new Set([globalCssPath, ...hot.getCssImports(item.file)]),
    (file) => path.relative(hot.projectSrc, file),
  );

  // -- metadata --
  const renderedMetaPromise = Promise.resolve(
    typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
  ).then((m) => meta.render({
    canonical: item.id,
    ...m,
  }, metaTemplate));

  // -- html --
  let page = render.element(Page);
  if (layout?.default) {
    page = render.element(layout.default, { children: page });
  }
  const bodyPromise = render.async(page, {
    [sg.userData.key]: sg.initRender(),
  });

  const [{ text, addon }, renderedMeta] = await Promise.all([
    bodyPromise,
    renderedMetaPromise,
  ]);
  if (!renderedMeta.includes("<title>")) {
    throw new Error(
      "Page is missing 'meta.title'. " + "All pages need a title tag.",
    );
  }

  const styleKey = css.styleKey(cssImports, theme);
  return {
    type: "page",
    id: item.id,
    file: item.file,
    regenerate,
    html: text,
    meta: renderedMeta,
    cssImports,
    theme: theme ?? null,
    styleKey,
    clientRefs: Array.from(addon[sg.userData.key].scripts),
  } as const;
}

export async function prepareView(io: Io, item: sg.FileItem) {
  const module = await io.import<any>(item.file);
  if (!module.meta) throw new Error(`View is missing 'export const meta'`);
  if (!module.default) throw new Error(`View is missing a default export.`);
  const pageTheme = module.layout?.theme ?? module.theme;
  const theme: css.Theme = { ...css.defaultTheme, ...pageTheme };
  const cssImports = Array.from(
    new Set([globalCssPath, ...hot.getCssImports(item.file)]),
    (file) => path.relative(hot.projectSrc, file),
  );
  const styleKey = css.styleKey(cssImports, theme);
  return {
    type: "view",
    id: item.id,
    file: item.file,
    cssImports,
    theme,
    clientRefs: hot.getClientScriptRefs(item.file),
    hasLayout: !!module.layout?.default,
    styleKey,
  } as const;
}
export type PreparedView = Awaited<ReturnType<typeof prepareView>>;

export function prepareInlineCss(
  items: Array<{ styleKey: string; cssImports: string[]; theme: css.Theme }>,
) {
  const map = new Map<string, incr.Ref<string>>();
  for (const { styleKey, cssImports, theme } of items) {
    if (map.has(styleKey)) continue;
    map.set(
      styleKey,
      incr.work(css.bundleCssFiles, { cssImports, theme, dev: false }),
    );
  }
  return map;
}

export type PreparedPage = Awaited<ReturnType<typeof preparePage>>;
export async function assembleAndWritePage(
  pageWork: incr.Ref<PreparedPage>,
  styleMap: Map<string, incr.Ref<string>>,
  scriptWork: incr.Ref<Record<string, string>>,
) {
  const page = await pageWork;
  return incr.work(
    async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {
      const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));

      const scriptIds = clientRefs.map(hot.getScriptId);
      const scriptMap = await io.readWork(scriptWork);
      const scripts = scriptIds
        .map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))
        .map((x) => `{${x}}`)
        .join("\n");

      const buffer = sg.wrapDocument({
        body: html,
        head: meta,
        inlineCss,
        scripts,
      });
      await io.writeAsset({
        pathname: id,
        buffer,
        headers: {
          "Content-Type": "text/html;charset=utf8",
        },
        regenerative: !!regenerate,
      });
    },
    page,
  );
}

export type PageOrView = PreparedPage | PreparedView;

import * as path from "node:path";

import * as fs from "#sitegen/fs";
import * as meta from "#sitegen/meta";
import * as render from "#engine/render";
import * as sg from "#sitegen";
import type { FileItem } from "#sitegen";

import * as bundle from "./bundle.ts";
import * as css from "./css.ts";
import * as fonts from "./font.ts";
import * as hot from "./hot.ts";
import * as incr from "./incremental.ts";
import { Io } from "./incremental.ts";