the asset system is reworked to support "dynamic" entries, where each entry is a separate file on disk containing the latest generation's headers+raw+gzip+zstd. when calling view.regenerate, it will look for pages that had "export const regenerate" during generation, and render those pages using the view system, but then store the results as assets instead of sending as a response. pages configured as regenerable are also bundled as views, using the non-aliasing key "page:${page.id}". this cannot alias because file paths may not contain a colon.
337 lines
9.8 KiB
TypeScript
337 lines
9.8 KiB
TypeScript
// This file contains the main site generator build process.
|
|
// By using `incr.work`'s ability to cache work between runs,
|
|
// the site generator is very fast to re-run.
|
|
//
|
|
// See `watch.ts` for a live development environment.
|
|
const { toRel, toAbs } = incr;
|
|
const globalCssPath = toAbs("src/global.css");
|
|
|
|
export async function main() {
|
|
if (!process.argv.includes("-f")) await incr.restore();
|
|
await incr.compile(generate);
|
|
}
|
|
|
|
export async function generate() {
|
|
// -- read config and discover files --
|
|
const siteConfig = await incr.work(readManifest);
|
|
const { staticFiles, scripts, views, pages } = await discoverAllFiles(
|
|
siteConfig,
|
|
);
|
|
|
|
// TODO: make sure that `static` and `pages` does not overlap
|
|
|
|
// TODO: loadMarkoCache
|
|
|
|
// -- perform build-time rendering --
|
|
const builtPages = pages.map((item) => incr.work(preparePage, item));
|
|
const builtViews = views.map((item) => incr.work(prepareView, item));
|
|
const builtStaticFiles = Promise.all(
|
|
staticFiles.map((item) =>
|
|
incr.work(
|
|
async (io, { id, file }) =>
|
|
void (await io.writeAsset({
|
|
pathname: id,
|
|
buffer: await io.readFile(file),
|
|
})),
|
|
item,
|
|
)
|
|
),
|
|
);
|
|
const routes = await Promise.all([...builtViews, ...builtPages]);
|
|
|
|
const viewsAndDynPages: incr.Ref<PageOrView>[] = [
|
|
...builtViews,
|
|
...builtPages.filter((page) => UNWRAP(page.value).regenerate),
|
|
];
|
|
|
|
// -- page resources --
|
|
const scriptMap = incr.work(bundle.bundleClientJavaScript, {
|
|
clientRefs: routes.flatMap((x) => x.clientRefs),
|
|
extraPublicScripts: scripts.map((entry) => entry.file),
|
|
dev: false,
|
|
});
|
|
const styleMap = prepareInlineCss(routes);
|
|
|
|
// -- backend --
|
|
const builtBackend = bundle.bundleServerJavaScript({
|
|
entries: siteConfig.backends,
|
|
platform: "node",
|
|
styleMap,
|
|
scriptMap,
|
|
viewItems: viewsAndDynPages.map((ref) => {
|
|
const { id, file, type } = UNWRAP(ref.value);
|
|
return { id: type === "page" ? `page:${id}` : id, file };
|
|
}),
|
|
viewRefs: viewsAndDynPages,
|
|
});
|
|
|
|
// -- assemble page assets --
|
|
const pAssemblePages = builtPages.map((page) =>
|
|
assembleAndWritePage(page, styleMap, scriptMap)
|
|
);
|
|
|
|
await Promise.all([builtBackend, builtStaticFiles, ...pAssemblePages]);
|
|
}
|
|
|
|
export async function readManifest(io: Io) {
|
|
const cfg = await io.import<typeof import("../src/site.ts")>("src/site.ts");
|
|
return {
|
|
siteSections: cfg.siteSections.map((section) => ({
|
|
root: toRel(section.root),
|
|
})),
|
|
backends: cfg.backends.map(toRel),
|
|
};
|
|
}
|
|
|
|
export async function discoverAllFiles(
|
|
siteConfig: Awaited<ReturnType<typeof readManifest>>,
|
|
) {
|
|
return (
|
|
await Promise.all(
|
|
siteConfig.siteSections.map(({ root: sectionRoot }) =>
|
|
incr.work(scanSiteSection, toAbs(sectionRoot))
|
|
),
|
|
)
|
|
).reduce((acc, next) => ({
|
|
staticFiles: acc.staticFiles.concat(next.staticFiles),
|
|
pages: acc.pages.concat(next.pages),
|
|
views: acc.views.concat(next.views),
|
|
scripts: acc.scripts.concat(next.scripts),
|
|
}));
|
|
}
|
|
|
|
export async function scanSiteSection(io: Io, sectionRoot: string) {
|
|
// Static files are compressed and served as-is.
|
|
// - "{section}/static/*.png"
|
|
let staticFiles: FileItem[] = [];
|
|
// Pages are rendered then served as static files.
|
|
// - "{section}/pages/*.marko"
|
|
let pages: FileItem[] = [];
|
|
// Views are dynamically rendered pages called via backend code.
|
|
// - "{section}/views/*.tsx"
|
|
let views: FileItem[] = [];
|
|
// Public scripts are bundled for the client as static assets under "/js/[...]"
|
|
// This is used for the file viewer's canvases.
|
|
// Note that '.client.ts' can be placed anywhere in the file structure.
|
|
// - "{section}/scripts/*.client.ts"
|
|
let scripts: FileItem[] = [];
|
|
|
|
const sectionPath = (...sub: string[]) => path.join(sectionRoot, ...sub);
|
|
const rootPrefix = hot.projectSrc === sectionRoot
|
|
? ""
|
|
: path.relative(hot.projectSrc, sectionRoot) + "/";
|
|
const kinds = [
|
|
{
|
|
dir: sectionPath("pages"),
|
|
list: pages,
|
|
prefix: "/",
|
|
include: [".tsx", ".mdx", ".marko"],
|
|
exclude: [".client.ts", ".client.tsx"],
|
|
},
|
|
{
|
|
dir: sectionPath("static"),
|
|
list: staticFiles,
|
|
prefix: "/",
|
|
ext: true,
|
|
},
|
|
{
|
|
dir: sectionPath("scripts"),
|
|
list: scripts,
|
|
prefix: rootPrefix,
|
|
include: [".client.ts", ".client.tsx"],
|
|
},
|
|
{
|
|
dir: sectionPath("views"),
|
|
list: views,
|
|
prefix: rootPrefix,
|
|
include: [".tsx", ".mdx", ".marko"],
|
|
exclude: [".client.ts", ".client.tsx"],
|
|
},
|
|
];
|
|
for (const kind of kinds) {
|
|
const {
|
|
dir,
|
|
list,
|
|
prefix,
|
|
include = [""],
|
|
exclude = [],
|
|
ext = false,
|
|
} = kind;
|
|
|
|
let items;
|
|
try {
|
|
items = await io.readDirRecursive(dir);
|
|
} catch (err: any) {
|
|
if (err.code === "ENOENT") continue;
|
|
throw err;
|
|
}
|
|
for (const subPath of items) {
|
|
const file = path.join(dir, subPath);
|
|
const stat = fs.statSync(file);
|
|
if (stat.isDirectory()) continue;
|
|
if (!include.some((e) => subPath.endsWith(e))) continue;
|
|
if (exclude.some((e) => subPath.endsWith(e))) continue;
|
|
const trim = ext
|
|
? subPath
|
|
: subPath.slice(0, -path.extname(subPath).length).replaceAll(".", "/");
|
|
let id = prefix + trim.replaceAll("\\", "/");
|
|
if (prefix === "/" && id.endsWith("/index")) {
|
|
id = id.slice(0, -"/index".length) || "/";
|
|
}
|
|
list.push({ id, file: path.relative(hot.projectRoot, file) });
|
|
}
|
|
}
|
|
|
|
return { staticFiles, pages, views, scripts };
|
|
}
|
|
|
|
export async function preparePage(io: Io, item: sg.FileItem) {
|
|
// -- load and validate module --
|
|
let {
|
|
default: Page,
|
|
meta: metadata,
|
|
theme: pageTheme,
|
|
layout,
|
|
regenerate,
|
|
} = await io.import<sg.PageExports>(item.file);
|
|
if (!Page) throw new Error("Page is missing a 'default' export.");
|
|
if (!metadata) throw new Error("Page is missing 'meta' export with a title.");
|
|
|
|
// -- css --
|
|
if (layout?.theme) pageTheme = layout.theme;
|
|
const theme: css.Theme = {
|
|
...css.defaultTheme,
|
|
...pageTheme,
|
|
};
|
|
const cssImports = Array.from(
|
|
new Set([globalCssPath, ...hot.getCssImports(item.file)]),
|
|
(file) => path.relative(hot.projectSrc, file),
|
|
);
|
|
|
|
// -- metadata --
|
|
const renderedMetaPromise = Promise.resolve(
|
|
typeof metadata === "function" ? metadata({ ssr: true }) : metadata,
|
|
).then((m) => meta.renderMeta(m));
|
|
|
|
// -- html --
|
|
let page = render.element(Page);
|
|
if (layout?.default) {
|
|
page = render.element(layout.default, { children: page });
|
|
}
|
|
const bodyPromise = render.async(page, {
|
|
[sg.userData.key]: sg.initRender(),
|
|
});
|
|
|
|
const [{ text, addon }, renderedMeta] = await Promise.all([
|
|
bodyPromise,
|
|
renderedMetaPromise,
|
|
]);
|
|
if (!renderedMeta.includes("<title>")) {
|
|
throw new Error(
|
|
"Page is missing 'meta.title'. " + "All pages need a title tag.",
|
|
);
|
|
}
|
|
|
|
const styleKey = css.styleKey(cssImports, theme);
|
|
return {
|
|
type: "page",
|
|
id: item.id,
|
|
file: item.file,
|
|
regenerate,
|
|
html: text,
|
|
meta: renderedMeta,
|
|
cssImports,
|
|
theme: theme ?? null,
|
|
styleKey,
|
|
clientRefs: Array.from(addon[sg.userData.key].scripts),
|
|
} as const;
|
|
}
|
|
|
|
export async function prepareView(io: Io, item: sg.FileItem) {
|
|
const module = await io.import<any>(item.file);
|
|
if (!module.meta) throw new Error(`View is missing 'export const meta'`);
|
|
if (!module.default) throw new Error(`View is missing a default export.`);
|
|
const pageTheme = module.layout?.theme ?? module.theme;
|
|
const theme: css.Theme = { ...css.defaultTheme, ...pageTheme };
|
|
const cssImports = Array.from(
|
|
new Set([globalCssPath, ...hot.getCssImports(item.file)]),
|
|
(file) => path.relative(hot.projectSrc, file),
|
|
);
|
|
const styleKey = css.styleKey(cssImports, theme);
|
|
return {
|
|
type: "view",
|
|
id: item.id,
|
|
file: item.file,
|
|
cssImports,
|
|
theme,
|
|
clientRefs: hot.getClientScriptRefs(item.file),
|
|
hasLayout: !!module.layout?.default,
|
|
styleKey,
|
|
} as const;
|
|
}
|
|
export type PreparedView = Awaited<ReturnType<typeof prepareView>>;
|
|
|
|
export function prepareInlineCss(
|
|
items: Array<{ styleKey: string; cssImports: string[]; theme: css.Theme }>,
|
|
) {
|
|
const map = new Map<string, incr.Ref<string>>();
|
|
for (const { styleKey, cssImports, theme } of items) {
|
|
if (map.has(styleKey)) continue;
|
|
map.set(
|
|
styleKey,
|
|
incr.work(css.bundleCssFiles, { cssImports, theme, dev: false }),
|
|
);
|
|
}
|
|
return map;
|
|
}
|
|
|
|
export type PreparedPage = Awaited<ReturnType<typeof preparePage>>;
|
|
export async function assembleAndWritePage(
|
|
pageWork: incr.Ref<PreparedPage>,
|
|
styleMap: Map<string, incr.Ref<string>>,
|
|
scriptWork: incr.Ref<Record<string, string>>,
|
|
) {
|
|
const page = await pageWork;
|
|
return incr.work(
|
|
async (io, { id, html, meta, styleKey, clientRefs, regenerate }) => {
|
|
const inlineCss = await io.readWork(UNWRAP(styleMap.get(styleKey)));
|
|
|
|
const scriptIds = clientRefs.map(hot.getScriptId);
|
|
const scriptMap = await io.readWork(scriptWork);
|
|
const scripts = scriptIds
|
|
.map((ref) => UNWRAP(scriptMap[ref], `Missing script ${ref}`))
|
|
.map((x) => `{${x}}`)
|
|
.join("\n");
|
|
|
|
const buffer = sg.wrapDocument({
|
|
body: html,
|
|
head: meta,
|
|
inlineCss,
|
|
scripts,
|
|
});
|
|
await io.writeAsset({
|
|
pathname: id,
|
|
buffer,
|
|
headers: {
|
|
"Content-Type": "text/html",
|
|
},
|
|
regenerative: !!regenerate,
|
|
});
|
|
},
|
|
page,
|
|
);
|
|
}
|
|
|
|
export type PageOrView = PreparedPage | PreparedView;
|
|
|
|
import * as sg from "#sitegen";
|
|
import * as incr from "./incremental.ts";
|
|
import { Io } from "./incremental.ts";
|
|
import * as bundle from "./bundle.ts";
|
|
import * as css from "./css.ts";
|
|
import * as render from "#engine/render";
|
|
import * as hot from "./hot.ts";
|
|
import * as fs from "#sitegen/fs";
|
|
import type { FileItem } from "#sitegen";
|
|
import * as path from "node:path";
|
|
import * as meta from "#sitegen/meta";
|