sitegen/src/file-viewer/rsync.ts

185 lines
5.3 KiB
TypeScript
Raw Normal View History

2025-07-08 20:48:30 -07:00
// Utilities for spawning rsync and consuming its output as a `Progress`
// A headless parser is available with `Parse`
export type Line =
| { kind: "ignore" }
| { kind: "log"; level: "info" | "warn" | "error"; message: string }
| { kind: "count"; files: number }
| {
kind: "progress";
currentFile: string;
bytesTransferred: number;
percentage: number;
timeElapsed: string;
transferNumber: number;
filesToCheck: number;
totalFiles: number;
speed: string | null;
};
export const defaultExtraOptions = [
"--progress",
];
export interface SpawnOptions {
args: string[];
title: string;
rsync?: string;
progress?: Progress;
cwd: string;
}
export async function spawn(options: SpawnOptions) {
const { rsync = "rsync", args, title, cwd } = options;
const proc = child_process.spawn(rsync, [...defaultExtraOptions, ...args], {
stdio: ["ignore", "pipe", "pipe"],
cwd,
});
const parser = new Parse();
const bar = options.progress ?? new Progress({ text: title });
let running = true;
const stdoutSplitter = readline.createInterface({ input: proc.stdout });
const stderrSplitter = readline.createInterface({ input: proc.stderr });
const handleLine = (line: string) => {
const result = parser.onLine(line);
if (result.kind === "ignore") {
return;
} else if (result.kind === "log") {
console[result.level](result.message);
} else if (result.kind === "count") {
if (!running) return;
bar.text = `${result.files} files...`;
} else if (result.kind === "progress") {
if (!running) return;
const {
transferNumber,
bytesTransferred,
totalFiles,
filesToCheck,
currentFile,
speed,
} = result;
bar.value = transferNumber;
bar.total = totalFiles;
const extras = [
formatSize(bytesTransferred),
(totalFiles > filesToCheck)
? `${totalFiles - filesToCheck} unchecked`
: null,
speed,
].filter(Boolean).join(", ");
const fileName = currentFile.length > 20
? `${currentFile.slice(0, 3)}..${currentFile.slice(-15)}`
: currentFile;
bar.text = `[${transferNumber}/${totalFiles}] ${fileName} ${
extras.length > 0 ? `(${extras})` : ""
}`;
} else result satisfies never;
};
stdoutSplitter.on("line", handleLine);
stderrSplitter.on("line", handleLine);
const [code, signal] = await events.once(proc, "close");
running = false;
if (code !== 0) {
const fmt = code ? `code ${code}` : `signal ${signal}`;
const e: any = new Error(`rsync failed with ${fmt}`);
e.args = [rsync, ...args].join(" ");
e.code = code;
e.signal = signal;
bar.error(e.message);
return e;
}
bar.success(title);
}
export class Parse {
totalFiles = 0;
currentTransfer = 0;
toCheck = 0;
onLine(line: string): Line {
line = line.trimEnd();
// Parse progress lines like:
// 20c83c16735608fc3de4aac61e36770d7774e0c6/au26.m4s
// 238,377 100% 460.06kB/s 0:00:00 (xfr#557, to-chk=194111/194690)
const progressMatch = line.match(
/^\s+([\d,]+)\s+(\d+)%\s+(\S+)\s+(?:(\S+)\s+)?(?:\(xfr#(\d+), to-chk=(\d+)\/(\d+)\))?/,
);
if (progressMatch) {
const [
,
bytesStr,
percentageStr,
speed,
timeElapsed,
transferStr,
toCheckStr,
totalStr,
] = progressMatch;
this.currentTransfer = Number(transferStr);
return {
kind: "progress",
currentFile: this.lastSeenFile || "",
bytesTransferred: Number(bytesStr.replaceAll(",", "")),
percentage: Number(percentageStr),
timeElapsed,
transferNumber: this.currentTransfer,
filesToCheck: toCheckStr ? this.toCheck = Number(toCheckStr) : this.toCheck,
totalFiles: totalStr ? this.totalFiles = Number(totalStr) : this.totalFiles,
speed: speed || null,
};
}
// Skip common rsync info lines
if (!line.startsWith(" ") && !line.startsWith("rsync")) {
if (
line.startsWith("sending incremental file list") ||
line.startsWith("sent ") ||
line.startsWith("total size is ") ||
line.includes("speedup is ") ||
line.startsWith("building file list")
) {
return { kind: "ignore" };
}
if (line.trim().length > 0) {
this.lastSeenFile = line;
}
return { kind: "ignore" };
}
if (line.startsWith(" ")) {
const match = line.match(/ (\d+) files.../);
if (match) {
return { kind: "count", files: Number(match[1]) };
}
}
if (
line.toLowerCase().includes("error") ||
line.toLowerCase().includes("failed")
) {
return { kind: "log", level: "error", message: line };
}
if (
line.toLowerCase().includes("warning") ||
line.toLowerCase().includes("skipping")
) {
return { kind: "log", level: "warn", message: line };
}
return { kind: "log", level: "info", message: line };
}
private lastSeenFile: string | null = null;
}
import * as child_process from "node:child_process";
import * as readline from "node:readline";
import events from "node:events";
import { Progress } from "@paperclover/console/Progress";
import { formatSize } from "@/file-viewer/format.ts";