sitegen/src/file-viewer/rsync.ts

// Utilities for spawning rsync and consuming its output as a `Progress`
// A headless parser is available with `Parse`

export type Line =
  | { kind: "ignore" }
  | { kind: "log"; level: "info" | "warn" | "error"; message: string }
  | { kind: "count"; files: number }
  | {
    kind: "progress";
    currentFile: string;
    bytesTransferred: number;
    percentage: number;
    timeElapsed: string;
    transferNumber: number;
    filesToCheck: number;
    totalFiles: number;
    speed: string | null;
  };

export const defaultExtraOptions = [
  "--progress",
];

export interface SpawnOptions {
  args: string[];
  title: string;
  rsync?: string;
  progress?: Progress;
  cwd: string;
}

export async function spawn(options: SpawnOptions) {
  const { rsync = "rsync", args, title, cwd } = options;
  const proc = child_process.spawn(rsync, [...defaultExtraOptions, ...args], {
    stdio: ["ignore", "pipe", "pipe"],
    cwd,
  });
  const parser = new Parse();
  const bar = options.progress ?? new Progress({ text: title });
  let running = true;

  const stdoutSplitter = readline.createInterface({ input: proc.stdout });
  const stderrSplitter = readline.createInterface({ input: proc.stderr });

  const handleLine = (line: string) => {
    const result = parser.onLine(line);
    if (result.kind === "ignore") {
      return;
    } else if (result.kind === "log") {
      console[result.level](result.message);
    } else if (result.kind === "count") {
      if (!running) return;
      bar.text = `${result.files} files...`;
    } else if (result.kind === "progress") {
      if (!running) return;
      const {
        transferNumber,
        bytesTransferred,
        totalFiles,
        filesToCheck,
        currentFile,
        speed,
      } = result;
      bar.value = transferNumber;
      bar.total = totalFiles;
      const extras = [
        formatSize(bytesTransferred),
        (totalFiles > filesToCheck)
          ? `${totalFiles - filesToCheck} unchecked`
          : null,
        speed,
      ].filter(Boolean).join(", ");
      const fileName = currentFile.length > 20
        ? `${currentFile.slice(0, 3)}..${currentFile.slice(-15)}`
        : currentFile;
      bar.text = `[${transferNumber}/${totalFiles}] ${fileName} ${
        extras.length > 0 ? `(${extras})` : ""
      }`;
    } else result satisfies never;
  };

  stdoutSplitter.on("line", handleLine);
  stderrSplitter.on("line", handleLine);

  const [code, signal] = await events.once(proc, "close");
  running = false;
  if (code !== 0) {
    const fmt = code ? `code ${code}` : `signal ${signal}`;
    const e: any = new Error(`rsync failed with ${fmt}`);
    e.args = [rsync, ...args].join(" ");
    e.code = code;
    e.signal = signal;
    bar.error(e.message);
    return e;
  }
  bar.success(title);
}

export class Parse {
  totalFiles = 0;
  currentTransfer = 0;
  toCheck = 0;

  onLine(line: string): Line {
    line = line.trimEnd();

    // Parse progress lines like:
    // 20c83c16735608fc3de4aac61e36770d7774e0c6/au26.m4s
    //         238,377 100%  460.06kB/s    0:00:00 (xfr#557, to-chk=194111/194690)
    const progressMatch = line.match(
      /^\s+([\d,]+)\s+(\d+)%\s+(\S+)\s+(?:(\S+)\s+)?(?:\(xfr#(\d+), to-chk=(\d+)\/(\d+)\))?/,
    );
    if (progressMatch) {
      const [
        ,
        bytesStr,
        percentageStr,
        speed,
        timeElapsed,
        transferStr,
        toCheckStr,
        totalStr,
      ] = progressMatch;

      this.currentTransfer = Number(transferStr);

      return {
        kind: "progress",
        currentFile: this.lastSeenFile || "",
        bytesTransferred: Number(bytesStr.replaceAll(",", "")),
        percentage: Number(percentageStr),
        timeElapsed,
        transferNumber: this.currentTransfer,
        filesToCheck: toCheckStr
          ? this.toCheck = Number(toCheckStr)
          : this.toCheck,
        totalFiles: totalStr
          ? this.totalFiles = Number(totalStr)
          : this.totalFiles,
        speed: speed || null,
      };
    }

    // Skip common rsync info lines
    if (!line.startsWith(" ") && !line.startsWith("rsync")) {
      if (
        line.startsWith("sending incremental file list") ||
        line.startsWith("sent ") ||
        line.startsWith("total size is ") ||
        line.includes("speedup is ") ||
        line.startsWith("building file list")
      ) {
        return { kind: "ignore" };
      }
      if (line.trim().length > 0) {
        this.lastSeenFile = line;
      }
      return { kind: "ignore" };
    }
    if (line.startsWith(" ")) {
      const match = line.match(/ (\d+) files.../);
      if (match) {
        return { kind: "count", files: Number(match[1]) };
      }
    }
    if (
      line.toLowerCase().includes("error") ||
      line.toLowerCase().includes("failed")
    ) {
      return { kind: "log", level: "error", message: line };
    }
    if (
      line.toLowerCase().includes("warning") ||
      line.toLowerCase().includes("skipping")
    ) {
      return { kind: "log", level: "warn", message: line };
    }
    return { kind: "log", level: "info", message: line };
  }

  private lastSeenFile: string | null = null;
}

import * as child_process from "node:child_process";
import * as readline from "node:readline";
import events from "node:events";
import { Progress } from "@paperclover/console/Progress";
import { formatSize } from "@/file-viewer/format.ts";