ai-coding-skills/scripts/verify-generated.mjs

#!/usr/bin/env node
/**
 * verify-generated.mjs — drift detector for generator-owned files (M3, S-306)
 *
 * Verifies that every declared generated root on disk matches the content that
 * the generator would produce from the current canonical sources.
 *
 * Two verification modes:
 *
 *   Production mode (default, no generatedRootsOverride):
 *     Calls generateSkills() into a temp directory and compares the freshly
 *     generated output file-by-file against the on-disk generated roots.
 *     Detects ALL forms of drift:
 *       (a) Direct edits to generated files
 *       (b) Canonical source changes without running `pnpm run sync:pi`
 *       (c) Stale files added to a generated root
 *       (d) Files deleted from a generated root
 *
 *   Test mode (generatedRootsOverride set):
 *     Uses `.generated-manifest.json` as the oracle (manifest-based checks).
 *     Suitable for unit tests that use artificial generated-root fixtures
 *     without a full canonical source tree.
 *
 * Walk scope: only the declared generated roots returned by generate-skills.mjs.
 *   - `skills/<skill>/_source/` and `skills/<skill>/shared/` are NEVER walked.
 *   - `.generated-manifest.json` is excluded from content comparison.
 *   - `node_modules/` is always excluded.
 *
 * Exit codes:
 *   0 — all generated roots match
 *   1 — one or more mismatches detected
 *
 * Usage:
 *   node scripts/verify-generated.mjs
 *   pnpm run verify:generated
 *
 * Exported:
 *   verifyGenerated(repoRoot, options?) → Promise<{ok: boolean, errors: string[]}>
 */

import crypto from "node:crypto";
import { lstat, mkdtemp, readdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";

import { buildManifest, generateSkills, getGeneratedRoots } from "./generate-skills.mjs";

const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), "..");
const MANIFEST_FILENAME = ".generated-manifest.json";

// ── Helpers ───────────────────────────────────────────────────────────────

async function sha256File(absPath) {
  const buf = await readFile(absPath);
  return crypto.createHash("sha256").update(buf).digest("hex");
}

async function pathExists(p) {
  try {
    await lstat(p);
    return true;
  } catch {
    return false;
  }
}

/**
 * Walk a directory and return all file relative paths (excluding node_modules
 * and the manifest file itself, which is handled separately).
 */
async function walkRootFiles(rootDir) {
  const results = [];
  let entries;
  try {
    entries = await readdir(rootDir, { withFileTypes: true });
  } catch {
    return results;
  }

  for (const entry of entries) {
    if (entry.name === "node_modules") continue;
    if (entry.name === MANIFEST_FILENAME) continue; // manifest handled separately

    const full = path.join(rootDir, entry.name);
    if (entry.isDirectory()) {
      const sub = await walkSubDir(full, entry.name);
      results.push(...sub);
    } else if (entry.isFile()) {
      results.push(entry.name);
    }
  }
  return results;
}

async function walkSubDir(dir, prefix) {
  const results = [];
  let entries;
  try {
    entries = await readdir(dir, { withFileTypes: true });
  } catch {
    return results;
  }
  for (const entry of entries) {
    if (entry.name === "node_modules") continue;
    const relPath = `${prefix}/${entry.name}`;
    const full = path.join(dir, entry.name);
    if (entry.isDirectory()) {
      const sub = await walkSubDir(full, relPath);
      results.push(...sub);
    } else if (entry.isFile()) {
      results.push(relPath);
    }
  }
  return results;
}

/**
 * Load and parse the .generated-manifest.json from a generated root.
 * Returns null if missing or malformed.
 */
async function loadManifest(rootDir) {
  const manifestPath = path.join(rootDir, MANIFEST_FILENAME);
  try {
    const raw = await readFile(manifestPath, "utf8");
    const obj = JSON.parse(raw);
    // Structural validation: must have $schema and generator markers
    if (!obj.$schema || !obj.generator || !Array.isArray(obj.files)) {
      return null;
    }
    return obj;
  } catch {
    return null;
  }
}

/**
 * Verify structural equality of two manifests (ignoring ephemeral fields).
 * Returns array of difference descriptions, or empty array if equal.
 */
function diffManifests(expected, actual, rootRel) {
  const diffs = [];

  if (expected.$schema !== actual.$schema) {
    diffs.push(`${rootRel}/.generated-manifest.json: $schema mismatch`);
  }
  if (expected.generator !== actual.generator) {
    diffs.push(`${rootRel}/.generated-manifest.json: generator mismatch`);
  }
  if (expected.generatedRoot !== actual.generatedRoot) {
    diffs.push(
      `${rootRel}/.generated-manifest.json: generatedRoot mismatch ` +
        `(expected ${expected.generatedRoot}, got ${actual.generatedRoot})`,
    );
  }

  // Compare files arrays structurally — paths, kind, mode, and sha256
  const expByPath = new Map(expected.files.map((f) => [f.path, f]));
  const actByPath = new Map(actual.files.map((f) => [f.path, f]));

  for (const p of expByPath.keys()) {
    if (!actByPath.has(p)) {
      diffs.push(`${rootRel}/.generated-manifest.json: expected file entry missing: ${p}`);
    }
  }
  for (const p of actByPath.keys()) {
    if (!expByPath.has(p)) {
      diffs.push(`${rootRel}/.generated-manifest.json: unexpected file entry: ${p}`);
    }
  }

  // For entries present in both, compare kind, mode, and sha256
  for (const [p, expEntry] of expByPath) {
    const actEntry = actByPath.get(p);
    if (!actEntry) continue; // missing already reported above

    if (expEntry.kind !== actEntry.kind) {
      diffs.push(
        `${rootRel}/.generated-manifest.json: ${p}: kind mismatch ` +
          `(expected ${expEntry.kind}, got ${actEntry.kind})`,
      );
    }
    if (expEntry.mode !== actEntry.mode) {
      diffs.push(
        `${rootRel}/.generated-manifest.json: ${p}: mode mismatch ` +
          `(expected ${expEntry.mode}, got ${actEntry.mode})`,
      );
    }
    if (expEntry.sha256 !== actEntry.sha256) {
      diffs.push(
        `${rootRel}/.generated-manifest.json: ${p}: sha256 mismatch ` +
          `(expected ${expEntry.sha256.slice(0, 8)}…, got ${actEntry.sha256.slice(0, 8)}…)`,
      );
    }
  }

  return diffs;
}

// ── Core verifier ─────────────────────────────────────────────────────────

/**
 * Verify all declared generated roots against freshly generated output.
 *
 * Calls generateSkills() into a temp directory and compares the resulting
 * files to the on-disk roots.  Detects all drift types:
 *   - Canonical source changed without running `pnpm run sync:pi`
 *   - Generated file edited directly
 *   - Stale file added to generated root
 *   - Generated file deleted from root
 *
 * Does NOT walk _source/, shared/, or node_modules/.
 *
 * @param {string} repoRoot - Absolute path to repo root.
 * @param {string[]} rootsRelative - Generated root paths to verify.
 * @param {string[]} errors - Error array to append to.
 */
async function verifyWithFreshGeneration(repoRoot, rootsRelative, errors) {
  const tmpDir = await mkdtemp(path.join(tmpdir(), "verify-gen-"));
  try {
    await generateSkills(repoRoot, { targetRoot: tmpDir });

    for (const rootRel of rootsRelative) {
      const freshRootAbs = path.join(tmpDir, rootRel);
      const onDiskRootAbs = path.join(repoRoot, rootRel);

      if (!(await pathExists(onDiskRootAbs))) {
        errors.push(`generated root missing: ${rootRel}`);
        continue;
      }

      const freshFiles = await walkRootFiles(freshRootAbs);
      const onDiskFiles = await walkRootFiles(onDiskRootAbs);

      const freshSet = new Set(freshFiles);
      const diskSet = new Set(onDiskFiles);

      // Files the generator produces but are absent on disk
      for (const f of freshFiles) {
        if (!diskSet.has(f)) {
          errors.push(`${rootRel}/${f}: missing (expected per canonical sources — run \`pnpm run sync:pi\`)`);
        }
      }

      // On-disk files the generator would NOT produce (stale)
      for (const f of onDiskFiles) {
        if (!freshSet.has(f)) {
          errors.push(`${rootRel}/${f}: stale (not produced from canonical sources)`);
        }
      }

      // Content comparison for files present in both sets
      for (const f of freshFiles) {
        if (!diskSet.has(f)) continue; // missing already reported
        const freshHash = await sha256File(path.join(freshRootAbs, f));
        const diskHash = await sha256File(path.join(onDiskRootAbs, f));
        if (freshHash !== diskHash) {
          errors.push(
            `${rootRel}/${f}: content drift ` +
              `(on-disk sha256=${diskHash.slice(0, 8)}…, expected=${freshHash.slice(0, 8)}… — run \`pnpm run sync:pi\`)`,
          );
        }
      }

      // Validate .generated-manifest.json on disk matches what the generator produced
      const freshManifestPath = path.join(freshRootAbs, MANIFEST_FILENAME);
      const diskManifestPath = path.join(onDiskRootAbs, MANIFEST_FILENAME);
      if (await pathExists(freshManifestPath)) {
        if (!(await pathExists(diskManifestPath))) {
          errors.push(`${rootRel}/${MANIFEST_FILENAME}: missing`);
        } else {
          const freshManifest = await loadManifest(freshRootAbs);
          const diskManifest = await loadManifest(onDiskRootAbs);
          if (!diskManifest) {
            errors.push(`${rootRel}/${MANIFEST_FILENAME}: missing or malformed`);
          } else if (freshManifest) {
            const manifestDiffs = diffManifests(freshManifest, diskManifest, rootRel);
            errors.push(...manifestDiffs);
          }
        }
      }
    }
  } finally {
    await rm(tmpDir, { recursive: true, force: true });
  }
}

/**
 * Verify using .generated-manifest.json as the oracle (test / no-canonical-sources mode).
 *
 * Used when generatedRootsOverride is set: the test fixture creates generated
 * roots with manifests but does not provide a full canonical source tree.
 *
 * For each root:
 *   1. Load the on-disk .generated-manifest.json (report missing manifest).
 *   2. For every file listed in the manifest: verify existence, mode, sha256.
 *   3. Walk the root for any files NOT listed in the manifest (stale files).
 *   4. Verify the manifest's own structural fields match expected schema.
 *
 * @param {string} repoRoot - Absolute path to repo root.
 * @param {string[]} rootsRelative - Generated root paths to verify.
 * @param {string[]} errors - Error array to append to.
 */
async function verifyWithManifest(repoRoot, rootsRelative, errors) {
  for (const rootRel of rootsRelative) {
    const rootAbs = path.join(repoRoot, rootRel);

    // Check the root directory exists
    if (!(await pathExists(rootAbs))) {
      errors.push(`generated root missing: ${rootRel}`);
      continue;
    }

    // Load on-disk manifest
    const manifest = await loadManifest(rootAbs);
    if (!manifest) {
      errors.push(`${rootRel}/.generated-manifest.json: missing or malformed`);
      // Can't verify files without a manifest; report all on-disk files as stale
      const onDisk = await walkRootFiles(rootAbs);
      for (const f of onDisk) {
        errors.push(`${rootRel}/${f}: stale (no valid manifest)`);
      }
      continue;
    }

    // Build expected manifest from current disk state (structural check)
    const expectedManifest = await buildManifest(rootAbs, rootRel);
    const manifestDiffs = diffManifests(expectedManifest, manifest, rootRel);
    errors.push(...manifestDiffs);

    // Build maps for file checks
    const manifestByPath = new Map(manifest.files.map((f) => [f.path, f]));

    // Check each file listed in manifest
    for (const entry of manifest.files) {
      const fileAbs = path.join(rootAbs, entry.path);

      if (!(await pathExists(fileAbs))) {
        errors.push(`${rootRel}/${entry.path}: missing (listed in manifest)`);
        continue;
      }

      // Check content hash
      const actualHash = await sha256File(fileAbs);
      if (actualHash !== entry.sha256) {
        errors.push(
          `${rootRel}/${entry.path}: content mismatch ` +
            `(manifest sha256=${entry.sha256.slice(0, 8)}…, actual=${actualHash.slice(0, 8)}…)`,
        );
      }

      // Check mode
      const st = await lstat(fileAbs);
      const actualMode = (st.mode & 0o777).toString(8).padStart(3, "0");
      if (actualMode !== entry.mode) {
        errors.push(
          `${rootRel}/${entry.path}: mode mismatch ` +
            `(manifest=${entry.mode}, actual=${actualMode})`,
        );
      }
    }

    // Detect stale files: on-disk files not listed in manifest
    const onDiskFiles = await walkRootFiles(rootAbs);
    for (const relPath of onDiskFiles) {
      if (!manifestByPath.has(relPath)) {
        errors.push(`${rootRel}/${relPath}: stale (not in manifest)`);
      }
    }
  }
}

/**
 * Verify all declared generated roots in a repo.
 *
 * In production mode (no generatedRootsOverride): generates into a temp
 * directory and compares file-by-file against on-disk roots.  This detects
 * both direct edits to generated files AND changes to canonical sources that
 * were not followed by `pnpm run sync:pi`.
 *
 * In test mode (generatedRootsOverride set): uses manifest-based checks.
 * Suitable for unit tests that provide artificial generated-root fixtures
 * without a full canonical source tree.
 *
 * @param {string} [repoRoot] - Absolute path to repo root.
 * @param {object} [options]
 * @param {string[]} [options.generatedRootsOverride] - Override root list (test mode only).
 * @returns {Promise<{ok: boolean, errors: string[]}>}
 */
export async function verifyGenerated(repoRoot = REPO_ROOT, options = {}) {
  const rootsRelative = options.generatedRootsOverride ?? getGeneratedRoots();
  const errors = [];

  if (options.generatedRootsOverride) {
    // Test mode: no canonical source tree available — use manifest oracle
    await verifyWithManifest(repoRoot, rootsRelative, errors);
  } else {
    // Production mode: generate fresh from canonical sources and compare
    await verifyWithFreshGeneration(repoRoot, rootsRelative, errors);
  }

  return { ok: errors.length === 0, errors };
}

// ── CLI entry point ────────────────────────────────────────────────────────

if (import.meta.url === pathToFileURL(process.argv[1]).href) {
  const result = await verifyGenerated(REPO_ROOT);

  if (result.ok) {
    console.log("verify:generated: all generated roots are up to date.");
    process.exit(0);
  } else {
    console.error("verify:generated: drift detected:\n");
    for (const err of result.errors) {
      console.error(`  ✗ ${err}`);
    }
    console.error(`\n${result.errors.length} error(s). Run \`pnpm run sync:pi\` to regenerate.`);
    process.exit(1);
  }
}