#!/usr/bin/env node /** * verify-generated.mjs — drift detector for generator-owned files (M3, S-306) * * Verifies that every declared generated root on disk matches the content that * the generator would produce from the current canonical sources. * * Two verification modes: * * Production mode (default, no generatedRootsOverride): * Calls generateSkills() into a temp directory and compares the freshly * generated output file-by-file against the on-disk generated roots. * Detects ALL forms of drift: * (a) Direct edits to generated files * (b) Canonical source changes without running `pnpm run sync:pi` * (c) Stale files added to a generated root * (d) Files deleted from a generated root * * Test mode (generatedRootsOverride set): * Uses `.generated-manifest.json` as the oracle (manifest-based checks). * Suitable for unit tests that use artificial generated-root fixtures * without a full canonical source tree. * * Walk scope: only the declared generated roots returned by generate-skills.mjs. * - `skills//_source/` and `skills//shared/` are NEVER walked. * - `.generated-manifest.json` is excluded from content comparison. * - `node_modules/` is always excluded. * * Exit codes: * 0 — all generated roots match * 1 — one or more mismatches detected * * Usage: * node scripts/verify-generated.mjs * pnpm run verify:generated * * Exported: * verifyGenerated(repoRoot, options?) → Promise<{ok: boolean, errors: string[]}> */ import crypto from "node:crypto"; import { lstat, mkdtemp, readdir, readFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { pathToFileURL } from "node:url"; import { buildManifest, generateSkills, getGeneratedRoots } from "./generate-skills.mjs"; const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), ".."); const MANIFEST_FILENAME = ".generated-manifest.json"; // ── Helpers ─────────────────────────────────────────────────────────────── async function sha256File(absPath) { const buf = await readFile(absPath); return crypto.createHash("sha256").update(buf).digest("hex"); } async function pathExists(p) { try { await lstat(p); return true; } catch { return false; } } /** * Walk a directory and return all file relative paths (excluding node_modules * and the manifest file itself, which is handled separately). */ async function walkRootFiles(rootDir) { const results = []; let entries; try { entries = await readdir(rootDir, { withFileTypes: true }); } catch { return results; } for (const entry of entries) { if (entry.name === "node_modules") continue; if (entry.name === MANIFEST_FILENAME) continue; // manifest handled separately const full = path.join(rootDir, entry.name); if (entry.isDirectory()) { const sub = await walkSubDir(full, entry.name); results.push(...sub); } else if (entry.isFile()) { results.push(entry.name); } } return results; } async function walkSubDir(dir, prefix) { const results = []; let entries; try { entries = await readdir(dir, { withFileTypes: true }); } catch { return results; } for (const entry of entries) { if (entry.name === "node_modules") continue; const relPath = `${prefix}/${entry.name}`; const full = path.join(dir, entry.name); if (entry.isDirectory()) { const sub = await walkSubDir(full, relPath); results.push(...sub); } else if (entry.isFile()) { results.push(relPath); } } return results; } /** * Load and parse the .generated-manifest.json from a generated root. * Returns null if missing or malformed. */ async function loadManifest(rootDir) { const manifestPath = path.join(rootDir, MANIFEST_FILENAME); try { const raw = await readFile(manifestPath, "utf8"); const obj = JSON.parse(raw); // Structural validation: must have $schema and generator markers if (!obj.$schema || !obj.generator || !Array.isArray(obj.files)) { return null; } return obj; } catch { return null; } } /** * Verify structural equality of two manifests (ignoring ephemeral fields). * Returns array of difference descriptions, or empty array if equal. */ function diffManifests(expected, actual, rootRel) { const diffs = []; if (expected.$schema !== actual.$schema) { diffs.push(`${rootRel}/.generated-manifest.json: $schema mismatch`); } if (expected.generator !== actual.generator) { diffs.push(`${rootRel}/.generated-manifest.json: generator mismatch`); } if (expected.generatedRoot !== actual.generatedRoot) { diffs.push( `${rootRel}/.generated-manifest.json: generatedRoot mismatch ` + `(expected ${expected.generatedRoot}, got ${actual.generatedRoot})`, ); } // Compare files arrays structurally — paths, kind, mode, and sha256 const expByPath = new Map(expected.files.map((f) => [f.path, f])); const actByPath = new Map(actual.files.map((f) => [f.path, f])); for (const p of expByPath.keys()) { if (!actByPath.has(p)) { diffs.push(`${rootRel}/.generated-manifest.json: expected file entry missing: ${p}`); } } for (const p of actByPath.keys()) { if (!expByPath.has(p)) { diffs.push(`${rootRel}/.generated-manifest.json: unexpected file entry: ${p}`); } } // For entries present in both, compare kind, mode, and sha256 for (const [p, expEntry] of expByPath) { const actEntry = actByPath.get(p); if (!actEntry) continue; // missing already reported above if (expEntry.kind !== actEntry.kind) { diffs.push( `${rootRel}/.generated-manifest.json: ${p}: kind mismatch ` + `(expected ${expEntry.kind}, got ${actEntry.kind})`, ); } if (expEntry.mode !== actEntry.mode) { diffs.push( `${rootRel}/.generated-manifest.json: ${p}: mode mismatch ` + `(expected ${expEntry.mode}, got ${actEntry.mode})`, ); } if (expEntry.sha256 !== actEntry.sha256) { diffs.push( `${rootRel}/.generated-manifest.json: ${p}: sha256 mismatch ` + `(expected ${expEntry.sha256.slice(0, 8)}…, got ${actEntry.sha256.slice(0, 8)}…)`, ); } } return diffs; } // ── Core verifier ───────────────────────────────────────────────────────── /** * Verify all declared generated roots against freshly generated output. * * Calls generateSkills() into a temp directory and compares the resulting * files to the on-disk roots. Detects all drift types: * - Canonical source changed without running `pnpm run sync:pi` * - Generated file edited directly * - Stale file added to generated root * - Generated file deleted from root * * Does NOT walk _source/, shared/, or node_modules/. * * @param {string} repoRoot - Absolute path to repo root. * @param {string[]} rootsRelative - Generated root paths to verify. * @param {string[]} errors - Error array to append to. */ async function verifyWithFreshGeneration(repoRoot, rootsRelative, errors) { const tmpDir = await mkdtemp(path.join(tmpdir(), "verify-gen-")); try { await generateSkills(repoRoot, { targetRoot: tmpDir }); for (const rootRel of rootsRelative) { const freshRootAbs = path.join(tmpDir, rootRel); const onDiskRootAbs = path.join(repoRoot, rootRel); if (!(await pathExists(onDiskRootAbs))) { errors.push(`generated root missing: ${rootRel}`); continue; } const freshFiles = await walkRootFiles(freshRootAbs); const onDiskFiles = await walkRootFiles(onDiskRootAbs); const freshSet = new Set(freshFiles); const diskSet = new Set(onDiskFiles); // Files the generator produces but are absent on disk for (const f of freshFiles) { if (!diskSet.has(f)) { errors.push(`${rootRel}/${f}: missing (expected per canonical sources — run \`pnpm run sync:pi\`)`); } } // On-disk files the generator would NOT produce (stale) for (const f of onDiskFiles) { if (!freshSet.has(f)) { errors.push(`${rootRel}/${f}: stale (not produced from canonical sources)`); } } // Content comparison for files present in both sets for (const f of freshFiles) { if (!diskSet.has(f)) continue; // missing already reported const freshHash = await sha256File(path.join(freshRootAbs, f)); const diskHash = await sha256File(path.join(onDiskRootAbs, f)); if (freshHash !== diskHash) { errors.push( `${rootRel}/${f}: content drift ` + `(on-disk sha256=${diskHash.slice(0, 8)}…, expected=${freshHash.slice(0, 8)}… — run \`pnpm run sync:pi\`)`, ); } } // Validate .generated-manifest.json on disk matches what the generator produced const freshManifestPath = path.join(freshRootAbs, MANIFEST_FILENAME); const diskManifestPath = path.join(onDiskRootAbs, MANIFEST_FILENAME); if (await pathExists(freshManifestPath)) { if (!(await pathExists(diskManifestPath))) { errors.push(`${rootRel}/${MANIFEST_FILENAME}: missing`); } else { const freshManifest = await loadManifest(freshRootAbs); const diskManifest = await loadManifest(onDiskRootAbs); if (!diskManifest) { errors.push(`${rootRel}/${MANIFEST_FILENAME}: missing or malformed`); } else if (freshManifest) { const manifestDiffs = diffManifests(freshManifest, diskManifest, rootRel); errors.push(...manifestDiffs); } } } } } finally { await rm(tmpDir, { recursive: true, force: true }); } } /** * Verify using .generated-manifest.json as the oracle (test / no-canonical-sources mode). * * Used when generatedRootsOverride is set: the test fixture creates generated * roots with manifests but does not provide a full canonical source tree. * * For each root: * 1. Load the on-disk .generated-manifest.json (report missing manifest). * 2. For every file listed in the manifest: verify existence, mode, sha256. * 3. Walk the root for any files NOT listed in the manifest (stale files). * 4. Verify the manifest's own structural fields match expected schema. * * @param {string} repoRoot - Absolute path to repo root. * @param {string[]} rootsRelative - Generated root paths to verify. * @param {string[]} errors - Error array to append to. */ async function verifyWithManifest(repoRoot, rootsRelative, errors) { for (const rootRel of rootsRelative) { const rootAbs = path.join(repoRoot, rootRel); // Check the root directory exists if (!(await pathExists(rootAbs))) { errors.push(`generated root missing: ${rootRel}`); continue; } // Load on-disk manifest const manifest = await loadManifest(rootAbs); if (!manifest) { errors.push(`${rootRel}/.generated-manifest.json: missing or malformed`); // Can't verify files without a manifest; report all on-disk files as stale const onDisk = await walkRootFiles(rootAbs); for (const f of onDisk) { errors.push(`${rootRel}/${f}: stale (no valid manifest)`); } continue; } // Build expected manifest from current disk state (structural check) const expectedManifest = await buildManifest(rootAbs, rootRel); const manifestDiffs = diffManifests(expectedManifest, manifest, rootRel); errors.push(...manifestDiffs); // Build maps for file checks const manifestByPath = new Map(manifest.files.map((f) => [f.path, f])); // Check each file listed in manifest for (const entry of manifest.files) { const fileAbs = path.join(rootAbs, entry.path); if (!(await pathExists(fileAbs))) { errors.push(`${rootRel}/${entry.path}: missing (listed in manifest)`); continue; } // Check content hash const actualHash = await sha256File(fileAbs); if (actualHash !== entry.sha256) { errors.push( `${rootRel}/${entry.path}: content mismatch ` + `(manifest sha256=${entry.sha256.slice(0, 8)}…, actual=${actualHash.slice(0, 8)}…)`, ); } // Check mode const st = await lstat(fileAbs); const actualMode = (st.mode & 0o777).toString(8).padStart(3, "0"); if (actualMode !== entry.mode) { errors.push( `${rootRel}/${entry.path}: mode mismatch ` + `(manifest=${entry.mode}, actual=${actualMode})`, ); } } // Detect stale files: on-disk files not listed in manifest const onDiskFiles = await walkRootFiles(rootAbs); for (const relPath of onDiskFiles) { if (!manifestByPath.has(relPath)) { errors.push(`${rootRel}/${relPath}: stale (not in manifest)`); } } } } /** * Verify all declared generated roots in a repo. * * In production mode (no generatedRootsOverride): generates into a temp * directory and compares file-by-file against on-disk roots. This detects * both direct edits to generated files AND changes to canonical sources that * were not followed by `pnpm run sync:pi`. * * In test mode (generatedRootsOverride set): uses manifest-based checks. * Suitable for unit tests that provide artificial generated-root fixtures * without a full canonical source tree. * * @param {string} [repoRoot] - Absolute path to repo root. * @param {object} [options] * @param {string[]} [options.generatedRootsOverride] - Override root list (test mode only). * @returns {Promise<{ok: boolean, errors: string[]}>} */ export async function verifyGenerated(repoRoot = REPO_ROOT, options = {}) { const rootsRelative = options.generatedRootsOverride ?? getGeneratedRoots(); const errors = []; if (options.generatedRootsOverride) { // Test mode: no canonical source tree available — use manifest oracle await verifyWithManifest(repoRoot, rootsRelative, errors); } else { // Production mode: generate fresh from canonical sources and compare await verifyWithFreshGeneration(repoRoot, rootsRelative, errors); } return { ok: errors.length === 0, errors }; } // ── CLI entry point ──────────────────────────────────────────────────────── if (import.meta.url === pathToFileURL(process.argv[1]).href) { const result = await verifyGenerated(REPO_ROOT); if (result.ok) { console.log("verify:generated: all generated roots are up to date."); process.exit(0); } else { console.error("verify:generated: drift detected:\n"); for (const err of result.errors) { console.error(` ✗ ${err}`); } console.error(`\n${result.errors.length} error(s). Run \`pnpm run sync:pi\` to regenerate.`); process.exit(1); } }