/** * Unit tests for verify-generated.mjs — RED phase of TDD. * * Key contract from acceptance criteria: * - A stray file under skills//_source/ or skills//shared/ * does NOT cause verify:generated to flag it as stale. * - A stray file under skills/// (other than * .generated-manifest.json) DOES cause verify:generated to flag it. */ import assert from "node:assert/strict"; import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import test from "node:test"; import { fileURLToPath } from "node:url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const SCRIPTS_DIR = path.resolve(__dirname, ".."); const { verifyGenerated } = await import(`${SCRIPTS_DIR}/verify-generated.mjs`); // ── Stray-file detection boundary tests ────────────────────────────────── test("verifyGenerated: stray file in _source/ is NOT flagged as stale", async () => { // This uses the real repo's canonical source - if stray file under _source // is added, verify-generated should not complain about it. // We test this by verifying the function signature accepts a repoRoot and // generatedRoots override, and that the verifier only walks declared roots. // (Full integration test against the real repo runs in pnpm run verify:generated) const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-source-")); try { // Create a fake skill structure with _source/ and a generated root const skillName = "test-skill"; const agentName = "claude-code"; const sourceDir = path.join(dir, "skills", skillName, "_source", agentName); const agentDir = path.join(dir, "skills", skillName, agentName); const sharedDir = path.join(dir, "skills", skillName, "shared"); await mkdir(sourceDir, { recursive: true }); await mkdir(sharedDir, { recursive: true }); await mkdir(agentDir, { recursive: true }); const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n"; await writeFile(path.join(sourceDir, "SKILL.md"), skillContent); // Add a STRAY file in _source/ — this should NOT trigger stale detection await writeFile(path.join(sourceDir, "STRAY.md"), "stray content"); // Add a STRAY file in shared/ — this should NOT trigger stale detection await writeFile(path.join(sharedDir, "shared-stray.txt"), "shared stray"); // The generated root is EMPTY (no manifest, no files) — but we're testing // that _source/ and shared/ stray files don't appear in stale detection. // We can test this indirectly: verifyGenerated with no declared roots for // this dir returns no stale errors about _source/ or shared/. const result = await verifyGenerated(dir, { // Override generated roots to be empty for this minimal test generatedRootsOverride: [], }); // No stale errors from _source/ or shared/ const sourceErrors = result.errors.filter( (e) => e.includes("_source") || e.includes("shared"), ); assert.equal(sourceErrors.length, 0, `unexpected stale errors from _source/shared: ${JSON.stringify(sourceErrors)}`); } finally { await rm(dir, { recursive: true, force: true }); } }); test("verifyGenerated: stray file in agent dir IS flagged as stale", async () => { const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-agent-")); try { const skillName = "test-skill"; const agentName = "claude-code"; const sourceDir = path.join(dir, "skills", skillName, "_source", agentName); const agentDir = path.join(dir, "skills", skillName, agentName); await mkdir(sourceDir, { recursive: true }); await mkdir(agentDir, { recursive: true }); const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n"; await writeFile(path.join(sourceDir, "SKILL.md"), skillContent); // Generate the agent dir with proper content + manifest const headerLine = ``; const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`; await writeFile(path.join(agentDir, "SKILL.md"), generatedContent); // Add a STRAY file in the agent dir — this SHOULD be flagged await writeFile(path.join(agentDir, "STRAY.md"), "stray content"); // Write a manifest that does NOT include STRAY.md const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`); const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`); // Remove STRAY.md from manifest (simulate pre-stray-add manifest) manifest.files = manifest.files.filter((f) => f.path !== "STRAY.md"); await writeFile( path.join(agentDir, ".generated-manifest.json"), JSON.stringify(manifest, null, 2) + "\n", ); const result = await verifyGenerated(dir, { generatedRootsOverride: [`skills/${skillName}/${agentName}`], }); assert.equal(result.ok, false, "should fail when stray file present"); const strayError = result.errors.some((e) => e.includes("STRAY.md")); assert.ok(strayError, `STRAY.md should appear in errors: ${JSON.stringify(result.errors)}`); } finally { await rm(dir, { recursive: true, force: true }); } }); test("verifyGenerated: .generated-manifest.json is excluded from stale-file detection", async () => { // Even though .generated-manifest.json is in the generated root, it should // not be considered a "stale file" just because it's not in the files list const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-self-")); try { const skillName = "test-skill"; const agentName = "pi"; const sourceDir = path.join(dir, "skills", skillName, "_source", agentName); const agentDir = path.join(dir, "skills", skillName, agentName); await mkdir(sourceDir, { recursive: true }); await mkdir(agentDir, { recursive: true }); const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n"; await writeFile(path.join(sourceDir, "SKILL.md"), skillContent); const headerLine = ``; const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`; await writeFile(path.join(agentDir, "SKILL.md"), generatedContent); // Write manifest (will include SKILL.md, not itself) const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`); const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`); await writeFile( path.join(agentDir, ".generated-manifest.json"), JSON.stringify(manifest, null, 2) + "\n", ); const result = await verifyGenerated(dir, { generatedRootsOverride: [`skills/${skillName}/${agentName}`], }); // Should pass — .generated-manifest.json is excluded from stale detection const manifestErrors = result.errors.filter((e) => e.includes(".generated-manifest.json")); assert.equal(manifestErrors.length, 0, `manifest file should not appear as stale: ${JSON.stringify(manifestErrors)}`); } finally { await rm(dir, { recursive: true, force: true }); } }); test("verifyGenerated: missing file from manifest is flagged as deleted", async () => { const dir = await mkdtemp(path.join(tmpdir(), "vg-missing-file-")); try { const skillName = "test-skill"; const agentName = "cursor"; const sourceDir = path.join(dir, "skills", skillName, "_source", agentName); const agentDir = path.join(dir, "skills", skillName, agentName); await mkdir(sourceDir, { recursive: true }); await mkdir(agentDir, { recursive: true }); const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n"; await writeFile(path.join(sourceDir, "SKILL.md"), skillContent); await writeFile(path.join(agentDir, "SKILL.md"), "generated content\n"); // Manifest claims templates/plan.md exists, but the file doesn't const manifest = { $schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json", generator: "scripts/generate-skills.mjs", generatedRoot: `skills/${skillName}/${agentName}`, files: [ { path: "SKILL.md", kind: "file", mode: "644", sha256: "aaa" }, { path: "templates/plan.md", kind: "file", mode: "644", sha256: "bbb" }, ], }; await writeFile( path.join(agentDir, ".generated-manifest.json"), JSON.stringify(manifest, null, 2) + "\n", ); const result = await verifyGenerated(dir, { generatedRootsOverride: [`skills/${skillName}/${agentName}`], }); assert.equal(result.ok, false, "should fail on missing file"); const missingError = result.errors.some((e) => e.includes("templates/plan.md")); assert.ok(missingError, `missing file should appear in errors: ${JSON.stringify(result.errors)}`); } finally { await rm(dir, { recursive: true, force: true }); } }); test("verifyGenerated: content mismatch is flagged", async () => { const dir = await mkdtemp(path.join(tmpdir(), "vg-content-mismatch-")); try { const skillName = "test-skill"; const agentName = "opencode"; const sourceDir = path.join(dir, "skills", skillName, "_source", agentName); const agentDir = path.join(dir, "skills", skillName, agentName); await mkdir(sourceDir, { recursive: true }); await mkdir(agentDir, { recursive: true }); await writeFile(path.join(sourceDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Original\n"); // Agent dir has DIFFERENT content than what manifest says await writeFile(path.join(agentDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Modified!\n"); const manifest = { $schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json", generator: "scripts/generate-skills.mjs", generatedRoot: `skills/${skillName}/${agentName}`, files: [ { path: "SKILL.md", kind: "file", mode: "644", // SHA of the ORIGINAL content (not what's on disk) sha256: "0000000000000000000000000000000000000000000000000000000000000000", }, ], }; await writeFile( path.join(agentDir, ".generated-manifest.json"), JSON.stringify(manifest, null, 2) + "\n", ); const result = await verifyGenerated(dir, { generatedRootsOverride: [`skills/${skillName}/${agentName}`], }); assert.equal(result.ok, false, "should fail on content mismatch"); const mismatchError = result.errors.some((e) => e.includes("SKILL.md")); assert.ok(mismatchError, `content mismatch should appear in errors: ${JSON.stringify(result.errors)}`); } finally { await rm(dir, { recursive: true, force: true }); } }); test("verifyGenerated: manifest entry with wrong sha256 is flagged even when paths match", async () => { const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-sha-")); try { const skillName = "test-skill"; const agentName = "codex"; const sourceDir = path.join(dir, "skills", skillName, "_source", agentName); const agentDir = path.join(dir, "skills", skillName, agentName); await mkdir(sourceDir, { recursive: true }); await mkdir(agentDir, { recursive: true }); const content = "---\nname: test-skill\n---\n\n# Test Skill\n"; await writeFile(path.join(sourceDir, "SKILL.md"), content); await writeFile(path.join(agentDir, "SKILL.md"), content); // Manifest has correct path but deliberately wrong sha256 (simulates corrupted metadata) const manifest = { $schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json", generator: "scripts/generate-skills.mjs", generatedRoot: `skills/${skillName}/${agentName}`, files: [ { path: "SKILL.md", kind: "file", mode: "644", sha256: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef", }, ], }; await writeFile( path.join(agentDir, ".generated-manifest.json"), JSON.stringify(manifest, null, 2) + "\n", ); const result = await verifyGenerated(dir, { generatedRootsOverride: [`skills/${skillName}/${agentName}`], }); // Even though file paths match, the sha256 mismatch in the manifest should be detected assert.equal(result.ok, false, "should fail on sha256 mismatch in manifest"); const sha256Error = result.errors.some( (e) => e.includes("sha256") || e.includes("SKILL.md"), ); assert.ok(sha256Error, `sha256 mismatch should appear in errors: ${JSON.stringify(result.errors)}`); } finally { await rm(dir, { recursive: true, force: true }); } }); test("verifyGenerated: manifest entry with wrong mode is flagged even when paths match", async () => { const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-mode-")); try { const skillName = "test-skill"; const agentName = "cursor"; const agentDir = path.join(dir, "skills", skillName, agentName); await mkdir(agentDir, { recursive: true }); const content = "# Test Skill\n"; await writeFile(path.join(agentDir, "SKILL.md"), content); // Build a correct manifest first (with real sha256) const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`); const correctManifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`); // Tamper: change the mode field for the SKILL.md entry const tamperedManifest = { ...correctManifest, files: correctManifest.files.map((f) => f.path === "SKILL.md" ? { ...f, mode: "777" } : f, ), }; await writeFile( path.join(agentDir, ".generated-manifest.json"), JSON.stringify(tamperedManifest, null, 2) + "\n", ); const result = await verifyGenerated(dir, { generatedRootsOverride: [`skills/${skillName}/${agentName}`], }); // The mode mismatch in the manifest should be detected by diffManifests assert.equal(result.ok, false, "should fail on mode mismatch in manifest"); const modeError = result.errors.some( (e) => e.includes("mode") || e.includes("SKILL.md"), ); assert.ok(modeError, `mode mismatch should appear in errors: ${JSON.stringify(result.errors)}`); } finally { await rm(dir, { recursive: true, force: true }); } });