ai-coding-skills/scripts/tests/verify-generated.test.mjs

/**
 * Unit tests for verify-generated.mjs — RED phase of TDD.
 *
 * Key contract from acceptance criteria:
 *   - A stray file under skills/<skill>/_source/ or skills/<skill>/shared/
 *     does NOT cause verify:generated to flag it as stale.
 *   - A stray file under skills/<skill>/<agent>/ (other than
 *     .generated-manifest.json) DOES cause verify:generated to flag it.
 */

import assert from "node:assert/strict";
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";

const __dirname = path.dirname(fileURLToPath(import.meta.url));
const SCRIPTS_DIR = path.resolve(__dirname, "..");

const { verifyGenerated } = await import(`${SCRIPTS_DIR}/verify-generated.mjs`);

// ── Stray-file detection boundary tests ──────────────────────────────────

test("verifyGenerated: stray file in _source/ is NOT flagged as stale", async () => {
  // This uses the real repo's canonical source - if stray file under _source
  // is added, verify-generated should not complain about it.

  // We test this by verifying the function signature accepts a repoRoot and
  // generatedRoots override, and that the verifier only walks declared roots.
  // (Full integration test against the real repo runs in pnpm run verify:generated)

  const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-source-"));
  try {
    // Create a fake skill structure with _source/ and a generated root
    const skillName = "test-skill";
    const agentName = "claude-code";

    const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
    const agentDir = path.join(dir, "skills", skillName, agentName);
    const sharedDir = path.join(dir, "skills", skillName, "shared");

    await mkdir(sourceDir, { recursive: true });
    await mkdir(sharedDir, { recursive: true });
    await mkdir(agentDir, { recursive: true });

    const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
    await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);

    // Add a STRAY file in _source/ — this should NOT trigger stale detection
    await writeFile(path.join(sourceDir, "STRAY.md"), "stray content");

    // Add a STRAY file in shared/ — this should NOT trigger stale detection
    await writeFile(path.join(sharedDir, "shared-stray.txt"), "shared stray");

    // The generated root is EMPTY (no manifest, no files) — but we're testing
    // that _source/ and shared/ stray files don't appear in stale detection.
    // We can test this indirectly: verifyGenerated with no declared roots for
    // this dir returns no stale errors about _source/ or shared/.
    const result = await verifyGenerated(dir, {
      // Override generated roots to be empty for this minimal test
      generatedRootsOverride: [],
    });

    // No stale errors from _source/ or shared/
    const sourceErrors = result.errors.filter(
      (e) => e.includes("_source") || e.includes("shared"),
    );
    assert.equal(sourceErrors.length, 0, `unexpected stale errors from _source/shared: ${JSON.stringify(sourceErrors)}`);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});

test("verifyGenerated: stray file in agent dir IS flagged as stale", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-agent-"));
  try {
    const skillName = "test-skill";
    const agentName = "claude-code";

    const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
    const agentDir = path.join(dir, "skills", skillName, agentName);

    await mkdir(sourceDir, { recursive: true });
    await mkdir(agentDir, { recursive: true });

    const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
    await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);

    // Generate the agent dir with proper content + manifest
    const headerLine =
      `<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
      `Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
      `and run \`pnpm run sync:pi\`. -->`;
    const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
    await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);

    // Add a STRAY file in the agent dir — this SHOULD be flagged
    await writeFile(path.join(agentDir, "STRAY.md"), "stray content");

    // Write a manifest that does NOT include STRAY.md
    const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
    const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
    // Remove STRAY.md from manifest (simulate pre-stray-add manifest)
    manifest.files = manifest.files.filter((f) => f.path !== "STRAY.md");
    await writeFile(
      path.join(agentDir, ".generated-manifest.json"),
      JSON.stringify(manifest, null, 2) + "\n",
    );

    const result = await verifyGenerated(dir, {
      generatedRootsOverride: [`skills/${skillName}/${agentName}`],
    });

    assert.equal(result.ok, false, "should fail when stray file present");
    const strayError = result.errors.some((e) => e.includes("STRAY.md"));
    assert.ok(strayError, `STRAY.md should appear in errors: ${JSON.stringify(result.errors)}`);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});

test("verifyGenerated: .generated-manifest.json is excluded from stale-file detection", async () => {
  // Even though .generated-manifest.json is in the generated root, it should
  // not be considered a "stale file" just because it's not in the files list
  const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-self-"));
  try {
    const skillName = "test-skill";
    const agentName = "pi";

    const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
    const agentDir = path.join(dir, "skills", skillName, agentName);

    await mkdir(sourceDir, { recursive: true });
    await mkdir(agentDir, { recursive: true });

    const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
    await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);

    const headerLine =
      `<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
      `Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
      `and run \`pnpm run sync:pi\`. -->`;
    const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
    await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);

    // Write manifest (will include SKILL.md, not itself)
    const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
    const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
    await writeFile(
      path.join(agentDir, ".generated-manifest.json"),
      JSON.stringify(manifest, null, 2) + "\n",
    );

    const result = await verifyGenerated(dir, {
      generatedRootsOverride: [`skills/${skillName}/${agentName}`],
    });

    // Should pass — .generated-manifest.json is excluded from stale detection
    const manifestErrors = result.errors.filter((e) => e.includes(".generated-manifest.json"));
    assert.equal(manifestErrors.length, 0, `manifest file should not appear as stale: ${JSON.stringify(manifestErrors)}`);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});

test("verifyGenerated: missing file from manifest is flagged as deleted", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "vg-missing-file-"));
  try {
    const skillName = "test-skill";
    const agentName = "cursor";

    const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
    const agentDir = path.join(dir, "skills", skillName, agentName);

    await mkdir(sourceDir, { recursive: true });
    await mkdir(agentDir, { recursive: true });

    const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
    await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
    await writeFile(path.join(agentDir, "SKILL.md"), "generated content\n");

    // Manifest claims templates/plan.md exists, but the file doesn't
    const manifest = {
      $schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
      generator: "scripts/generate-skills.mjs",
      generatedRoot: `skills/${skillName}/${agentName}`,
      files: [
        { path: "SKILL.md", kind: "file", mode: "644", sha256: "aaa" },
        { path: "templates/plan.md", kind: "file", mode: "644", sha256: "bbb" },
      ],
    };
    await writeFile(
      path.join(agentDir, ".generated-manifest.json"),
      JSON.stringify(manifest, null, 2) + "\n",
    );

    const result = await verifyGenerated(dir, {
      generatedRootsOverride: [`skills/${skillName}/${agentName}`],
    });

    assert.equal(result.ok, false, "should fail on missing file");
    const missingError = result.errors.some((e) => e.includes("templates/plan.md"));
    assert.ok(missingError, `missing file should appear in errors: ${JSON.stringify(result.errors)}`);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});

test("verifyGenerated: content mismatch is flagged", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "vg-content-mismatch-"));
  try {
    const skillName = "test-skill";
    const agentName = "opencode";

    const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
    const agentDir = path.join(dir, "skills", skillName, agentName);

    await mkdir(sourceDir, { recursive: true });
    await mkdir(agentDir, { recursive: true });

    await writeFile(path.join(sourceDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Original\n");
    // Agent dir has DIFFERENT content than what manifest says
    await writeFile(path.join(agentDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Modified!\n");

    const manifest = {
      $schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
      generator: "scripts/generate-skills.mjs",
      generatedRoot: `skills/${skillName}/${agentName}`,
      files: [
        {
          path: "SKILL.md",
          kind: "file",
          mode: "644",
          // SHA of the ORIGINAL content (not what's on disk)
          sha256: "0000000000000000000000000000000000000000000000000000000000000000",
        },
      ],
    };
    await writeFile(
      path.join(agentDir, ".generated-manifest.json"),
      JSON.stringify(manifest, null, 2) + "\n",
    );

    const result = await verifyGenerated(dir, {
      generatedRootsOverride: [`skills/${skillName}/${agentName}`],
    });

    assert.equal(result.ok, false, "should fail on content mismatch");
    const mismatchError = result.errors.some((e) => e.includes("SKILL.md"));
    assert.ok(mismatchError, `content mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});

test("verifyGenerated: manifest entry with wrong sha256 is flagged even when paths match", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-sha-"));
  try {
    const skillName = "test-skill";
    const agentName = "codex";

    const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
    const agentDir = path.join(dir, "skills", skillName, agentName);

    await mkdir(sourceDir, { recursive: true });
    await mkdir(agentDir, { recursive: true });

    const content = "---\nname: test-skill\n---\n\n# Test Skill\n";
    await writeFile(path.join(sourceDir, "SKILL.md"), content);
    await writeFile(path.join(agentDir, "SKILL.md"), content);

    // Manifest has correct path but deliberately wrong sha256 (simulates corrupted metadata)
    const manifest = {
      $schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
      generator: "scripts/generate-skills.mjs",
      generatedRoot: `skills/${skillName}/${agentName}`,
      files: [
        {
          path: "SKILL.md",
          kind: "file",
          mode: "644",
          sha256: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
        },
      ],
    };
    await writeFile(
      path.join(agentDir, ".generated-manifest.json"),
      JSON.stringify(manifest, null, 2) + "\n",
    );

    const result = await verifyGenerated(dir, {
      generatedRootsOverride: [`skills/${skillName}/${agentName}`],
    });

    // Even though file paths match, the sha256 mismatch in the manifest should be detected
    assert.equal(result.ok, false, "should fail on sha256 mismatch in manifest");
    const sha256Error = result.errors.some(
      (e) => e.includes("sha256") || e.includes("SKILL.md"),
    );
    assert.ok(sha256Error, `sha256 mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});

test("verifyGenerated: manifest entry with wrong mode is flagged even when paths match", async () => {
  const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-mode-"));
  try {
    const skillName = "test-skill";
    const agentName = "cursor";

    const agentDir = path.join(dir, "skills", skillName, agentName);
    await mkdir(agentDir, { recursive: true });

    const content = "# Test Skill\n";
    await writeFile(path.join(agentDir, "SKILL.md"), content);

    // Build a correct manifest first (with real sha256)
    const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
    const correctManifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);

    // Tamper: change the mode field for the SKILL.md entry
    const tamperedManifest = {
      ...correctManifest,
      files: correctManifest.files.map((f) =>
        f.path === "SKILL.md" ? { ...f, mode: "777" } : f,
      ),
    };
    await writeFile(
      path.join(agentDir, ".generated-manifest.json"),
      JSON.stringify(tamperedManifest, null, 2) + "\n",
    );

    const result = await verifyGenerated(dir, {
      generatedRootsOverride: [`skills/${skillName}/${agentName}`],
    });

    // The mode mismatch in the manifest should be detected by diffManifests
    assert.equal(result.ok, false, "should fail on mode mismatch in manifest");
    const modeError = result.errors.some(
      (e) => e.includes("mode") || e.includes("SKILL.md"),
    );
    assert.ok(modeError, `mode mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
  } finally {
    await rm(dir, { recursive: true, force: true });
  }
});