Stefano Fiorini
349 lines
14 KiB
JavaScript
349 lines
14 KiB
JavaScript
/**
|
||
* Unit tests for verify-generated.mjs — RED phase of TDD.
|
||
*
|
||
* Key contract from acceptance criteria:
|
||
* - A stray file under skills/<skill>/_source/ or skills/<skill>/shared/
|
||
* does NOT cause verify:generated to flag it as stale.
|
||
* - A stray file under skills/<skill>/<agent>/ (other than
|
||
* .generated-manifest.json) DOES cause verify:generated to flag it.
|
||
*/
|
||
|
||
import assert from "node:assert/strict";
|
||
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
|
||
import { tmpdir } from "node:os";
|
||
import path from "node:path";
|
||
import test from "node:test";
|
||
import { fileURLToPath } from "node:url";
|
||
|
||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||
const SCRIPTS_DIR = path.resolve(__dirname, "..");
|
||
|
||
const { verifyGenerated } = await import(`${SCRIPTS_DIR}/verify-generated.mjs`);
|
||
|
||
// ── Stray-file detection boundary tests ──────────────────────────────────
|
||
|
||
test("verifyGenerated: stray file in _source/ is NOT flagged as stale", async () => {
|
||
// This uses the real repo's canonical source - if stray file under _source
|
||
// is added, verify-generated should not complain about it.
|
||
|
||
// We test this by verifying the function signature accepts a repoRoot and
|
||
// generatedRoots override, and that the verifier only walks declared roots.
|
||
// (Full integration test against the real repo runs in pnpm run verify:generated)
|
||
|
||
const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-source-"));
|
||
try {
|
||
// Create a fake skill structure with _source/ and a generated root
|
||
const skillName = "test-skill";
|
||
const agentName = "claude-code";
|
||
|
||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||
const sharedDir = path.join(dir, "skills", skillName, "shared");
|
||
|
||
await mkdir(sourceDir, { recursive: true });
|
||
await mkdir(sharedDir, { recursive: true });
|
||
await mkdir(agentDir, { recursive: true });
|
||
|
||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||
|
||
// Add a STRAY file in _source/ — this should NOT trigger stale detection
|
||
await writeFile(path.join(sourceDir, "STRAY.md"), "stray content");
|
||
|
||
// Add a STRAY file in shared/ — this should NOT trigger stale detection
|
||
await writeFile(path.join(sharedDir, "shared-stray.txt"), "shared stray");
|
||
|
||
// The generated root is EMPTY (no manifest, no files) — but we're testing
|
||
// that _source/ and shared/ stray files don't appear in stale detection.
|
||
// We can test this indirectly: verifyGenerated with no declared roots for
|
||
// this dir returns no stale errors about _source/ or shared/.
|
||
const result = await verifyGenerated(dir, {
|
||
// Override generated roots to be empty for this minimal test
|
||
generatedRootsOverride: [],
|
||
});
|
||
|
||
// No stale errors from _source/ or shared/
|
||
const sourceErrors = result.errors.filter(
|
||
(e) => e.includes("_source") || e.includes("shared"),
|
||
);
|
||
assert.equal(sourceErrors.length, 0, `unexpected stale errors from _source/shared: ${JSON.stringify(sourceErrors)}`);
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test("verifyGenerated: stray file in agent dir IS flagged as stale", async () => {
|
||
const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-agent-"));
|
||
try {
|
||
const skillName = "test-skill";
|
||
const agentName = "claude-code";
|
||
|
||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||
|
||
await mkdir(sourceDir, { recursive: true });
|
||
await mkdir(agentDir, { recursive: true });
|
||
|
||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||
|
||
// Generate the agent dir with proper content + manifest
|
||
const headerLine =
|
||
`<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
|
||
`Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
|
||
`and run \`pnpm run sync:pi\`. -->`;
|
||
const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
|
||
await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);
|
||
|
||
// Add a STRAY file in the agent dir — this SHOULD be flagged
|
||
await writeFile(path.join(agentDir, "STRAY.md"), "stray content");
|
||
|
||
// Write a manifest that does NOT include STRAY.md
|
||
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
|
||
const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
|
||
// Remove STRAY.md from manifest (simulate pre-stray-add manifest)
|
||
manifest.files = manifest.files.filter((f) => f.path !== "STRAY.md");
|
||
await writeFile(
|
||
path.join(agentDir, ".generated-manifest.json"),
|
||
JSON.stringify(manifest, null, 2) + "\n",
|
||
);
|
||
|
||
const result = await verifyGenerated(dir, {
|
||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||
});
|
||
|
||
assert.equal(result.ok, false, "should fail when stray file present");
|
||
const strayError = result.errors.some((e) => e.includes("STRAY.md"));
|
||
assert.ok(strayError, `STRAY.md should appear in errors: ${JSON.stringify(result.errors)}`);
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test("verifyGenerated: .generated-manifest.json is excluded from stale-file detection", async () => {
|
||
// Even though .generated-manifest.json is in the generated root, it should
|
||
// not be considered a "stale file" just because it's not in the files list
|
||
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-self-"));
|
||
try {
|
||
const skillName = "test-skill";
|
||
const agentName = "pi";
|
||
|
||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||
|
||
await mkdir(sourceDir, { recursive: true });
|
||
await mkdir(agentDir, { recursive: true });
|
||
|
||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||
|
||
const headerLine =
|
||
`<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
|
||
`Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
|
||
`and run \`pnpm run sync:pi\`. -->`;
|
||
const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
|
||
await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);
|
||
|
||
// Write manifest (will include SKILL.md, not itself)
|
||
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
|
||
const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
|
||
await writeFile(
|
||
path.join(agentDir, ".generated-manifest.json"),
|
||
JSON.stringify(manifest, null, 2) + "\n",
|
||
);
|
||
|
||
const result = await verifyGenerated(dir, {
|
||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||
});
|
||
|
||
// Should pass — .generated-manifest.json is excluded from stale detection
|
||
const manifestErrors = result.errors.filter((e) => e.includes(".generated-manifest.json"));
|
||
assert.equal(manifestErrors.length, 0, `manifest file should not appear as stale: ${JSON.stringify(manifestErrors)}`);
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test("verifyGenerated: missing file from manifest is flagged as deleted", async () => {
|
||
const dir = await mkdtemp(path.join(tmpdir(), "vg-missing-file-"));
|
||
try {
|
||
const skillName = "test-skill";
|
||
const agentName = "cursor";
|
||
|
||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||
|
||
await mkdir(sourceDir, { recursive: true });
|
||
await mkdir(agentDir, { recursive: true });
|
||
|
||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||
await writeFile(path.join(agentDir, "SKILL.md"), "generated content\n");
|
||
|
||
// Manifest claims templates/plan.md exists, but the file doesn't
|
||
const manifest = {
|
||
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
|
||
generator: "scripts/generate-skills.mjs",
|
||
generatedRoot: `skills/${skillName}/${agentName}`,
|
||
files: [
|
||
{ path: "SKILL.md", kind: "file", mode: "644", sha256: "aaa" },
|
||
{ path: "templates/plan.md", kind: "file", mode: "644", sha256: "bbb" },
|
||
],
|
||
};
|
||
await writeFile(
|
||
path.join(agentDir, ".generated-manifest.json"),
|
||
JSON.stringify(manifest, null, 2) + "\n",
|
||
);
|
||
|
||
const result = await verifyGenerated(dir, {
|
||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||
});
|
||
|
||
assert.equal(result.ok, false, "should fail on missing file");
|
||
const missingError = result.errors.some((e) => e.includes("templates/plan.md"));
|
||
assert.ok(missingError, `missing file should appear in errors: ${JSON.stringify(result.errors)}`);
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test("verifyGenerated: content mismatch is flagged", async () => {
|
||
const dir = await mkdtemp(path.join(tmpdir(), "vg-content-mismatch-"));
|
||
try {
|
||
const skillName = "test-skill";
|
||
const agentName = "opencode";
|
||
|
||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||
|
||
await mkdir(sourceDir, { recursive: true });
|
||
await mkdir(agentDir, { recursive: true });
|
||
|
||
await writeFile(path.join(sourceDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Original\n");
|
||
// Agent dir has DIFFERENT content than what manifest says
|
||
await writeFile(path.join(agentDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Modified!\n");
|
||
|
||
const manifest = {
|
||
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
|
||
generator: "scripts/generate-skills.mjs",
|
||
generatedRoot: `skills/${skillName}/${agentName}`,
|
||
files: [
|
||
{
|
||
path: "SKILL.md",
|
||
kind: "file",
|
||
mode: "644",
|
||
// SHA of the ORIGINAL content (not what's on disk)
|
||
sha256: "0000000000000000000000000000000000000000000000000000000000000000",
|
||
},
|
||
],
|
||
};
|
||
await writeFile(
|
||
path.join(agentDir, ".generated-manifest.json"),
|
||
JSON.stringify(manifest, null, 2) + "\n",
|
||
);
|
||
|
||
const result = await verifyGenerated(dir, {
|
||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||
});
|
||
|
||
assert.equal(result.ok, false, "should fail on content mismatch");
|
||
const mismatchError = result.errors.some((e) => e.includes("SKILL.md"));
|
||
assert.ok(mismatchError, `content mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test("verifyGenerated: manifest entry with wrong sha256 is flagged even when paths match", async () => {
|
||
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-sha-"));
|
||
try {
|
||
const skillName = "test-skill";
|
||
const agentName = "codex";
|
||
|
||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||
|
||
await mkdir(sourceDir, { recursive: true });
|
||
await mkdir(agentDir, { recursive: true });
|
||
|
||
const content = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||
await writeFile(path.join(sourceDir, "SKILL.md"), content);
|
||
await writeFile(path.join(agentDir, "SKILL.md"), content);
|
||
|
||
// Manifest has correct path but deliberately wrong sha256 (simulates corrupted metadata)
|
||
const manifest = {
|
||
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
|
||
generator: "scripts/generate-skills.mjs",
|
||
generatedRoot: `skills/${skillName}/${agentName}`,
|
||
files: [
|
||
{
|
||
path: "SKILL.md",
|
||
kind: "file",
|
||
mode: "644",
|
||
sha256: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
||
},
|
||
],
|
||
};
|
||
await writeFile(
|
||
path.join(agentDir, ".generated-manifest.json"),
|
||
JSON.stringify(manifest, null, 2) + "\n",
|
||
);
|
||
|
||
const result = await verifyGenerated(dir, {
|
||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||
});
|
||
|
||
// Even though file paths match, the sha256 mismatch in the manifest should be detected
|
||
assert.equal(result.ok, false, "should fail on sha256 mismatch in manifest");
|
||
const sha256Error = result.errors.some(
|
||
(e) => e.includes("sha256") || e.includes("SKILL.md"),
|
||
);
|
||
assert.ok(sha256Error, `sha256 mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
});
|
||
|
||
test("verifyGenerated: manifest entry with wrong mode is flagged even when paths match", async () => {
|
||
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-mode-"));
|
||
try {
|
||
const skillName = "test-skill";
|
||
const agentName = "cursor";
|
||
|
||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||
await mkdir(agentDir, { recursive: true });
|
||
|
||
const content = "# Test Skill\n";
|
||
await writeFile(path.join(agentDir, "SKILL.md"), content);
|
||
|
||
// Build a correct manifest first (with real sha256)
|
||
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
|
||
const correctManifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
|
||
|
||
// Tamper: change the mode field for the SKILL.md entry
|
||
const tamperedManifest = {
|
||
...correctManifest,
|
||
files: correctManifest.files.map((f) =>
|
||
f.path === "SKILL.md" ? { ...f, mode: "777" } : f,
|
||
),
|
||
};
|
||
await writeFile(
|
||
path.join(agentDir, ".generated-manifest.json"),
|
||
JSON.stringify(tamperedManifest, null, 2) + "\n",
|
||
);
|
||
|
||
const result = await verifyGenerated(dir, {
|
||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||
});
|
||
|
||
// The mode mismatch in the manifest should be detected by diffManifests
|
||
assert.equal(result.ok, false, "should fail on mode mismatch in manifest");
|
||
const modeError = result.errors.some(
|
||
(e) => e.includes("mode") || e.includes("SKILL.md"),
|
||
);
|
||
assert.ok(modeError, `mode mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
|
||
} finally {
|
||
await rm(dir, { recursive: true, force: true });
|
||
}
|
||
});
|