feat(M3): Shared-source generator for agent variants
This commit is contained in:
Stefano Fiorini
@@ -0,0 +1,348 @@
|
||||
/**
|
||||
* Unit tests for verify-generated.mjs — RED phase of TDD.
|
||||
*
|
||||
* Key contract from acceptance criteria:
|
||||
* - A stray file under skills/<skill>/_source/ or skills/<skill>/shared/
|
||||
* does NOT cause verify:generated to flag it as stale.
|
||||
* - A stray file under skills/<skill>/<agent>/ (other than
|
||||
* .generated-manifest.json) DOES cause verify:generated to flag it.
|
||||
*/
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
|
||||
import { tmpdir } from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const SCRIPTS_DIR = path.resolve(__dirname, "..");
|
||||
|
||||
const { verifyGenerated } = await import(`${SCRIPTS_DIR}/verify-generated.mjs`);
|
||||
|
||||
// ── Stray-file detection boundary tests ──────────────────────────────────
|
||||
|
||||
test("verifyGenerated: stray file in _source/ is NOT flagged as stale", async () => {
|
||||
// This uses the real repo's canonical source - if stray file under _source
|
||||
// is added, verify-generated should not complain about it.
|
||||
|
||||
// We test this by verifying the function signature accepts a repoRoot and
|
||||
// generatedRoots override, and that the verifier only walks declared roots.
|
||||
// (Full integration test against the real repo runs in pnpm run verify:generated)
|
||||
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-source-"));
|
||||
try {
|
||||
// Create a fake skill structure with _source/ and a generated root
|
||||
const skillName = "test-skill";
|
||||
const agentName = "claude-code";
|
||||
|
||||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||||
const sharedDir = path.join(dir, "skills", skillName, "shared");
|
||||
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
await mkdir(sharedDir, { recursive: true });
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
|
||||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||||
|
||||
// Add a STRAY file in _source/ — this should NOT trigger stale detection
|
||||
await writeFile(path.join(sourceDir, "STRAY.md"), "stray content");
|
||||
|
||||
// Add a STRAY file in shared/ — this should NOT trigger stale detection
|
||||
await writeFile(path.join(sharedDir, "shared-stray.txt"), "shared stray");
|
||||
|
||||
// The generated root is EMPTY (no manifest, no files) — but we're testing
|
||||
// that _source/ and shared/ stray files don't appear in stale detection.
|
||||
// We can test this indirectly: verifyGenerated with no declared roots for
|
||||
// this dir returns no stale errors about _source/ or shared/.
|
||||
const result = await verifyGenerated(dir, {
|
||||
// Override generated roots to be empty for this minimal test
|
||||
generatedRootsOverride: [],
|
||||
});
|
||||
|
||||
// No stale errors from _source/ or shared/
|
||||
const sourceErrors = result.errors.filter(
|
||||
(e) => e.includes("_source") || e.includes("shared"),
|
||||
);
|
||||
assert.equal(sourceErrors.length, 0, `unexpected stale errors from _source/shared: ${JSON.stringify(sourceErrors)}`);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("verifyGenerated: stray file in agent dir IS flagged as stale", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-agent-"));
|
||||
try {
|
||||
const skillName = "test-skill";
|
||||
const agentName = "claude-code";
|
||||
|
||||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||||
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
|
||||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||||
|
||||
// Generate the agent dir with proper content + manifest
|
||||
const headerLine =
|
||||
`<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
|
||||
`Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
|
||||
`and run \`pnpm run sync:pi\`. -->`;
|
||||
const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
|
||||
await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);
|
||||
|
||||
// Add a STRAY file in the agent dir — this SHOULD be flagged
|
||||
await writeFile(path.join(agentDir, "STRAY.md"), "stray content");
|
||||
|
||||
// Write a manifest that does NOT include STRAY.md
|
||||
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
|
||||
const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
|
||||
// Remove STRAY.md from manifest (simulate pre-stray-add manifest)
|
||||
manifest.files = manifest.files.filter((f) => f.path !== "STRAY.md");
|
||||
await writeFile(
|
||||
path.join(agentDir, ".generated-manifest.json"),
|
||||
JSON.stringify(manifest, null, 2) + "\n",
|
||||
);
|
||||
|
||||
const result = await verifyGenerated(dir, {
|
||||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false, "should fail when stray file present");
|
||||
const strayError = result.errors.some((e) => e.includes("STRAY.md"));
|
||||
assert.ok(strayError, `STRAY.md should appear in errors: ${JSON.stringify(result.errors)}`);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("verifyGenerated: .generated-manifest.json is excluded from stale-file detection", async () => {
|
||||
// Even though .generated-manifest.json is in the generated root, it should
|
||||
// not be considered a "stale file" just because it's not in the files list
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-self-"));
|
||||
try {
|
||||
const skillName = "test-skill";
|
||||
const agentName = "pi";
|
||||
|
||||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||||
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
|
||||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||||
|
||||
const headerLine =
|
||||
`<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
|
||||
`Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
|
||||
`and run \`pnpm run sync:pi\`. -->`;
|
||||
const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
|
||||
await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);
|
||||
|
||||
// Write manifest (will include SKILL.md, not itself)
|
||||
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
|
||||
const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
|
||||
await writeFile(
|
||||
path.join(agentDir, ".generated-manifest.json"),
|
||||
JSON.stringify(manifest, null, 2) + "\n",
|
||||
);
|
||||
|
||||
const result = await verifyGenerated(dir, {
|
||||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||||
});
|
||||
|
||||
// Should pass — .generated-manifest.json is excluded from stale detection
|
||||
const manifestErrors = result.errors.filter((e) => e.includes(".generated-manifest.json"));
|
||||
assert.equal(manifestErrors.length, 0, `manifest file should not appear as stale: ${JSON.stringify(manifestErrors)}`);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("verifyGenerated: missing file from manifest is flagged as deleted", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "vg-missing-file-"));
|
||||
try {
|
||||
const skillName = "test-skill";
|
||||
const agentName = "cursor";
|
||||
|
||||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||||
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
|
||||
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||||
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
|
||||
await writeFile(path.join(agentDir, "SKILL.md"), "generated content\n");
|
||||
|
||||
// Manifest claims templates/plan.md exists, but the file doesn't
|
||||
const manifest = {
|
||||
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
|
||||
generator: "scripts/generate-skills.mjs",
|
||||
generatedRoot: `skills/${skillName}/${agentName}`,
|
||||
files: [
|
||||
{ path: "SKILL.md", kind: "file", mode: "644", sha256: "aaa" },
|
||||
{ path: "templates/plan.md", kind: "file", mode: "644", sha256: "bbb" },
|
||||
],
|
||||
};
|
||||
await writeFile(
|
||||
path.join(agentDir, ".generated-manifest.json"),
|
||||
JSON.stringify(manifest, null, 2) + "\n",
|
||||
);
|
||||
|
||||
const result = await verifyGenerated(dir, {
|
||||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false, "should fail on missing file");
|
||||
const missingError = result.errors.some((e) => e.includes("templates/plan.md"));
|
||||
assert.ok(missingError, `missing file should appear in errors: ${JSON.stringify(result.errors)}`);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("verifyGenerated: content mismatch is flagged", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "vg-content-mismatch-"));
|
||||
try {
|
||||
const skillName = "test-skill";
|
||||
const agentName = "opencode";
|
||||
|
||||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||||
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
|
||||
await writeFile(path.join(sourceDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Original\n");
|
||||
// Agent dir has DIFFERENT content than what manifest says
|
||||
await writeFile(path.join(agentDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Modified!\n");
|
||||
|
||||
const manifest = {
|
||||
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
|
||||
generator: "scripts/generate-skills.mjs",
|
||||
generatedRoot: `skills/${skillName}/${agentName}`,
|
||||
files: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
kind: "file",
|
||||
mode: "644",
|
||||
// SHA of the ORIGINAL content (not what's on disk)
|
||||
sha256: "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
},
|
||||
],
|
||||
};
|
||||
await writeFile(
|
||||
path.join(agentDir, ".generated-manifest.json"),
|
||||
JSON.stringify(manifest, null, 2) + "\n",
|
||||
);
|
||||
|
||||
const result = await verifyGenerated(dir, {
|
||||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||||
});
|
||||
|
||||
assert.equal(result.ok, false, "should fail on content mismatch");
|
||||
const mismatchError = result.errors.some((e) => e.includes("SKILL.md"));
|
||||
assert.ok(mismatchError, `content mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("verifyGenerated: manifest entry with wrong sha256 is flagged even when paths match", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-sha-"));
|
||||
try {
|
||||
const skillName = "test-skill";
|
||||
const agentName = "codex";
|
||||
|
||||
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
|
||||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||||
|
||||
await mkdir(sourceDir, { recursive: true });
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
|
||||
const content = "---\nname: test-skill\n---\n\n# Test Skill\n";
|
||||
await writeFile(path.join(sourceDir, "SKILL.md"), content);
|
||||
await writeFile(path.join(agentDir, "SKILL.md"), content);
|
||||
|
||||
// Manifest has correct path but deliberately wrong sha256 (simulates corrupted metadata)
|
||||
const manifest = {
|
||||
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
|
||||
generator: "scripts/generate-skills.mjs",
|
||||
generatedRoot: `skills/${skillName}/${agentName}`,
|
||||
files: [
|
||||
{
|
||||
path: "SKILL.md",
|
||||
kind: "file",
|
||||
mode: "644",
|
||||
sha256: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
|
||||
},
|
||||
],
|
||||
};
|
||||
await writeFile(
|
||||
path.join(agentDir, ".generated-manifest.json"),
|
||||
JSON.stringify(manifest, null, 2) + "\n",
|
||||
);
|
||||
|
||||
const result = await verifyGenerated(dir, {
|
||||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||||
});
|
||||
|
||||
// Even though file paths match, the sha256 mismatch in the manifest should be detected
|
||||
assert.equal(result.ok, false, "should fail on sha256 mismatch in manifest");
|
||||
const sha256Error = result.errors.some(
|
||||
(e) => e.includes("sha256") || e.includes("SKILL.md"),
|
||||
);
|
||||
assert.ok(sha256Error, `sha256 mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("verifyGenerated: manifest entry with wrong mode is flagged even when paths match", async () => {
|
||||
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-mode-"));
|
||||
try {
|
||||
const skillName = "test-skill";
|
||||
const agentName = "cursor";
|
||||
|
||||
const agentDir = path.join(dir, "skills", skillName, agentName);
|
||||
await mkdir(agentDir, { recursive: true });
|
||||
|
||||
const content = "# Test Skill\n";
|
||||
await writeFile(path.join(agentDir, "SKILL.md"), content);
|
||||
|
||||
// Build a correct manifest first (with real sha256)
|
||||
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
|
||||
const correctManifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
|
||||
|
||||
// Tamper: change the mode field for the SKILL.md entry
|
||||
const tamperedManifest = {
|
||||
...correctManifest,
|
||||
files: correctManifest.files.map((f) =>
|
||||
f.path === "SKILL.md" ? { ...f, mode: "777" } : f,
|
||||
),
|
||||
};
|
||||
await writeFile(
|
||||
path.join(agentDir, ".generated-manifest.json"),
|
||||
JSON.stringify(tamperedManifest, null, 2) + "\n",
|
||||
);
|
||||
|
||||
const result = await verifyGenerated(dir, {
|
||||
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
|
||||
});
|
||||
|
||||
// The mode mismatch in the manifest should be detected by diffManifests
|
||||
assert.equal(result.ok, false, "should fail on mode mismatch in manifest");
|
||||
const modeError = result.errors.some(
|
||||
(e) => e.includes("mode") || e.includes("SKILL.md"),
|
||||
);
|
||||
assert.ok(modeError, `mode mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
|
||||
} finally {
|
||||
await rm(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user