Home/ai-coding-skills
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(M3): Shared-source generator for agent variants

Browse Source
This commit is contained in:
Stefano Fiorini
2026-05-03 21:09:22 -05:00
parent be993429c1
commit 86ad783f82
339 changed files with 20650 additions and 145 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/generate-skills.mjs
+600
View File
@@ -0,0 +1,600 @@
#!/usr/bin/env node
/**
* generate-skills.mjs — shared-source generator for agent variants (M3, S-302)
*
* Generates every agent-variant directory (`skills/<skill>/<agent>/`) and
* `pi-package/skills/<skill>/` mirror from canonical sources. Generated files
* carry file-type-aware headers; each generated root gets a non-self-referential
* `.generated-manifest.json`.
*
* Usage:
* node scripts/generate-skills.mjs # regenerate everything
* pnpm run sync:pi # same via pnpm alias
*
* Exported helpers (used by verify-generated.mjs and tests):
* detectFileType(filePath) → string
* applyHeader(content, fileType, canonicalHint) → string
* makePackageJsonContent(sourcePkg, skillName, agentName) → object
* getGeneratedRoots(repoRoot?) → string[]
* buildManifest(generatedRootAbs, generatedRootRel) → Promise<object>
* generateSkills(repoRoot, options?) → Promise<{generatedRoots: string[]}>
*/
import {
lstat,
mkdir,
readdir,
readFile,
rm,
writeFile,
} from "node:fs/promises";
import crypto from "node:crypto";
import path from "node:path";
import { pathToFileURL } from "node:url";
const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), "..");
// ── Constants ──────────────────────────────────────────────────────────────
const MANIFEST_SCHEMA =
"https://ai-coding-skills.dev/schemas/generated-manifest/v1.json";
const MANIFEST_GENERATOR = "scripts/generate-skills.mjs";
const MANIFEST_FILENAME = ".generated-manifest.json";
const AGENTS = ["claude-code", "codex", "cursor", "opencode", "pi"];
/**
* Canonical list of all generated roots, relative to repo root.
* Verifier uses this to know which directories to walk.
*/
const GENERATED_ROOTS = [
// atlassian agent variants
"skills/atlassian/claude-code",
"skills/atlassian/codex",
"skills/atlassian/cursor",
"skills/atlassian/opencode",
"skills/atlassian/pi",
// web-automation agent variants
"skills/web-automation/claude-code",
"skills/web-automation/codex",
"skills/web-automation/cursor",
"skills/web-automation/opencode",
"skills/web-automation/pi",
// create-plan agent variants
"skills/create-plan/claude-code",
"skills/create-plan/codex",
"skills/create-plan/cursor",
"skills/create-plan/opencode",
"skills/create-plan/pi",
// do-task agent variants
"skills/do-task/claude-code",
"skills/do-task/codex",
"skills/do-task/cursor",
"skills/do-task/opencode",
"skills/do-task/pi",
// implement-plan agent variants
"skills/implement-plan/claude-code",
"skills/implement-plan/codex",
"skills/implement-plan/cursor",
"skills/implement-plan/opencode",
"skills/implement-plan/pi",
// reviewer-runtime pi variant
"skills/reviewer-runtime/pi",
// pi-package mirrors
"pi-package/skills/atlassian",
"pi-package/skills/create-plan",
"pi-package/skills/do-task",
"pi-package/skills/implement-plan",
"pi-package/skills/web-automation",
];
// ── File-type detection ────────────────────────────────────────────────────
/**
* Classify a file path into a header policy category.
* @param {string} filePath - Relative or absolute path to the file.
* @returns {'markdown'|'shell'|'ts'|'js'|'json'|'yaml'|'jsonc'|'unknown'}
*/
export function detectFileType(filePath) {
const base = path.basename(filePath);
const ext = path.extname(base).toLowerCase();
if (ext === ".md") return "markdown";
if (ext === ".sh") return "shell";
if (ext === ".ts") return "ts";
if (ext === ".js") return "js";
if (ext === ".jsonc") return "jsonc";
if (ext === ".json") return "json";
if (ext === ".yaml" || ext === ".yml") return "yaml";
return "unknown";
}
// ── Header insertion ──────────────────────────────────────────────────────
const HEADER_MSG = (hint) =>
`⚠️ GENERATED FILE – do not edit directly. Edit the canonical source in ${hint} and run \`pnpm run sync:pi\`.`;
/**
* Insert a file-type-aware generated-file header into content.
*
* Policy:
* markdown → HTML comment after YAML front matter (or at top if no front matter)
* shell → # comment after shebang (never before it)
* ts/js → // comment at top
* jsonc → // comment at top
* yaml → # comment at top (pnpm-lock.yaml skipped by caller)
* json → no header (recorded in .generated-manifest.json)
* unknown → no header
*
* @param {string} content - Original file content.
* @param {string} fileType - Output of detectFileType().
* @param {string} canonicalHint - Human-readable hint to canonical source location.
* @returns {string} Content with header inserted, or original if no header applies.
*/
export function applyHeader(content, fileType, canonicalHint) {
const msg = HEADER_MSG(canonicalHint);
switch (fileType) {
case "markdown": {
// Insert HTML comment after YAML front matter closing ---, or at top
if (content.startsWith("---\n")) {
const closingIdx = content.indexOf("\n---\n", 4);
if (closingIdx !== -1) {
const after = closingIdx + 5; // length of "\n---\n"
const before = content.slice(0, after);
const rest = content.slice(after);
// Ensure the comment is on its own line, with a blank line before/after
return `${before}\n<!-- ${msg} -->\n${rest}`;
}
}
return `<!-- ${msg} -->\n${content}`;
}
case "shell": {
// Insert # comment AFTER shebang line (never before it)
const lines = content.split("\n");
if (lines[0].startsWith("#!")) {
return [lines[0], `# ${msg}`, ...lines.slice(1)].join("\n");
}
return `# ${msg}\n${content}`;
}
case "ts":
case "js":
case "jsonc": {
// Insert after shebang if present (TypeScript requires #! on line 1)
const lines = content.split("\n");
if (lines[0].startsWith("#!")) {
return [lines[0], `// ${msg}`, ...lines.slice(1)].join("\n");
}
return `// ${msg}\n${content}`;
}
case "yaml": {
return `# ${msg}\n${content}`;
}
case "json":
case "unknown":
default:
return content;
}
}
// ── package.json transformation ───────────────────────────────────────────
/**
* Produce a modified package.json object with a unique scoped name and
* `"private": true` for an agent-variant generated root.
*
* @param {object} sourcePkg - Parsed source package.json object.
* @param {string} skillName - Skill identifier (e.g. "atlassian").
* @param {string} agentName - Agent identifier (e.g. "claude-code").
* @returns {object} New object (source is not mutated).
*/
export function makePackageJsonContent(sourcePkg, skillName, agentName) {
return {
...sourcePkg,
name: `@ai-coding-skills/${skillName}-${agentName}`,
private: true,
};
}
// ── Generated-root list ───────────────────────────────────────────────────
/**
* Return the authoritative list of generated roots as repo-relative paths.
* Callers (verify-generated.mjs) use this to know which directories to walk.
*
* @returns {string[]} Sorted array of relative paths.
*/
export function getGeneratedRoots() {
return [...GENERATED_ROOTS];
}
// ── Manifest construction ─────────────────────────────────────────────────
/**
* Walk a directory recursively and return all file paths (abs).
* Skips node_modules.
*/
async function walkDir(dir) {
const results = [];
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch {
return results;
}
for (const entry of entries) {
if (entry.name === "node_modules") continue;
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
const sub = await walkDir(full);
results.push(...sub);
} else if (entry.isFile()) {
results.push(full);
}
}
return results;
}
/**
* Build a .generated-manifest.json object for a generated root.
*
* The manifest lists every file in the generated root EXCEPT itself.
* Files are sorted by relative path for stable canonical serialization.
*
* @param {string} generatedRootAbs - Absolute path to the generated root directory.
* @param {string} generatedRootRel - Repo-relative path (e.g. "skills/create-plan/pi").
* @returns {Promise<object>} Manifest object (not yet serialized to disk).
*/
export async function buildManifest(generatedRootAbs, generatedRootRel) {
const allFiles = await walkDir(generatedRootAbs);
const entries = [];
for (const absPath of allFiles) {
const relPath = path.relative(generatedRootAbs, absPath).replace(/\\/g, "/");
// Non-self-referential: the manifest never lists itself
if (relPath === MANIFEST_FILENAME) continue;
const contentBuf = await readFile(absPath);
const sha256 = crypto.createHash("sha256").update(contentBuf).digest("hex");
const st = await lstat(absPath);
const mode = (st.mode & 0o777).toString(8).padStart(3, "0");
entries.push({
path: relPath,
kind: "file",
mode,
sha256,
});
}
entries.sort((a, b) => a.path.localeCompare(b.path));
return {
$schema: MANIFEST_SCHEMA,
generator: MANIFEST_GENERATOR,
generatedRoot: generatedRootRel,
files: entries,
};
}
// ── Core generation helpers ───────────────────────────────────────────────
/**
* Copy a single file from source to destination, inserting a header.
* Skips header for:
* - pnpm-lock.yaml (managed by pnpm, would be stripped on next install)
* - JSON files (per header policy, no in-file header)
* - node_modules (never copied)
*/
async function copyWithHeader(srcAbs, dstAbs, canonicalHint) {
const basename = path.basename(srcAbs);
// Skip pnpm-lock.yaml header (pnpm regenerates without comment)
const skipHeader = basename === "pnpm-lock.yaml";
const raw = await readFile(srcAbs, "utf8");
const fileType = skipHeader ? "unknown" : detectFileType(srcAbs);
const content = applyHeader(raw, fileType, canonicalHint);
await mkdir(path.dirname(dstAbs), { recursive: true });
await writeFile(dstAbs, content, "utf8");
}
/**
* Recursively copy a directory tree, adding headers to text files.
*/
async function copyDirWithHeaders(srcDir, dstDir, canonicalHint) {
const entries = await readdir(srcDir, { withFileTypes: true });
await mkdir(dstDir, { recursive: true });
for (const entry of entries) {
if (entry.name === "node_modules") continue;
const src = path.join(srcDir, entry.name);
const dst = path.join(dstDir, entry.name);
if (entry.isDirectory()) {
await copyDirWithHeaders(src, dst, canonicalHint);
} else if (entry.isFile()) {
await copyWithHeader(src, dst, canonicalHint);
}
}
}
/**
* Write a .generated-manifest.json file into a generated root.
*/
async function writeManifest(generatedRootAbs, generatedRootRel) {
const manifest = await buildManifest(generatedRootAbs, generatedRootRel);
const dstPath = path.join(generatedRootAbs, MANIFEST_FILENAME);
await writeFile(dstPath, JSON.stringify(manifest, null, 2) + "\n", "utf8");
}
// ── Skill-family generators ───────────────────────────────────────────────
/**
* Generate one agent variant for a "skills-only" skill
* (create-plan, do-task, implement-plan).
*
* Canonical source: skills/<skill>/_source/<agent>/
* Generated root: skills/<skill>/<agent>/
*
* @param {string} writeRoot - Root directory to write output into (defaults to repoRoot).
*/
async function generateSkillOnlyVariant(repoRoot, writeRoot, skillName, agentName, generatedRootRel) {
const sourceDir = path.join(repoRoot, "skills", skillName, "_source", agentName);
const targetDir = path.join(writeRoot, generatedRootRel);
const canonicalHint = `skills/${skillName}/_source/${agentName}/`;
// Clear previous generated content (preserve node_modules if any)
await clearGeneratedRoot(targetDir);
// Copy all files from source with headers
await copyDirWithHeaders(sourceDir, targetDir, canonicalHint);
// Write manifest
await writeManifest(targetDir, generatedRootRel);
}
/**
* Generate one agent variant for a "scripts+skill" skill
* (atlassian, web-automation).
*
* For atlassian:
* - SKILL.md from skills/atlassian/_source/<agent>/SKILL.md
* - scripts/* from skills/atlassian/shared/scripts/
* (only src/, tsconfig.json, pnpm-lock.yaml — not tests/ or scripts/sync-*)
*
* For web-automation:
* - SKILL.md from skills/web-automation/_source/<agent>/SKILL.md
* - scripts/* from skills/web-automation/shared/
*
* @param {string} [packageAgentName] - Override the agent name used only for the
* package.json `name` field. Defaults to `agentName`. Use this to give
* pi-package mirrors a distinct name (e.g. "pi-mirror") so workspace package
* names are unique even when two roots share the same source agent.
*/
async function generateScriptsSkillVariant(
repoRoot,
writeRoot,
skillName,
agentName,
generatedRootRel,
config,
packageAgentName,
) {
const targetDir = path.join(writeRoot, generatedRootRel);
await clearGeneratedRoot(targetDir);
// 1. Copy SKILL.md from per-agent canonical source
const skillMdSrc = path.join(repoRoot, "skills", skillName, "_source", agentName, "SKILL.md");
const skillMdDst = path.join(targetDir, "SKILL.md");
const skillMdHint = `skills/${skillName}/_source/${agentName}/SKILL.md`;
await copyWithHeader(skillMdSrc, skillMdDst, skillMdHint);
// 2. Copy scripts
const scriptsTargetDir = path.join(targetDir, "scripts");
await mkdir(scriptsTargetDir, { recursive: true });
const canonicalScripts = path.join(repoRoot, config.canonicalScripts);
const scriptsHint = `${config.canonicalScripts}/`;
for (const entry of config.scriptFiles) {
const srcPath = path.join(canonicalScripts, entry);
const dstPath = path.join(scriptsTargetDir, entry); // scriptsTargetDir already uses writeRoot
// Check if source exists
let st;
try {
st = await lstat(srcPath);
} catch {
continue; // skip if file doesn't exist in canonical
}
if (st.isDirectory()) {
await copyDirWithHeaders(srcPath, dstPath, scriptsHint);
} else {
await copyWithHeader(srcPath, dstPath, scriptsHint);
}
}
// 3. Generate modified package.json
const srcPkgPath = path.join(canonicalScripts, "package.json");
const srcPkg = JSON.parse(await readFile(srcPkgPath, "utf8"));
let targetPkg = makePackageJsonContent(srcPkg, skillName, packageAgentName ?? agentName);
// For atlassian variants, strip test/sync scripts (not in agent variants)
if (skillName === "atlassian") {
const agentScripts = {};
for (const [k, v] of Object.entries(targetPkg.scripts ?? {})) {
if (k !== "test" && k !== "sync:agents") agentScripts[k] = v;
}
targetPkg = { ...targetPkg, scripts: agentScripts };
}
const dstPkgPath = path.join(scriptsTargetDir, "package.json");
await writeFile(dstPkgPath, JSON.stringify(targetPkg, null, 2) + "\n", "utf8");
// 4. Write manifest
await writeManifest(targetDir, generatedRootRel);
}
/**
* Generate the reviewer-runtime pi variant from the non-Pi canonical scripts.
*
* Canonical source: skills/reviewer-runtime/{run-review.sh,notify-telegram.sh}
* Generated root: skills/reviewer-runtime/pi/
*
* The pi variant is byte-identical to the canonical except for:
* - A generated # comment after the shebang (replaces old "keep in sync" comment)
*
* @param {string} writeRoot - Root directory to write output into (defaults to repoRoot).
*/
async function generateReviewerRuntimePi(repoRoot, writeRoot) {
const srcDir = path.join(repoRoot, "skills", "reviewer-runtime");
const dstDir = path.join(writeRoot, "skills", "reviewer-runtime", "pi");
const canonicalHint = "skills/reviewer-runtime/";
// Clear old generated content (preserve tests/ which is canonical)
await clearGeneratedRoot(dstDir);
for (const fname of ["run-review.sh", "notify-telegram.sh"]) {
const srcPath = path.join(srcDir, fname);
const dstPath = path.join(dstDir, fname);
const raw = await readFile(srcPath, "utf8");
let content = applyHeader(raw, "shell", `${canonicalHint}${fname}`);
await mkdir(path.dirname(dstPath), { recursive: true });
await writeFile(dstPath, content, "utf8");
// Preserve executable bit
const st = await lstat(srcPath);
if (st.mode & 0o100) {
const { chmod } = await import("node:fs/promises");
await chmod(dstPath, 0o755);
}
}
// Write manifest (generatedRootRel is always relative to repo root, not writeRoot)
await writeManifest(dstDir, "skills/reviewer-runtime/pi");
}
/**
* Clear generated content in a root, preserving:
* - node_modules (installed by pnpm)
* - .generated-manifest.json (will be rewritten after generation)
*/
async function clearGeneratedRoot(rootDir) {
let entries;
try {
entries = await readdir(rootDir, { withFileTypes: true });
} catch {
return; // dir doesn't exist yet — nothing to clear
}
for (const entry of entries) {
if (entry.name === "node_modules") continue;
if (entry.name === MANIFEST_FILENAME) continue;
await rm(path.join(rootDir, entry.name), { recursive: true, force: true });
}
}
// ── Skill configurations ──────────────────────────────────────────────────
const SCRIPTS_SKILL_CONFIGS = {
atlassian: {
canonicalScripts: "skills/atlassian/shared/scripts",
// Files to copy from canonicalScripts into each agent's scripts/ dir
scriptFiles: ["src", "tsconfig.json", "pnpm-lock.yaml"],
},
"web-automation": {
canonicalScripts: "skills/web-automation/shared",
scriptFiles: [
"auth.ts",
"browse.ts",
"check-install.js",
"extract.js",
"flow.ts",
"scan-local-app.ts",
"scrape.ts",
"test-full.ts",
"test-minimal.ts",
"test-profile.ts",
"tsconfig.json",
"turndown-plugin-gfm.d.ts",
"pnpm-lock.yaml",
],
},
};
// ── Main generator ────────────────────────────────────────────────────────
/**
* Regenerate all agent variants from canonical sources.
*
* @param {string} repoRoot - Absolute path to repo root (canonical sources are read from here).
* @param {object} [options]
* @param {boolean} [options.dryRun] - If true, don't write files (future use).
* @param {string} [options.targetRoot] - Write generated output here instead of `repoRoot`.
* Canonical sources are always read from `repoRoot`. Use this to generate into a
* temp directory for drift detection without modifying on-disk files.
* @returns {Promise<{generatedRoots: string[]}>}
*/
export async function generateSkills(repoRoot = REPO_ROOT, options = {}) {
const { dryRun = false, targetRoot } = options;
const writeRoot = targetRoot ?? repoRoot;
if (dryRun) {
return { generatedRoots: GENERATED_ROOTS };
}
const skillOnlySkills = ["create-plan", "do-task", "implement-plan"];
// 1. Generate skill-only skills (create-plan, do-task, implement-plan)
for (const skillName of skillOnlySkills) {
for (const agentName of AGENTS) {
const rootRel = `skills/${skillName}/${agentName}`;
await generateSkillOnlyVariant(repoRoot, writeRoot, skillName, agentName, rootRel);
}
// pi-package mirror (same source as pi variant)
const piPackageRel = `pi-package/skills/${skillName}`;
await generateSkillOnlyVariant(repoRoot, writeRoot, skillName, "pi", piPackageRel);
}
// 2. Generate scripts skills (atlassian, web-automation)
for (const [skillName, config] of Object.entries(SCRIPTS_SKILL_CONFIGS)) {
for (const agentName of AGENTS) {
const rootRel = `skills/${skillName}/${agentName}`;
await generateScriptsSkillVariant(repoRoot, writeRoot, skillName, agentName, rootRel, config);
}
// pi-package mirror: same source as pi variant but a distinct package name
// ("pi-mirror" suffix) so the workspace has no duplicate package names.
const piPackageRel = `pi-package/skills/${skillName}`;
await generateScriptsSkillVariant(repoRoot, writeRoot, skillName, "pi", piPackageRel, config, "pi-mirror");
}
// 3. Generate reviewer-runtime pi variant
await generateReviewerRuntimePi(repoRoot, writeRoot);
return { generatedRoots: GENERATED_ROOTS };
}
// ── CLI entry point ────────────────────────────────────────────────────────
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
try {
const result = await generateSkills(REPO_ROOT);
console.log(
`Generated ${result.generatedRoots.length} roots from canonical sources.`,
);
process.exit(0);
} catch (err) {
console.error("generate-skills: fatal error:", err.message ?? err);
process.exit(1);
}
}
scripts/lib/run-link-check.mjs
+8 -1
View File
@@ -47,13 +47,20 @@ const SKIP_PATHS = new Set([
"pi-package",
]);
// Also skip any _source/ subdirectory within skills — canonical source files
// use relative paths calibrated to the generated location (one level shallower).
const SKIP_SEGMENT = "_source";
function shouldSkip(absPath) {
const rel = path.relative(REPO_ROOT, absPath);
for (const skip of SKIP_PATHS) {
if (rel === skip || rel.startsWith(skip + path.sep)) return true;
}
const parts = rel.split(path.sep);
return parts.includes("node_modules");
if (parts.includes("node_modules")) return true;
// Skip canonical _source/ directories (links calibrated to generated location)
if (parts.includes(SKIP_SEGMENT)) return true;
return false;
}
function collectMarkdownFiles(dir) {
scripts/lib/skill-manager-core.mjs
+5 -2
View File
@@ -279,7 +279,7 @@ async function findClaudeCodeSuperpowersPluginRoots(homeDir) {
async function findCursorSuperpowersPluginRoots(homeDir) {
const pluginRoot = path.join(homeDir, ".cursor", "plugins", "cache", "cursor-public", "superpowers");
let entries = [];
let entries;
try {
entries = await readdir(pluginRoot, { withFileTypes: true });
} catch (error) {
@@ -598,7 +598,10 @@ export async function executeOperation(op) {
if (op.action === "unsupported" || op.status === "skipped") return { ...op, status: "skipped" };
if (op.kind === "package-skill") return { ...op, status: "included" };
if (op.kind === "sync-pi-package") {
runCommand(path.join(op.repoRoot, "scripts", "sync-pi-package-skills.sh"), [], { cwd: op.repoRoot });
// Use the canonical generator (pnpm run sync:pi / node scripts/generate-skills.mjs).
// The legacy sync-pi-package-skills.sh is retired in M3; it bypassed the
// generator and copied skills/*/pi into pi-package directly, corrupting manifests.
runCommand(process.execPath, [path.join(op.repoRoot, "scripts", "generate-skills.mjs")], { cwd: op.repoRoot });
return { ...op, status: "ok" };
}
if (op.kind === "pi-package") {
scripts/manage-skills.mjs
-1
View File
@@ -267,7 +267,6 @@ async function main() {
const removeAnswer = await rl.question(`Remove Superpowers for ${prompt.clientId}/${prompt.scope} too? (yes/no) [no]: `);
if (removeAnswer.trim().toLowerCase() === "yes") {
const scope = resolveClientScope(prompt.clientId, prompt.scope, process.cwd());
const client = CLIENTS[prompt.clientId];
const target = `${scope.skillsRoot}/superpowers`;
plan.operations.push({ kind: "superpowers", clientId: prompt.clientId, scope: prompt.scope, action: "remove", target, skillsRoot: scope.skillsRoot });
}
scripts/tests/generate-skills.test.mjs
+350
View File
@@ -0,0 +1,350 @@
/**
* Unit tests for generate-skills.mjs — RED phase of TDD.
*
* Tests cover:
* - detectFileType: classification of files by extension
* - applyHeader: insertion per file-type-aware policy
* - makePackageJsonContent: unique name + private:true
* - getGeneratedRoots: returns the canonical generated-root list
*/
import assert from "node:assert/strict";
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
import crypto from "node:crypto";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const SCRIPTS_DIR = path.resolve(__dirname, "..");
const {
detectFileType,
applyHeader,
makePackageJsonContent,
getGeneratedRoots,
buildManifest,
} = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
// ── detectFileType ────────────────────────────────────────────────────────
test("detectFileType: .md files → markdown", () => {
assert.equal(detectFileType("SKILL.md"), "markdown");
assert.equal(detectFileType("templates/milestone-plan.md"), "markdown");
assert.equal(detectFileType("README.md"), "markdown");
});
test("detectFileType: .sh files → shell", () => {
assert.equal(detectFileType("run-review.sh"), "shell");
assert.equal(detectFileType("scripts/install.sh"), "shell");
});
test("detectFileType: .ts and .d.ts files → ts", () => {
assert.equal(detectFileType("src/cli.ts"), "ts");
assert.equal(detectFileType("turndown-plugin-gfm.d.ts"), "ts");
assert.equal(detectFileType("auth.ts"), "ts");
});
test("detectFileType: .js files → js", () => {
assert.equal(detectFileType("check-install.js"), "js");
assert.equal(detectFileType("extract.js"), "js");
});
test("detectFileType: .json files → json", () => {
assert.equal(detectFileType("package.json"), "json");
assert.equal(detectFileType("tsconfig.json"), "json");
});
test("detectFileType: .yaml and .yml files → yaml", () => {
assert.equal(detectFileType("pnpm-lock.yaml"), "yaml");
assert.equal(detectFileType("other.yml"), "yaml");
});
test("detectFileType: .jsonc files → jsonc", () => {
assert.equal(detectFileType(".markdownlint.jsonc"), "jsonc");
});
test("detectFileType: unknown extension → unknown", () => {
assert.equal(detectFileType("Makefile"), "unknown");
assert.equal(detectFileType("somefile"), "unknown");
});
// ── applyHeader ───────────────────────────────────────────────────────────
test("applyHeader: markdown with YAML front matter inserts HTML comment after closing ---", () => {
const content = "---\nname: create-plan\n---\n\n# Create Plan\n\nBody.\n";
const result = applyHeader(content, "markdown", "skills/create-plan/_source/claude-code/SKILL.md");
// Front matter block preserved verbatim at start
assert.ok(result.startsWith("---\nname: create-plan\n---\n"), "front matter at start");
// HTML comment present
assert.ok(result.includes("<!-- ⚠️"), "HTML comment present");
// Comment comes after front matter closer and before the title
const commentIdx = result.indexOf("<!-- ⚠️");
const titleIdx = result.indexOf("# Create Plan");
assert.ok(commentIdx !== -1 && titleIdx !== -1, "both positions found");
assert.ok(commentIdx < titleIdx, "comment before title");
// Comment must NOT appear before the first ---
assert.ok(commentIdx > 3, "comment not before front matter");
});
test("applyHeader: markdown without front matter inserts HTML comment at top", () => {
const content = "# Heading\n\nContent\n";
const result = applyHeader(content, "markdown", "skills/create-plan/_source/claude-code/templates/plan.md");
assert.ok(result.startsWith("<!-- ⚠️"), "comment at very top");
assert.ok(result.includes("# Heading"), "original content preserved");
});
test("applyHeader: shell with shebang inserts # comment after shebang", () => {
const content = "#!/usr/bin/env bash\nset -euo pipefail\necho hello\n";
const result = applyHeader(content, "shell", "skills/reviewer-runtime/run-review.sh");
assert.ok(result.startsWith("#!/usr/bin/env bash\n"), "shebang preserved at top");
assert.ok(result.includes("# ⚠️"), "hash comment present");
const shebangEnd = result.indexOf("\n") + 1;
const commentStart = result.indexOf("# ⚠️");
assert.ok(commentStart === shebangEnd, "comment immediately after shebang line");
});
test("applyHeader: ts file inserts // comment at top", () => {
const content = "import path from 'path';\nexport const x = 1;\n";
const result = applyHeader(content, "ts", "skills/atlassian/shared/scripts/src/cli.ts");
assert.ok(result.startsWith("// ⚠️"), "// comment at top");
assert.ok(result.includes("import path from 'path';"), "original content preserved");
});
test("applyHeader: ts file with shebang inserts // comment after shebang", () => {
const content = "#!/usr/bin/env npx tsx\nimport foo from 'foo';\n";
const result = applyHeader(content, "ts", "auth.ts");
assert.ok(result.startsWith("#!/usr/bin/env npx tsx\n"), "shebang preserved at top");
assert.ok(result.includes("// ⚠️"), "// comment present");
const shebangEnd = result.indexOf("\n") + 1;
const commentStart = result.indexOf("// ⚠️");
assert.ok(commentStart === shebangEnd, "// comment immediately after shebang");
});
test("applyHeader: js file with shebang inserts // comment after shebang", () => {
const content = "#!/usr/bin/env node\nconst x = 1;\n";
const result = applyHeader(content, "js", "check-install.js");
assert.ok(result.startsWith("#!/usr/bin/env node\n"), "shebang preserved at top");
assert.ok(result.includes("// ⚠️"), "// comment present");
});
test("applyHeader: js file inserts // comment at top", () => {
const content = "const x = require('x');\n";
const result = applyHeader(content, "js", "check-install.js");
assert.ok(result.startsWith("// ⚠️"), "// comment at top");
});
test("applyHeader: yaml file inserts # comment at top", () => {
const content = "lockfileVersion: '9.0'\n\npackages:\n";
const result = applyHeader(content, "yaml", "pnpm-lock.yaml");
assert.ok(result.startsWith("# ⚠️"), "# comment at top");
});
test("applyHeader: jsonc file inserts // comment at top", () => {
const content = "// existing comment\n{\n \"key\": true\n}\n";
const result = applyHeader(content, "jsonc", ".markdownlint.jsonc");
assert.ok(result.startsWith("// ⚠️"), "// comment at top");
});
test("applyHeader: json file returns content unchanged (no header)", () => {
const content = '{\n "name": "test"\n}\n';
const result = applyHeader(content, "json", "package.json");
assert.equal(result, content, "JSON file must not be modified");
});
test("applyHeader: unknown type returns content unchanged", () => {
const content = "raw content\n";
const result = applyHeader(content, "unknown", "Makefile");
assert.equal(result, content);
});
test("applyHeader: header contains canonical source hint", () => {
const hint = "skills/create-plan/_source/pi/SKILL.md";
const content = "---\nname: create-plan\n---\n\n# Title\n";
const result = applyHeader(content, "markdown", hint);
assert.ok(result.includes(hint), "canonical hint appears in header");
});
test("applyHeader: never inserts header before shebang", () => {
const content = "#!/usr/bin/env bash\nset -euo pipefail\n";
const result = applyHeader(content, "shell", "run.sh");
assert.ok(result.startsWith("#!/"), "shebang still first");
});
// ── makePackageJsonContent ────────────────────────────────────────────────
test("makePackageJsonContent: renames name to scoped unique form", () => {
const src = { name: "atlassian-skill-scripts", version: "1.0.0" };
const result = makePackageJsonContent(src, "atlassian", "claude-code");
assert.equal(result.name, "@ai-coding-skills/atlassian-claude-code");
});
test("makePackageJsonContent: adds private:true", () => {
const src = { name: "web-automation-scripts", version: "1.0.0" };
const result = makePackageJsonContent(src, "web-automation", "codex");
assert.equal(result.private, true);
});
test("makePackageJsonContent: preserves all other top-level fields", () => {
const src = {
name: "atlassian-skill-scripts",
version: "1.0.0",
type: "module",
scripts: { typecheck: "tsc --noEmit" },
dependencies: { commander: "^13.1.0" },
devDependencies: { tsx: "^4.20.5" },
packageManager: "pnpm@10.18.1",
};
const result = makePackageJsonContent(src, "atlassian", "cursor");
assert.equal(result.version, "1.0.0");
assert.equal(result.type, "module");
assert.deepEqual(result.scripts, { typecheck: "tsc --noEmit" });
assert.deepEqual(result.dependencies, { commander: "^13.1.0" });
assert.equal(result.packageManager, "pnpm@10.18.1");
});
test("makePackageJsonContent: pi-package mirror uses -pi agent suffix", () => {
const src = { name: "atlassian-skill-scripts" };
const result = makePackageJsonContent(src, "atlassian", "pi");
assert.equal(result.name, "@ai-coding-skills/atlassian-pi");
});
test("makePackageJsonContent: does not mutate the source object", () => {
const src = { name: "original", version: "1.0.0" };
makePackageJsonContent(src, "atlassian", "codex");
assert.equal(src.name, "original", "source object not mutated");
});
// ── getGeneratedRoots ─────────────────────────────────────────────────────
test("getGeneratedRoots: returns at least one root per agent per skills skill", () => {
const roots = getGeneratedRoots();
assert.ok(Array.isArray(roots), "returns array");
assert.ok(roots.length > 0, "non-empty");
const agents = ["claude-code", "codex", "cursor", "opencode", "pi"];
const skillsWithAgents = ["create-plan", "do-task", "implement-plan", "atlassian", "web-automation"];
for (const skill of skillsWithAgents) {
for (const agent of agents) {
const expected = `skills/${skill}/${agent}`;
assert.ok(roots.includes(expected), `missing generated root: ${expected}`);
}
}
});
test("getGeneratedRoots: includes pi-package mirrors for all skills", () => {
const roots = getGeneratedRoots();
const piPackageSkills = ["atlassian", "create-plan", "do-task", "implement-plan", "web-automation"];
for (const skill of piPackageSkills) {
const expected = `pi-package/skills/${skill}`;
assert.ok(roots.includes(expected), `missing pi-package mirror root: ${expected}`);
}
});
test("getGeneratedRoots: includes reviewer-runtime/pi", () => {
const roots = getGeneratedRoots();
assert.ok(roots.includes("skills/reviewer-runtime/pi"), "reviewer-runtime/pi missing");
});
test("getGeneratedRoots: does not include _source or shared directories", () => {
const roots = getGeneratedRoots();
for (const r of roots) {
assert.ok(!r.includes("_source"), `root should not contain _source: ${r}`);
assert.ok(!r.endsWith("/shared"), `root should not be shared: ${r}`);
assert.ok(!r.includes("shared/scripts"), `root should not contain shared/scripts: ${r}`);
}
});
// ── buildManifest ─────────────────────────────────────────────────────────
test("buildManifest: returns object with $schema and generator markers", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "manifest-test-"));
try {
await writeFile(path.join(dir, "SKILL.md"), "---\nname: test\n---\n# Test\n");
await mkdir(path.join(dir, "templates"), { recursive: true });
await writeFile(path.join(dir, "templates", "plan.md"), "# Plan\n");
// Write .generated-manifest.json itself (should be excluded from listing)
await writeFile(path.join(dir, ".generated-manifest.json"), "{}");
const manifest = await buildManifest(dir, "skills/test/claude-code");
assert.ok(manifest.$schema, "$schema field present");
assert.ok(manifest.generator, "generator field present");
assert.equal(manifest.generatedRoot, "skills/test/claude-code");
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("buildManifest: does not include .generated-manifest.json in files list", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "manifest-self-"));
try {
await writeFile(path.join(dir, "SKILL.md"), "# skill\n");
await writeFile(path.join(dir, ".generated-manifest.json"), "{}");
const manifest = await buildManifest(dir, "skills/test/pi");
const paths = manifest.files.map((f) => f.path);
assert.ok(!paths.includes(".generated-manifest.json"), "manifest must not list itself");
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("buildManifest: files list includes path, kind, mode, sha256", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "manifest-fields-"));
try {
await writeFile(path.join(dir, "SKILL.md"), "# skill\n");
const manifest = await buildManifest(dir, "skills/test/codex");
assert.equal(manifest.files.length, 1);
const entry = manifest.files[0];
assert.equal(entry.path, "SKILL.md");
assert.equal(entry.kind, "file");
assert.ok(typeof entry.mode === "string" && entry.mode.match(/^\d{3}$/), "mode is 3-digit octal string");
assert.ok(typeof entry.sha256 === "string" && entry.sha256.length === 64, "sha256 is 64-char hex");
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("buildManifest: files are sorted by path", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "manifest-sorted-"));
try {
await mkdir(path.join(dir, "templates"), { recursive: true });
await writeFile(path.join(dir, "SKILL.md"), "# skill\n");
await writeFile(path.join(dir, "templates", "z.md"), "z\n");
await writeFile(path.join(dir, "templates", "a.md"), "a\n");
const manifest = await buildManifest(dir, "skills/test/cursor");
const paths = manifest.files.map((f) => f.path);
const sorted = [...paths].sort();
assert.deepEqual(paths, sorted, "files must be sorted by path");
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("buildManifest: sha256 matches actual file content", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "manifest-hash-"));
try {
const content = "---\nname: test\n---\n# Title\n";
await writeFile(path.join(dir, "SKILL.md"), content);
const manifest = await buildManifest(dir, "skills/test/opencode");
const expected = crypto.createHash("sha256").update(content, "utf8").digest("hex");
assert.equal(manifest.files[0].sha256, expected, "sha256 matches file content");
} finally {
await rm(dir, { recursive: true, force: true });
}
});
scripts/tests/verify-generated.test.mjs
+348
View File
@@ -0,0 +1,348 @@
/**
* Unit tests for verify-generated.mjs — RED phase of TDD.
*
* Key contract from acceptance criteria:
* - A stray file under skills/<skill>/_source/ or skills/<skill>/shared/
* does NOT cause verify:generated to flag it as stale.
* - A stray file under skills/<skill>/<agent>/ (other than
* .generated-manifest.json) DOES cause verify:generated to flag it.
*/
import assert from "node:assert/strict";
import { mkdtemp, mkdir, writeFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import test from "node:test";
import { fileURLToPath } from "node:url";
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const SCRIPTS_DIR = path.resolve(__dirname, "..");
const { verifyGenerated } = await import(`${SCRIPTS_DIR}/verify-generated.mjs`);
// ── Stray-file detection boundary tests ──────────────────────────────────
test("verifyGenerated: stray file in _source/ is NOT flagged as stale", async () => {
// This uses the real repo's canonical source - if stray file under _source
// is added, verify-generated should not complain about it.
// We test this by verifying the function signature accepts a repoRoot and
// generatedRoots override, and that the verifier only walks declared roots.
// (Full integration test against the real repo runs in pnpm run verify:generated)
const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-source-"));
try {
// Create a fake skill structure with _source/ and a generated root
const skillName = "test-skill";
const agentName = "claude-code";
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
const agentDir = path.join(dir, "skills", skillName, agentName);
const sharedDir = path.join(dir, "skills", skillName, "shared");
await mkdir(sourceDir, { recursive: true });
await mkdir(sharedDir, { recursive: true });
await mkdir(agentDir, { recursive: true });
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
// Add a STRAY file in _source/ — this should NOT trigger stale detection
await writeFile(path.join(sourceDir, "STRAY.md"), "stray content");
// Add a STRAY file in shared/ — this should NOT trigger stale detection
await writeFile(path.join(sharedDir, "shared-stray.txt"), "shared stray");
// The generated root is EMPTY (no manifest, no files) — but we're testing
// that _source/ and shared/ stray files don't appear in stale detection.
// We can test this indirectly: verifyGenerated with no declared roots for
// this dir returns no stale errors about _source/ or shared/.
const result = await verifyGenerated(dir, {
// Override generated roots to be empty for this minimal test
generatedRootsOverride: [],
});
// No stale errors from _source/ or shared/
const sourceErrors = result.errors.filter(
(e) => e.includes("_source") || e.includes("shared"),
);
assert.equal(sourceErrors.length, 0, `unexpected stale errors from _source/shared: ${JSON.stringify(sourceErrors)}`);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("verifyGenerated: stray file in agent dir IS flagged as stale", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "vg-stray-agent-"));
try {
const skillName = "test-skill";
const agentName = "claude-code";
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
const agentDir = path.join(dir, "skills", skillName, agentName);
await mkdir(sourceDir, { recursive: true });
await mkdir(agentDir, { recursive: true });
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
// Generate the agent dir with proper content + manifest
const headerLine =
`<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
`Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
`and run \`pnpm run sync:pi\`. -->`;
const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);
// Add a STRAY file in the agent dir — this SHOULD be flagged
await writeFile(path.join(agentDir, "STRAY.md"), "stray content");
// Write a manifest that does NOT include STRAY.md
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
// Remove STRAY.md from manifest (simulate pre-stray-add manifest)
manifest.files = manifest.files.filter((f) => f.path !== "STRAY.md");
await writeFile(
path.join(agentDir, ".generated-manifest.json"),
JSON.stringify(manifest, null, 2) + "\n",
);
const result = await verifyGenerated(dir, {
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
});
assert.equal(result.ok, false, "should fail when stray file present");
const strayError = result.errors.some((e) => e.includes("STRAY.md"));
assert.ok(strayError, `STRAY.md should appear in errors: ${JSON.stringify(result.errors)}`);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("verifyGenerated: .generated-manifest.json is excluded from stale-file detection", async () => {
// Even though .generated-manifest.json is in the generated root, it should
// not be considered a "stale file" just because it's not in the files list
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-self-"));
try {
const skillName = "test-skill";
const agentName = "pi";
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
const agentDir = path.join(dir, "skills", skillName, agentName);
await mkdir(sourceDir, { recursive: true });
await mkdir(agentDir, { recursive: true });
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
const headerLine =
`<!-- ⚠️ GENERATED FILE – do not edit directly. ` +
`Edit the canonical source in skills/${skillName}/_source/${agentName}/SKILL.md ` +
`and run \`pnpm run sync:pi\`. -->`;
const generatedContent = `---\nname: test-skill\n---\n\n${headerLine}\n\n# Test Skill\n`;
await writeFile(path.join(agentDir, "SKILL.md"), generatedContent);
// Write manifest (will include SKILL.md, not itself)
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
const manifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
await writeFile(
path.join(agentDir, ".generated-manifest.json"),
JSON.stringify(manifest, null, 2) + "\n",
);
const result = await verifyGenerated(dir, {
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
});
// Should pass — .generated-manifest.json is excluded from stale detection
const manifestErrors = result.errors.filter((e) => e.includes(".generated-manifest.json"));
assert.equal(manifestErrors.length, 0, `manifest file should not appear as stale: ${JSON.stringify(manifestErrors)}`);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("verifyGenerated: missing file from manifest is flagged as deleted", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "vg-missing-file-"));
try {
const skillName = "test-skill";
const agentName = "cursor";
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
const agentDir = path.join(dir, "skills", skillName, agentName);
await mkdir(sourceDir, { recursive: true });
await mkdir(agentDir, { recursive: true });
const skillContent = "---\nname: test-skill\n---\n\n# Test Skill\n";
await writeFile(path.join(sourceDir, "SKILL.md"), skillContent);
await writeFile(path.join(agentDir, "SKILL.md"), "generated content\n");
// Manifest claims templates/plan.md exists, but the file doesn't
const manifest = {
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
generator: "scripts/generate-skills.mjs",
generatedRoot: `skills/${skillName}/${agentName}`,
files: [
{ path: "SKILL.md", kind: "file", mode: "644", sha256: "aaa" },
{ path: "templates/plan.md", kind: "file", mode: "644", sha256: "bbb" },
],
};
await writeFile(
path.join(agentDir, ".generated-manifest.json"),
JSON.stringify(manifest, null, 2) + "\n",
);
const result = await verifyGenerated(dir, {
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
});
assert.equal(result.ok, false, "should fail on missing file");
const missingError = result.errors.some((e) => e.includes("templates/plan.md"));
assert.ok(missingError, `missing file should appear in errors: ${JSON.stringify(result.errors)}`);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("verifyGenerated: content mismatch is flagged", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "vg-content-mismatch-"));
try {
const skillName = "test-skill";
const agentName = "opencode";
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
const agentDir = path.join(dir, "skills", skillName, agentName);
await mkdir(sourceDir, { recursive: true });
await mkdir(agentDir, { recursive: true });
await writeFile(path.join(sourceDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Original\n");
// Agent dir has DIFFERENT content than what manifest says
await writeFile(path.join(agentDir, "SKILL.md"), "---\nname: test-skill\n---\n\n# Modified!\n");
const manifest = {
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
generator: "scripts/generate-skills.mjs",
generatedRoot: `skills/${skillName}/${agentName}`,
files: [
{
path: "SKILL.md",
kind: "file",
mode: "644",
// SHA of the ORIGINAL content (not what's on disk)
sha256: "0000000000000000000000000000000000000000000000000000000000000000",
},
],
};
await writeFile(
path.join(agentDir, ".generated-manifest.json"),
JSON.stringify(manifest, null, 2) + "\n",
);
const result = await verifyGenerated(dir, {
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
});
assert.equal(result.ok, false, "should fail on content mismatch");
const mismatchError = result.errors.some((e) => e.includes("SKILL.md"));
assert.ok(mismatchError, `content mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("verifyGenerated: manifest entry with wrong sha256 is flagged even when paths match", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-sha-"));
try {
const skillName = "test-skill";
const agentName = "codex";
const sourceDir = path.join(dir, "skills", skillName, "_source", agentName);
const agentDir = path.join(dir, "skills", skillName, agentName);
await mkdir(sourceDir, { recursive: true });
await mkdir(agentDir, { recursive: true });
const content = "---\nname: test-skill\n---\n\n# Test Skill\n";
await writeFile(path.join(sourceDir, "SKILL.md"), content);
await writeFile(path.join(agentDir, "SKILL.md"), content);
// Manifest has correct path but deliberately wrong sha256 (simulates corrupted metadata)
const manifest = {
$schema: "https://ai-coding-skills.dev/schemas/generated-manifest/v1.json",
generator: "scripts/generate-skills.mjs",
generatedRoot: `skills/${skillName}/${agentName}`,
files: [
{
path: "SKILL.md",
kind: "file",
mode: "644",
sha256: "deadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeefdeadbeef",
},
],
};
await writeFile(
path.join(agentDir, ".generated-manifest.json"),
JSON.stringify(manifest, null, 2) + "\n",
);
const result = await verifyGenerated(dir, {
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
});
// Even though file paths match, the sha256 mismatch in the manifest should be detected
assert.equal(result.ok, false, "should fail on sha256 mismatch in manifest");
const sha256Error = result.errors.some(
(e) => e.includes("sha256") || e.includes("SKILL.md"),
);
assert.ok(sha256Error, `sha256 mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
test("verifyGenerated: manifest entry with wrong mode is flagged even when paths match", async () => {
const dir = await mkdtemp(path.join(tmpdir(), "vg-manifest-mode-"));
try {
const skillName = "test-skill";
const agentName = "cursor";
const agentDir = path.join(dir, "skills", skillName, agentName);
await mkdir(agentDir, { recursive: true });
const content = "# Test Skill\n";
await writeFile(path.join(agentDir, "SKILL.md"), content);
// Build a correct manifest first (with real sha256)
const { buildManifest } = await import(`${SCRIPTS_DIR}/generate-skills.mjs`);
const correctManifest = await buildManifest(agentDir, `skills/${skillName}/${agentName}`);
// Tamper: change the mode field for the SKILL.md entry
const tamperedManifest = {
...correctManifest,
files: correctManifest.files.map((f) =>
f.path === "SKILL.md" ? { ...f, mode: "777" } : f,
),
};
await writeFile(
path.join(agentDir, ".generated-manifest.json"),
JSON.stringify(tamperedManifest, null, 2) + "\n",
);
const result = await verifyGenerated(dir, {
generatedRootsOverride: [`skills/${skillName}/${agentName}`],
});
// The mode mismatch in the manifest should be detected by diffManifests
assert.equal(result.ok, false, "should fail on mode mismatch in manifest");
const modeError = result.errors.some(
(e) => e.includes("mode") || e.includes("SKILL.md"),
);
assert.ok(modeError, `mode mismatch should appear in errors: ${JSON.stringify(result.errors)}`);
} finally {
await rm(dir, { recursive: true, force: true });
}
});
scripts/verify-generated.mjs
+421
View File
@@ -0,0 +1,421 @@
#!/usr/bin/env node
/**
* verify-generated.mjs — drift detector for generator-owned files (M3, S-306)
*
* Verifies that every declared generated root on disk matches the content that
* the generator would produce from the current canonical sources.
*
* Two verification modes:
*
* Production mode (default, no generatedRootsOverride):
* Calls generateSkills() into a temp directory and compares the freshly
* generated output file-by-file against the on-disk generated roots.
* Detects ALL forms of drift:
* (a) Direct edits to generated files
* (b) Canonical source changes without running `pnpm run sync:pi`
* (c) Stale files added to a generated root
* (d) Files deleted from a generated root
*
* Test mode (generatedRootsOverride set):
* Uses `.generated-manifest.json` as the oracle (manifest-based checks).
* Suitable for unit tests that use artificial generated-root fixtures
* without a full canonical source tree.
*
* Walk scope: only the declared generated roots returned by generate-skills.mjs.
* - `skills/<skill>/_source/` and `skills/<skill>/shared/` are NEVER walked.
* - `.generated-manifest.json` is excluded from content comparison.
* - `node_modules/` is always excluded.
*
* Exit codes:
* 0 — all generated roots match
* 1 — one or more mismatches detected
*
* Usage:
* node scripts/verify-generated.mjs
* pnpm run verify:generated
*
* Exported:
* verifyGenerated(repoRoot, options?) → Promise<{ok: boolean, errors: string[]}>
*/
import crypto from "node:crypto";
import { lstat, mkdtemp, readdir, readFile, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { pathToFileURL } from "node:url";
import { buildManifest, generateSkills, getGeneratedRoots } from "./generate-skills.mjs";
const REPO_ROOT = path.resolve(path.dirname(new URL(import.meta.url).pathname), "..");
const MANIFEST_FILENAME = ".generated-manifest.json";
// ── Helpers ───────────────────────────────────────────────────────────────
async function sha256File(absPath) {
const buf = await readFile(absPath);
return crypto.createHash("sha256").update(buf).digest("hex");
}
async function pathExists(p) {
try {
await lstat(p);
return true;
} catch {
return false;
}
}
/**
* Walk a directory and return all file relative paths (excluding node_modules
* and the manifest file itself, which is handled separately).
*/
async function walkRootFiles(rootDir) {
const results = [];
let entries;
try {
entries = await readdir(rootDir, { withFileTypes: true });
} catch {
return results;
}
for (const entry of entries) {
if (entry.name === "node_modules") continue;
if (entry.name === MANIFEST_FILENAME) continue; // manifest handled separately
const full = path.join(rootDir, entry.name);
if (entry.isDirectory()) {
const sub = await walkSubDir(full, entry.name);
results.push(...sub);
} else if (entry.isFile()) {
results.push(entry.name);
}
}
return results;
}
async function walkSubDir(dir, prefix) {
const results = [];
let entries;
try {
entries = await readdir(dir, { withFileTypes: true });
} catch {
return results;
}
for (const entry of entries) {
if (entry.name === "node_modules") continue;
const relPath = `${prefix}/${entry.name}`;
const full = path.join(dir, entry.name);
if (entry.isDirectory()) {
const sub = await walkSubDir(full, relPath);
results.push(...sub);
} else if (entry.isFile()) {
results.push(relPath);
}
}
return results;
}
/**
* Load and parse the .generated-manifest.json from a generated root.
* Returns null if missing or malformed.
*/
async function loadManifest(rootDir) {
const manifestPath = path.join(rootDir, MANIFEST_FILENAME);
try {
const raw = await readFile(manifestPath, "utf8");
const obj = JSON.parse(raw);
// Structural validation: must have $schema and generator markers
if (!obj.$schema || !obj.generator || !Array.isArray(obj.files)) {
return null;
}
return obj;
} catch {
return null;
}
}
/**
* Verify structural equality of two manifests (ignoring ephemeral fields).
* Returns array of difference descriptions, or empty array if equal.
*/
function diffManifests(expected, actual, rootRel) {
const diffs = [];
if (expected.$schema !== actual.$schema) {
diffs.push(`${rootRel}/.generated-manifest.json: $schema mismatch`);
}
if (expected.generator !== actual.generator) {
diffs.push(`${rootRel}/.generated-manifest.json: generator mismatch`);
}
if (expected.generatedRoot !== actual.generatedRoot) {
diffs.push(
`${rootRel}/.generated-manifest.json: generatedRoot mismatch ` +
`(expected ${expected.generatedRoot}, got ${actual.generatedRoot})`,
);
}
// Compare files arrays structurally — paths, kind, mode, and sha256
const expByPath = new Map(expected.files.map((f) => [f.path, f]));
const actByPath = new Map(actual.files.map((f) => [f.path, f]));
for (const p of expByPath.keys()) {
if (!actByPath.has(p)) {
diffs.push(`${rootRel}/.generated-manifest.json: expected file entry missing: ${p}`);
}
}
for (const p of actByPath.keys()) {
if (!expByPath.has(p)) {
diffs.push(`${rootRel}/.generated-manifest.json: unexpected file entry: ${p}`);
}
}
// For entries present in both, compare kind, mode, and sha256
for (const [p, expEntry] of expByPath) {
const actEntry = actByPath.get(p);
if (!actEntry) continue; // missing already reported above
if (expEntry.kind !== actEntry.kind) {
diffs.push(
`${rootRel}/.generated-manifest.json: ${p}: kind mismatch ` +
`(expected ${expEntry.kind}, got ${actEntry.kind})`,
);
}
if (expEntry.mode !== actEntry.mode) {
diffs.push(
`${rootRel}/.generated-manifest.json: ${p}: mode mismatch ` +
`(expected ${expEntry.mode}, got ${actEntry.mode})`,
);
}
if (expEntry.sha256 !== actEntry.sha256) {
diffs.push(
`${rootRel}/.generated-manifest.json: ${p}: sha256 mismatch ` +
`(expected ${expEntry.sha256.slice(0, 8)}…, got ${actEntry.sha256.slice(0, 8)}…)`,
);
}
}
return diffs;
}
// ── Core verifier ─────────────────────────────────────────────────────────
/**
* Verify all declared generated roots against freshly generated output.
*
* Calls generateSkills() into a temp directory and compares the resulting
* files to the on-disk roots. Detects all drift types:
* - Canonical source changed without running `pnpm run sync:pi`
* - Generated file edited directly
* - Stale file added to generated root
* - Generated file deleted from root
*
* Does NOT walk _source/, shared/, or node_modules/.
*
* @param {string} repoRoot - Absolute path to repo root.
* @param {string[]} rootsRelative - Generated root paths to verify.
* @param {string[]} errors - Error array to append to.
*/
async function verifyWithFreshGeneration(repoRoot, rootsRelative, errors) {
const tmpDir = await mkdtemp(path.join(tmpdir(), "verify-gen-"));
try {
await generateSkills(repoRoot, { targetRoot: tmpDir });
for (const rootRel of rootsRelative) {
const freshRootAbs = path.join(tmpDir, rootRel);
const onDiskRootAbs = path.join(repoRoot, rootRel);
if (!(await pathExists(onDiskRootAbs))) {
errors.push(`generated root missing: ${rootRel}`);
continue;
}
const freshFiles = await walkRootFiles(freshRootAbs);
const onDiskFiles = await walkRootFiles(onDiskRootAbs);
const freshSet = new Set(freshFiles);
const diskSet = new Set(onDiskFiles);
// Files the generator produces but are absent on disk
for (const f of freshFiles) {
if (!diskSet.has(f)) {
errors.push(`${rootRel}/${f}: missing (expected per canonical sources — run \`pnpm run sync:pi\`)`);
}
}
// On-disk files the generator would NOT produce (stale)
for (const f of onDiskFiles) {
if (!freshSet.has(f)) {
errors.push(`${rootRel}/${f}: stale (not produced from canonical sources)`);
}
}
// Content comparison for files present in both sets
for (const f of freshFiles) {
if (!diskSet.has(f)) continue; // missing already reported
const freshHash = await sha256File(path.join(freshRootAbs, f));
const diskHash = await sha256File(path.join(onDiskRootAbs, f));
if (freshHash !== diskHash) {
errors.push(
`${rootRel}/${f}: content drift ` +
`(on-disk sha256=${diskHash.slice(0, 8)}…, expected=${freshHash.slice(0, 8)}… — run \`pnpm run sync:pi\`)`,
);
}
}
// Validate .generated-manifest.json on disk matches what the generator produced
const freshManifestPath = path.join(freshRootAbs, MANIFEST_FILENAME);
const diskManifestPath = path.join(onDiskRootAbs, MANIFEST_FILENAME);
if (await pathExists(freshManifestPath)) {
if (!(await pathExists(diskManifestPath))) {
errors.push(`${rootRel}/${MANIFEST_FILENAME}: missing`);
} else {
const freshManifest = await loadManifest(freshRootAbs);
const diskManifest = await loadManifest(onDiskRootAbs);
if (!diskManifest) {
errors.push(`${rootRel}/${MANIFEST_FILENAME}: missing or malformed`);
} else if (freshManifest) {
const manifestDiffs = diffManifests(freshManifest, diskManifest, rootRel);
errors.push(...manifestDiffs);
}
}
}
}
} finally {
await rm(tmpDir, { recursive: true, force: true });
}
}
/**
* Verify using .generated-manifest.json as the oracle (test / no-canonical-sources mode).
*
* Used when generatedRootsOverride is set: the test fixture creates generated
* roots with manifests but does not provide a full canonical source tree.
*
* For each root:
* 1. Load the on-disk .generated-manifest.json (report missing manifest).
* 2. For every file listed in the manifest: verify existence, mode, sha256.
* 3. Walk the root for any files NOT listed in the manifest (stale files).
* 4. Verify the manifest's own structural fields match expected schema.
*
* @param {string} repoRoot - Absolute path to repo root.
* @param {string[]} rootsRelative - Generated root paths to verify.
* @param {string[]} errors - Error array to append to.
*/
async function verifyWithManifest(repoRoot, rootsRelative, errors) {
for (const rootRel of rootsRelative) {
const rootAbs = path.join(repoRoot, rootRel);
// Check the root directory exists
if (!(await pathExists(rootAbs))) {
errors.push(`generated root missing: ${rootRel}`);
continue;
}
// Load on-disk manifest
const manifest = await loadManifest(rootAbs);
if (!manifest) {
errors.push(`${rootRel}/.generated-manifest.json: missing or malformed`);
// Can't verify files without a manifest; report all on-disk files as stale
const onDisk = await walkRootFiles(rootAbs);
for (const f of onDisk) {
errors.push(`${rootRel}/${f}: stale (no valid manifest)`);
}
continue;
}
// Build expected manifest from current disk state (structural check)
const expectedManifest = await buildManifest(rootAbs, rootRel);
const manifestDiffs = diffManifests(expectedManifest, manifest, rootRel);
errors.push(...manifestDiffs);
// Build maps for file checks
const manifestByPath = new Map(manifest.files.map((f) => [f.path, f]));
// Check each file listed in manifest
for (const entry of manifest.files) {
const fileAbs = path.join(rootAbs, entry.path);
if (!(await pathExists(fileAbs))) {
errors.push(`${rootRel}/${entry.path}: missing (listed in manifest)`);
continue;
}
// Check content hash
const actualHash = await sha256File(fileAbs);
if (actualHash !== entry.sha256) {
errors.push(
`${rootRel}/${entry.path}: content mismatch ` +
`(manifest sha256=${entry.sha256.slice(0, 8)}…, actual=${actualHash.slice(0, 8)}…)`,
);
}
// Check mode
const st = await lstat(fileAbs);
const actualMode = (st.mode & 0o777).toString(8).padStart(3, "0");
if (actualMode !== entry.mode) {
errors.push(
`${rootRel}/${entry.path}: mode mismatch ` +
`(manifest=${entry.mode}, actual=${actualMode})`,
);
}
}
// Detect stale files: on-disk files not listed in manifest
const onDiskFiles = await walkRootFiles(rootAbs);
for (const relPath of onDiskFiles) {
if (!manifestByPath.has(relPath)) {
errors.push(`${rootRel}/${relPath}: stale (not in manifest)`);
}
}
}
}
/**
* Verify all declared generated roots in a repo.
*
* In production mode (no generatedRootsOverride): generates into a temp
* directory and compares file-by-file against on-disk roots. This detects
* both direct edits to generated files AND changes to canonical sources that
* were not followed by `pnpm run sync:pi`.
*
* In test mode (generatedRootsOverride set): uses manifest-based checks.
* Suitable for unit tests that provide artificial generated-root fixtures
* without a full canonical source tree.
*
* @param {string} [repoRoot] - Absolute path to repo root.
* @param {object} [options]
* @param {string[]} [options.generatedRootsOverride] - Override root list (test mode only).
* @returns {Promise<{ok: boolean, errors: string[]}>}
*/
export async function verifyGenerated(repoRoot = REPO_ROOT, options = {}) {
const rootsRelative = options.generatedRootsOverride ?? getGeneratedRoots();
const errors = [];
if (options.generatedRootsOverride) {
// Test mode: no canonical source tree available — use manifest oracle
await verifyWithManifest(repoRoot, rootsRelative, errors);
} else {
// Production mode: generate fresh from canonical sources and compare
await verifyWithFreshGeneration(repoRoot, rootsRelative, errors);
}
return { ok: errors.length === 0, errors };
}
// ── CLI entry point ────────────────────────────────────────────────────────
if (import.meta.url === pathToFileURL(process.argv[1]).href) {
const result = await verifyGenerated(REPO_ROOT);
if (result.ok) {
console.log("verify:generated: all generated roots are up to date.");
process.exit(0);
} else {
console.error("verify:generated: drift detected:\n");
for (const err of result.errors) {
console.error(` ✗ ${err}`);
}
console.error(`\n${result.errors.length} error(s). Run \`pnpm run sync:pi\` to regenerate.`);
process.exit(1);
}
}
scripts/verify-pi-resources.sh
+12 -5
View File
@@ -21,7 +21,6 @@ REQUIRED_FILES=(
"skills/reviewer-runtime/pi/run-review.sh"
"skills/reviewer-runtime/pi/notify-telegram.sh"
"scripts/install-pi-package.sh"
"scripts/sync-pi-package-skills.sh"
"pi-package/skills/atlassian/SKILL.md"
"pi-package/skills/create-plan/SKILL.md"
"pi-package/skills/do-task/SKILL.md"
@@ -39,13 +38,13 @@ done
test -x skills/reviewer-runtime/pi/run-review.sh
test -x skills/reviewer-runtime/pi/notify-telegram.sh
test -x scripts/install-pi-package.sh
test -x scripts/sync-pi-package-skills.sh
find skills/web-automation/pi/scripts -type f -print -quit | grep -q .
find skills/atlassian/pi/scripts -type f -print -quit | grep -q .
for file in skills/create-plan/pi/SKILL.md skills/do-task/pi/SKILL.md skills/implement-plan/pi/SKILL.md; do
grep -q 'docs/PI-SUPERPOWERS.md' "$file"
grep -q 'docs/PI-COMMON-REVIEWER.md' "$file"
# shellcheck disable=SC2016
grep -q 'Reviewer CLI: `codex`, `claude`, `cursor`, `opencode`, `pi`, or `skip`' "$file"
grep -q 'pi --no-session --no-skills --no-prompt-templates --no-extensions --no-context-files' "$file"
grep -q -- '--tools read,grep,find,ls -p' "$file"
@@ -54,11 +53,13 @@ done
grep -q 'reviewer model is configured independently' docs/PI-COMMON-REVIEWER.md
grep -q 'provider-qualified model IDs' docs/PI-COMMON-REVIEWER.md
# shellcheck disable=SC2016
grep -q 'MUST NOT include `write`, `edit`, or `bash`' docs/PI-COMMON-REVIEWER.md
grep -q 'Reviewer CLI | codex \\| claude \\| cursor \\| opencode \\| pi' skills/do-task/pi/templates/task-plan.md
grep -q 'pi-package/skills/atlassian/scripts' skills/atlassian/pi/SKILL.md
grep -q 'pi-package/skills/web-automation/scripts' skills/web-automation/pi/SKILL.md
# shellcheck disable=SC2016
grep -q 'local checkout package install keeps the runtime in `pi-package/skills/<skill>/scripts`' docs/PI.md
grep -q 'install-pi-package.sh --global' docs/PI.md
@@ -81,6 +82,8 @@ for family in atlassian create-plan do-task implement-plan web-automation; do
diff -qr \
--exclude '.DS_Store' \
--exclude 'node_modules' \
--exclude '.generated-manifest.json' \
--exclude 'package.json' \
"$source_dir" "$mirror_dir" >/dev/null
while IFS= read -r -d '' source_path; do
@@ -93,10 +96,14 @@ for family in atlassian create-plan do-task implement-plan web-automation; do
-mindepth 1 -print0)
done
! grep -nE 'update_plan|plan mode|sub-agent|subagents' \
# SC2251: restructured to avoid ! outside condition
if grep -nE 'update_plan|plan mode|sub-agent|subagents' \
skills/create-plan/pi/SKILL.md \
skills/do-task/pi/SKILL.md \
skills/implement-plan/pi/SKILL.md
skills/implement-plan/pi/SKILL.md; then
echo "Error: pi SKILL.md files must not contain Codex-specific terms" >&2
exit 1
fi
node <<'EOF'
const fs = require("fs");
@@ -129,7 +136,7 @@ for (const requiredFile of [
"pi-package/skills",
"docs/PI-COMMON-REVIEWER.md",
"scripts/install-pi-package.sh",
"scripts/sync-pi-package-skills.sh",
"scripts/generate-skills.mjs",
]) {
if (!pkg.files.includes(requiredFile)) {
console.error(`package.json files must include ${requiredFile}`);
scripts/verify-pi-workflows.sh
+7 -1
View File
@@ -13,6 +13,7 @@ WORKFLOW_FILES=(
for file in "${WORKFLOW_FILES[@]}"; do
test -f "$file"
grep -q 'docs/PI-SUPERPOWERS.md' "$file"
# shellcheck disable=SC2016
grep -q 'Reviewer CLI: `codex`, `claude`, `cursor`, `opencode`, `pi`, or `skip`' "$file"
grep -q 'pi --no-session --no-skills --no-prompt-templates --no-extensions --no-context-files' "$file"
grep -q -- '--tools read,grep,find,ls -p' "$file"
@@ -21,6 +22,7 @@ done
grep -q 'reviewer model is configured independently' docs/PI-COMMON-REVIEWER.md
grep -q 'provider-qualified model IDs' docs/PI-COMMON-REVIEWER.md
# shellcheck disable=SC2016
grep -q 'MUST NOT include `write`, `edit`, or `bash`' docs/PI-COMMON-REVIEWER.md
if command -v pi >/dev/null 2>&1; then
@@ -43,6 +45,10 @@ grep -q 'Reviewer CLI | codex \\| claude \\| cursor \\| opencode \\| pi' skills/
test -x skills/reviewer-runtime/pi/run-review.sh
test -x skills/reviewer-runtime/pi/notify-telegram.sh
! rg -n 'update_plan|plan mode|sub-agent|subagents' "${WORKFLOW_FILES[@]}"
# SC2251: restructured to avoid ! outside condition
if rg -n 'update_plan|plan mode|sub-agent|subagents' "${WORKFLOW_FILES[@]}"; then
echo "Error: pi workflow SKILL.md files must not contain Codex-specific terms" >&2
exit 1
fi
echo "pi workflow skill docs verified"