test: add reviewer support verification

2026-04-23 20:49:09 -05:00
parent 912aed93a7
commit ce4746b769
1 changed files with 186 additions and 0 deletions
@@ -0,0 +1,186 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)
+export ROOT_DIR
+
+if ! command -v python3 >/dev/null 2>&1; then
+  echo "Missing required command: python3" >&2
+  exit 2
+fi
+
+python3 <<'PY'
+from pathlib import Path
+import os
+import re
+import sys
+
+ROOT = Path(os.environ["ROOT_DIR"])
+WORKFLOWS = {
+    "create-plan": {
+        "payload": "/tmp/plan-",
+        "review_output": "/tmp/plan-review-",
+    },
+    "implement-plan": {
+        "payload": "/tmp/milestone-",
+        "review_output": "/tmp/milestone-review-",
+    },
+    "do-task": {
+        "payload": "/tmp/do-task-",
+        "review_output": "/tmp/do-task-",
+    },
+}
+VARIANTS = ["claude-code", "codex", "cursor", "opencode", "pi"]
+PI_FLAGS = [
+    "--no-session",
+    "--no-skills",
+    "--no-prompt-templates",
+    "--no-extensions",
+    "--no-context-files",
+]
+FORBIDDEN_PI_TOOLS = {"write", "edit", "bash"}
+errors: list[str] = []
+
+
+def compact(text: str) -> str:
+    return re.sub(r"\\\s*\n", " ", text)
+
+
+def has_reviewer_choice(text: str) -> bool:
+    normalized = compact(text)
+    patterns = [
+        r"Reviewer CLI:\s*`codex`,\s*`claude`,\s*`cursor`,\s*`opencode`,\s*`pi`,\s*or\s*`skip`",
+        r"Reviewer CLI\s*\|[^\n]*\bpi\b[^\n]*\bskip\b",
+        r"REVIEWER_CLI[^\n]*`pi`[^\n]*`skip`",
+    ]
+    return any(re.search(pattern, normalized, re.I) for pattern in patterns)
+
+
+def fenced_code_blocks(text: str) -> list[str]:
+    return [match.group(1) for match in re.finditer(r"```(?:bash|sh|shell)?\s*\n(.*?)```", text, re.S)]
+
+
+def pi_command_blocks(text: str) -> list[str]:
+    blocks: list[str] = []
+    for block in fenced_code_blocks(text):
+        normalized = compact(block)
+        if re.search(r"\bpi\s+", normalized) and "--no-session" in normalized:
+            blocks.append(normalized)
+
+    # Fallback for inline examples outside fenced code. Walk from the pi command
+    # until the next blank line/heading instead of relying on a fixed window.
+    lines = text.splitlines()
+    for index, line in enumerate(lines):
+        if re.search(r"\bpi\s+.*--no-session", line):
+            collected = []
+            for candidate in lines[index:]:
+                if collected and (not candidate.strip() or candidate.startswith("#")):
+                    break
+                collected.append(candidate)
+            block = compact("\n".join(collected))
+            if block not in blocks:
+                blocks.append(block)
+    return blocks
+
+
+def block_tools(block: str) -> list[str] | None:
+    match = re.search(r"--tools(?:=|\s+)(['\"]?)([^\s'\"]+)\1", block)
+    if not match:
+        return None
+    return [tool.strip() for tool in match.group(2).split(",") if tool.strip()]
+
+
+def block_has_exact_tools(block: str) -> bool:
+    return block_tools(block) == ["read", "grep", "find", "ls"]
+
+
+for workflow, spec in WORKFLOWS.items():
+    for variant in VARIANTS:
+        path = ROOT / "skills" / workflow / variant / "SKILL.md"
+        if not path.exists():
+            errors.append(f"missing workflow variant: {path}")
+            continue
+        text = path.read_text()
+
+        if not has_reviewer_choice(text):
+            errors.append(f"{path}: reviewer choices must include pi and skip")
+
+        if "pi/<pi-model-name>" not in text and "pi/claude-opus-4-7" not in text:
+            errors.append(f"{path}: must document pi/<pi-model-name> reviewer shorthand")
+
+        if "pi --list-models [search]" not in text:
+            errors.append(f"{path}: must mention pi --list-models [search] for unavailable models")
+
+        if not re.search(r"reviewer model is configured independently|model is configured independently", text, re.I):
+            errors.append(f"{path}: must state Pi reviewer model is configured independently")
+
+        blocks = pi_command_blocks(text)
+        if not blocks:
+            errors.append(f"{path}: missing isolated pi reviewer command block")
+            continue
+
+        matching_blocks = [block for block in blocks if spec["payload"] in block]
+        if not matching_blocks:
+            errors.append(f"{path}: pi reviewer command must reference {spec['payload']} payload path")
+            matching_blocks = blocks
+
+        if not any(all(flag in block for flag in PI_FLAGS) for block in matching_blocks):
+            errors.append(f"{path}: pi reviewer command missing one or more isolation flags")
+
+        if not any(block_has_exact_tools(block) for block in matching_blocks):
+            errors.append(f"{path}: pi reviewer command must use exactly --tools read,grep,find,ls")
+
+        # The forbidden-tool check is scoped to the --tools allowlist in the Pi
+        # reviewer command. Prose that says these tools are forbidden is allowed.
+        for block in matching_blocks:
+            tools = block_tools(block)
+            if tools is None:
+                continue
+            forbidden = sorted(set(tools) & FORBIDDEN_PI_TOOLS)
+            if forbidden:
+                errors.append(f"{path}: pi reviewer command includes forbidden tools: {', '.join(forbidden)}")
+
+        if variant != "pi" and ".pi/skills/reviewer-runtime/pi" in text:
+            errors.append(f"{path}: non-Pi variant must not use Pi reviewer-runtime helper path as its own runtime")
+
+for variant in VARIANTS:
+    template = ROOT / "skills" / "do-task" / variant / "templates" / "task-plan.md"
+    if not template.exists():
+        errors.append(f"missing do-task template: {template}")
+        continue
+    text = template.read_text()
+    if "Reviewer CLI | codex \\| claude \\| cursor \\| opencode \\| pi" not in text:
+        errors.append(f"{template}: Reviewer CLI metadata must include pi")
+
+canonical = ROOT / "docs" / "PI-COMMON-REVIEWER.md"
+if not canonical.exists():
+    errors.append("docs/PI-COMMON-REVIEWER.md is missing")
+else:
+    text = canonical.read_text()
+    for flag in PI_FLAGS:
+        if flag not in text:
+            errors.append(f"docs/PI-COMMON-REVIEWER.md: missing {flag}")
+    if "--tools read,grep,find,ls" not in compact(text):
+        errors.append("docs/PI-COMMON-REVIEWER.md: missing exact read-only tool allowlist")
+    if "MUST NOT include `write`, `edit`, or `bash`" not in text:
+        errors.append("docs/PI-COMMON-REVIEWER.md: must forbid write/edit/bash tools")
+
+for doc in ["CREATE-PLAN.md", "IMPLEMENT-PLAN.md", "DO-TASK.md"]:
+    path = ROOT / "docs" / doc
+    if not path.exists():
+        errors.append(f"docs/{doc} is missing")
+        continue
+    text = path.read_text()
+    if "PI-COMMON-REVIEWER.md" not in text:
+        errors.append(f"docs/{doc}: must link to PI-COMMON-REVIEWER.md")
+    if "pi/claude-opus-4-7" not in text and "pi/<pi-model-name>" not in text:
+        errors.append(f"docs/{doc}: must document Pi reviewer shorthand")
+
+if errors:
+    print("Reviewer support verification failed:", file=sys.stderr)
+    for error in errors:
+        print(f"- {error}", file=sys.stderr)
+    sys.exit(1)
+
+print("reviewer support verified")
+PY