From ce4746b769b05aab391f81e60c4e1aa44d84fbd4 Mon Sep 17 00:00:00 2001 From: Stefano Fiorini Date: Thu, 23 Apr 2026 20:49:09 -0500 Subject: [PATCH] test: add reviewer support verification --- scripts/verify-reviewer-support.sh | 186 +++++++++++++++++++++++++++++ 1 file changed, 186 insertions(+) create mode 100755 scripts/verify-reviewer-support.sh diff --git a/scripts/verify-reviewer-support.sh b/scripts/verify-reviewer-support.sh new file mode 100755 index 0000000..a46e05c --- /dev/null +++ b/scripts/verify-reviewer-support.sh @@ -0,0 +1,186 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +export ROOT_DIR + +if ! command -v python3 >/dev/null 2>&1; then + echo "Missing required command: python3" >&2 + exit 2 +fi + +python3 <<'PY' +from pathlib import Path +import os +import re +import sys + +ROOT = Path(os.environ["ROOT_DIR"]) +WORKFLOWS = { + "create-plan": { + "payload": "/tmp/plan-", + "review_output": "/tmp/plan-review-", + }, + "implement-plan": { + "payload": "/tmp/milestone-", + "review_output": "/tmp/milestone-review-", + }, + "do-task": { + "payload": "/tmp/do-task-", + "review_output": "/tmp/do-task-", + }, +} +VARIANTS = ["claude-code", "codex", "cursor", "opencode", "pi"] +PI_FLAGS = [ + "--no-session", + "--no-skills", + "--no-prompt-templates", + "--no-extensions", + "--no-context-files", +] +FORBIDDEN_PI_TOOLS = {"write", "edit", "bash"} +errors: list[str] = [] + + +def compact(text: str) -> str: + return re.sub(r"\\\s*\n", " ", text) + + +def has_reviewer_choice(text: str) -> bool: + normalized = compact(text) + patterns = [ + r"Reviewer CLI:\s*`codex`,\s*`claude`,\s*`cursor`,\s*`opencode`,\s*`pi`,\s*or\s*`skip`", + r"Reviewer CLI\s*\|[^\n]*\bpi\b[^\n]*\bskip\b", + r"REVIEWER_CLI[^\n]*`pi`[^\n]*`skip`", + ] + return any(re.search(pattern, normalized, re.I) for pattern in patterns) + + +def fenced_code_blocks(text: str) -> list[str]: + return [match.group(1) for match in re.finditer(r"```(?:bash|sh|shell)?\s*\n(.*?)```", text, re.S)] + + +def pi_command_blocks(text: str) -> list[str]: + blocks: list[str] = [] + for block in fenced_code_blocks(text): + normalized = compact(block) + if re.search(r"\bpi\s+", normalized) and "--no-session" in normalized: + blocks.append(normalized) + + # Fallback for inline examples outside fenced code. Walk from the pi command + # until the next blank line/heading instead of relying on a fixed window. + lines = text.splitlines() + for index, line in enumerate(lines): + if re.search(r"\bpi\s+.*--no-session", line): + collected = [] + for candidate in lines[index:]: + if collected and (not candidate.strip() or candidate.startswith("#")): + break + collected.append(candidate) + block = compact("\n".join(collected)) + if block not in blocks: + blocks.append(block) + return blocks + + +def block_tools(block: str) -> list[str] | None: + match = re.search(r"--tools(?:=|\s+)(['\"]?)([^\s'\"]+)\1", block) + if not match: + return None + return [tool.strip() for tool in match.group(2).split(",") if tool.strip()] + + +def block_has_exact_tools(block: str) -> bool: + return block_tools(block) == ["read", "grep", "find", "ls"] + + +for workflow, spec in WORKFLOWS.items(): + for variant in VARIANTS: + path = ROOT / "skills" / workflow / variant / "SKILL.md" + if not path.exists(): + errors.append(f"missing workflow variant: {path}") + continue + text = path.read_text() + + if not has_reviewer_choice(text): + errors.append(f"{path}: reviewer choices must include pi and skip") + + if "pi/" not in text and "pi/claude-opus-4-7" not in text: + errors.append(f"{path}: must document pi/ reviewer shorthand") + + if "pi --list-models [search]" not in text: + errors.append(f"{path}: must mention pi --list-models [search] for unavailable models") + + if not re.search(r"reviewer model is configured independently|model is configured independently", text, re.I): + errors.append(f"{path}: must state Pi reviewer model is configured independently") + + blocks = pi_command_blocks(text) + if not blocks: + errors.append(f"{path}: missing isolated pi reviewer command block") + continue + + matching_blocks = [block for block in blocks if spec["payload"] in block] + if not matching_blocks: + errors.append(f"{path}: pi reviewer command must reference {spec['payload']} payload path") + matching_blocks = blocks + + if not any(all(flag in block for flag in PI_FLAGS) for block in matching_blocks): + errors.append(f"{path}: pi reviewer command missing one or more isolation flags") + + if not any(block_has_exact_tools(block) for block in matching_blocks): + errors.append(f"{path}: pi reviewer command must use exactly --tools read,grep,find,ls") + + # The forbidden-tool check is scoped to the --tools allowlist in the Pi + # reviewer command. Prose that says these tools are forbidden is allowed. + for block in matching_blocks: + tools = block_tools(block) + if tools is None: + continue + forbidden = sorted(set(tools) & FORBIDDEN_PI_TOOLS) + if forbidden: + errors.append(f"{path}: pi reviewer command includes forbidden tools: {', '.join(forbidden)}") + + if variant != "pi" and ".pi/skills/reviewer-runtime/pi" in text: + errors.append(f"{path}: non-Pi variant must not use Pi reviewer-runtime helper path as its own runtime") + +for variant in VARIANTS: + template = ROOT / "skills" / "do-task" / variant / "templates" / "task-plan.md" + if not template.exists(): + errors.append(f"missing do-task template: {template}") + continue + text = template.read_text() + if "Reviewer CLI | codex \\| claude \\| cursor \\| opencode \\| pi" not in text: + errors.append(f"{template}: Reviewer CLI metadata must include pi") + +canonical = ROOT / "docs" / "PI-COMMON-REVIEWER.md" +if not canonical.exists(): + errors.append("docs/PI-COMMON-REVIEWER.md is missing") +else: + text = canonical.read_text() + for flag in PI_FLAGS: + if flag not in text: + errors.append(f"docs/PI-COMMON-REVIEWER.md: missing {flag}") + if "--tools read,grep,find,ls" not in compact(text): + errors.append("docs/PI-COMMON-REVIEWER.md: missing exact read-only tool allowlist") + if "MUST NOT include `write`, `edit`, or `bash`" not in text: + errors.append("docs/PI-COMMON-REVIEWER.md: must forbid write/edit/bash tools") + +for doc in ["CREATE-PLAN.md", "IMPLEMENT-PLAN.md", "DO-TASK.md"]: + path = ROOT / "docs" / doc + if not path.exists(): + errors.append(f"docs/{doc} is missing") + continue + text = path.read_text() + if "PI-COMMON-REVIEWER.md" not in text: + errors.append(f"docs/{doc}: must link to PI-COMMON-REVIEWER.md") + if "pi/claude-opus-4-7" not in text and "pi/" not in text: + errors.append(f"docs/{doc}: must document Pi reviewer shorthand") + +if errors: + print("Reviewer support verification failed:", file=sys.stderr) + for error in errors: + print(f"- {error}", file=sys.stderr) + sys.exit(1) + +print("reviewer support verified") +PY