From aaa47e93c6df426945d51205d533f4636451024a Mon Sep 17 00:00:00 2001 From: Christopher Tso Date: Mon, 27 Apr 2026 19:11:46 +0200 Subject: [PATCH] feat(examples): add compliance-lint GitHub Action reference (Phase 3 of #1172) Reference workflow at examples/governance/compliance-lint/ that loads the agentv-compliance skill and lints governance: blocks in changed *.eval.yaml files on pull_request events. Posts violations as PR comments; exits non-zero on pass: false. Uses claude-haiku-4-5 (~3c/10-file PR). Closes #1172 (Phase 3) Co-Authored-By: Claude Sonnet 4.6 --- examples/governance/compliance-lint/README.md | 111 ++++++++ .../compliance-lint/compliance-lint.yml | 59 ++++ .../governance/compliance-lint/script/lint.py | 253 ++++++++++++++++++ 3 files changed, 423 insertions(+) create mode 100644 examples/governance/compliance-lint/README.md create mode 100644 examples/governance/compliance-lint/compliance-lint.yml create mode 100644 examples/governance/compliance-lint/script/lint.py diff --git a/examples/governance/compliance-lint/README.md b/examples/governance/compliance-lint/README.md new file mode 100644 index 00000000..06b97565 --- /dev/null +++ b/examples/governance/compliance-lint/README.md @@ -0,0 +1,111 @@ +# Governance Compliance Lint Action + +A reference GitHub Action that lints `governance:` blocks in changed `*.eval.yaml` files +using the `agentv-compliance` skill. The same skill that powers AI authoring also powers +CI enforcement — no separate runtime package needed. + +## How it works + +1. On every PR touching `*.eval.yaml` files, the Action extracts `governance:` blocks from + each changed file (suite-level and per-case). +2. Each block is passed to Claude with the `agentv-compliance` skill loaded. +3. Claude applies the rules in `plugins/agentv-dev/skills/agentv-compliance/references/lint-rules.md` + and returns a structured JSON report (`{ pass: bool, violations: [...] }`). +4. The Action posts a summary as a PR comment and exits non-zero on any `pass: false` result. + +## Adoption (5 minutes) + +### 1. Copy the workflow + +```bash +cp examples/governance/compliance-lint/compliance-lint.yml .github/workflows/compliance-lint.yml +``` + +### 2. Set the `ANTHROPIC_API_KEY` secret + +In your repository: **Settings → Secrets and variables → Actions → New repository secret** +Name: `ANTHROPIC_API_KEY`, value: your key from console.anthropic.com. + +### 3. Point at your skill location (optional) + +By default the workflow looks for the skill at +`plugins/agentv-dev/skills/agentv-compliance/` relative to the repo root. +If your skill lives elsewhere, set `SKILL_PATH` in the workflow env: + +```yaml +env: + SKILL_PATH: path/to/your/agentv-compliance +``` + +### 4. Push a PR with a `*.eval.yaml` change + +The Action runs automatically and posts a comment like: + +``` +## Governance Compliance Lint + +**examples/red-team/suites/my-suite.eval.yaml** ✅ + - `governance`: ✅ pass + +✅ All governance blocks passed. +``` + +Or for violations: + +``` +## Governance Compliance Lint + +**examples/red-team/suites/my-suite.eval.yaml** ❌ + - `governance`: ❌ 2 violation(s) + - **risk_tier_value** `risk_tier`: Unknown risk_tier value 'critical'. + *Suggestion:* Use one of: prohibited, high_risk, limited_risk, minimal_risk. + - **owasp_llm_ids** `owasp_llm_top_10_2025`: Invalid OWASP LLM ID 'LLM99'. + *Suggestion:* Use a valid ID from references/owasp-llm-top-10-2025.md. +``` + +## Cost + +Using `claude-haiku-4-5`, a 10-file PR with one governance block each costs approximately: +- ~500 tokens input per block (skill context + block YAML + instructions) +- ~200 tokens output (JSON report) +- ~$0.003 per block → **~$0.03 for 10 blocks** — well under the 5 cent target. + +The skill context is sent once per block. For large PRs, batch or cache the skill text +in-process (already done by `lint.py` — it loads the skill once and reuses it). + +## Making lint mandatory + +This Action is **opt-in** by default. To make it mandatory: + +1. In **Settings → Branches → Branch protection rules** for `main`, add + `compliance-lint` as a required status check. +2. Violations then block merge until the author fixes the governance block. + +## Customising the rules + +Edit `plugins/agentv-dev/skills/agentv-compliance/references/lint-rules.md` to add, remove, +or adjust rules. The Action picks up changes automatically on the next run — no code change needed. + +## Files + +``` +examples/governance/compliance-lint/ +├── compliance-lint.yml # GitHub Actions workflow (copy to .github/workflows/) +├── script/ +│ └── lint.py # Python script: extracts blocks, calls Claude, posts comment +└── README.md # This file +``` + +The skill lives at: +``` +plugins/agentv-dev/skills/agentv-compliance/ +├── SKILL.md +└── references/ + ├── governance-yaml-shape.md + ├── lint-rules.md ← rules applied by lint.py + ├── owasp-llm-top-10-2025.md + ├── owasp-agentic-top-10-2025.md + ├── mitre-atlas.md + ├── eu-ai-act-risk-tiers.md + └── iso-42001-controls.md +``` diff --git a/examples/governance/compliance-lint/compliance-lint.yml b/examples/governance/compliance-lint/compliance-lint.yml new file mode 100644 index 00000000..54c70390 --- /dev/null +++ b/examples/governance/compliance-lint/compliance-lint.yml @@ -0,0 +1,59 @@ +name: Governance Compliance Lint + +# Lints governance: blocks in changed *.eval.yaml files using the agentv-compliance skill. +# Posts violations as PR comments and exits non-zero on any failure. +# +# Adoption: copy this file to .github/workflows/compliance-lint.yml in your repo. +# Required secret: ANTHROPIC_API_KEY +# Optional: point SKILL_PATH at a fork of plugins/agentv-dev/skills/agentv-compliance/ + +on: + pull_request: + paths: + - '**/*.eval.yaml' + - '**/*.EVAL.yaml' + +permissions: + contents: read + pull-requests: write + +jobs: + compliance-lint: + runs-on: ubuntu-latest + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + + - name: Install dependencies + run: pip install anthropic pyyaml + + - name: Find changed eval files + id: changed + run: | + BASE="${{ github.event.pull_request.base.sha }}" + HEAD="${{ github.event.pull_request.head.sha }}" + FILES=$(git diff --name-only "$BASE" "$HEAD" -- '*.eval.yaml' '*.EVAL.yaml' | tr '\n' ' ') + echo "files=$FILES" >> "$GITHUB_OUTPUT" + + - name: Run governance compliance lint + id: lint + if: steps.changed.outputs.files != '' + env: + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + CHANGED_FILES: ${{ steps.changed.outputs.files }} + SKILL_PATH: ${{ github.workspace }}/plugins/agentv-dev/skills/agentv-compliance + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + PR_NUMBER: ${{ github.event.pull_request.number }} + REPO: ${{ github.repository }} + run: python examples/governance/compliance-lint/script/lint.py + + - name: No eval files changed + if: steps.changed.outputs.files == '' + run: echo "No *.eval.yaml files changed — skipping governance lint." diff --git a/examples/governance/compliance-lint/script/lint.py b/examples/governance/compliance-lint/script/lint.py new file mode 100644 index 00000000..0cef3dd9 --- /dev/null +++ b/examples/governance/compliance-lint/script/lint.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +""" +Governance compliance lint script. + +Reads each changed *.eval.yaml file, extracts governance: blocks, and calls +the Claude API with the agentv-compliance skill loaded to lint them. +Posts violations as a PR comment and exits non-zero on any failure. + +Environment variables: + ANTHROPIC_API_KEY - required + CHANGED_FILES - space-separated list of changed eval file paths + SKILL_PATH - path to the agentv-compliance skill directory + GITHUB_TOKEN - for posting PR comments (optional; skipped if absent) + PR_NUMBER - GitHub PR number (optional) + REPO - GitHub repo in "owner/repo" form (optional) + +Cost target: under 5 cents per 10-file PR using claude-haiku-4-5. +""" + +import json +import os +import re +import sys +import textwrap +from pathlib import Path + +import anthropic +import yaml + +# --------------------------------------------------------------------------- +# Config +# --------------------------------------------------------------------------- + +MODEL = "claude-haiku-4-5-20251001" +SKILL_PATH = Path(os.environ.get("SKILL_PATH", "plugins/agentv-dev/skills/agentv-compliance")) +CHANGED_FILES = os.environ.get("CHANGED_FILES", "").split() + + +def load_skill_content() -> str: + """Concatenate SKILL.md and all references/ into a single context string.""" + parts: list[str] = [] + + skill_md = SKILL_PATH / "SKILL.md" + if skill_md.exists(): + parts.append(f"# Skill: agentv-compliance\n\n{skill_md.read_text()}") + + refs_dir = SKILL_PATH / "references" + if refs_dir.is_dir(): + for ref_file in sorted(refs_dir.glob("*.md")): + parts.append(f"\n\n---\n## {ref_file.stem}\n\n{ref_file.read_text()}") + + return "\n".join(parts) + + +def extract_governance_blocks(eval_path: Path) -> list[dict]: + """ + Extract governance blocks from an eval file. + Returns a list of dicts: {location, block}. + """ + try: + text = eval_path.read_text() + doc = yaml.safe_load(text) + except Exception as exc: + print(f" Warning: failed to parse {eval_path}: {exc}", file=sys.stderr) + return [] + + if not isinstance(doc, dict): + return [] + + blocks: list[dict] = [] + + # Suite-level governance (top-level key) + if isinstance(doc.get("governance"), dict): + blocks.append({"location": "governance", "block": doc["governance"]}) + + # Case-level governance (tests[n].metadata.governance) + for i, case in enumerate(doc.get("tests") or []): + if not isinstance(case, dict): + continue + meta = case.get("metadata") + if not isinstance(meta, dict): + continue + gov = meta.get("governance") + if isinstance(gov, dict): + blocks.append({"location": f"tests[{i}].metadata.governance", "block": gov}) + + return blocks + + +def lint_block(client: anthropic.Anthropic, skill_context: str, location: str, block: dict) -> dict: + """Call Claude to lint one governance block. Returns structured lint report.""" + block_yaml = yaml.dump(block, default_flow_style=False, allow_unicode=True) + + prompt = textwrap.dedent(f""" + You are linting a governance block from an AgentV eval file. + Apply the rules in references/lint-rules.md and return ONLY a JSON object. + + Governance block at location `{location}`: + ```yaml + {block_yaml} + ``` + + Return ONLY valid JSON in this exact shape — no markdown, no explanation: + {{ + "pass": , + "violations": [ + {{ + "rule": "", + "key": "", + "value": "", + "message": "", + "suggestion": "" + }} + ] + }} + + If there are no violations, return {{"pass": true, "violations": []}}. + """).strip() + + message = client.messages.create( + model=MODEL, + max_tokens=1024, + system=f"You are a governance compliance linter. Load this skill:\n\n{skill_context}", + messages=[{"role": "user", "content": prompt}], + ) + + raw = message.content[0].text.strip() + # Strip markdown code fences if present + raw = re.sub(r"^```(?:json)?\n?", "", raw) + raw = re.sub(r"\n?```$", "", raw) + + try: + report = json.loads(raw) + except json.JSONDecodeError: + return { + "pass": False, + "violations": [ + { + "rule": "parse_error", + "key": "response", + "value": raw[:200], + "message": "Linter returned non-JSON response", + "suggestion": "Check ANTHROPIC_API_KEY and model availability", + } + ], + } + + return report + + +def post_pr_comment(body: str) -> None: + """Post a comment on the PR if GITHUB_TOKEN, PR_NUMBER, and REPO are set.""" + token = os.environ.get("GITHUB_TOKEN") + pr_number = os.environ.get("PR_NUMBER") + repo = os.environ.get("REPO") + if not (token and pr_number and repo): + return + + try: + import urllib.request + + url = f"https://api.github.com/repos/{repo}/issues/{pr_number}/comments" + payload = json.dumps({"body": body}).encode() + req = urllib.request.Request( + url, + data=payload, + headers={ + "Authorization": f"Bearer {token}", + "Accept": "application/vnd.github+json", + "Content-Type": "application/json", + "X-GitHub-Api-Version": "2022-11-28", + }, + method="POST", + ) + with urllib.request.urlopen(req, timeout=10): + pass + except Exception as exc: + print(f" Warning: failed to post PR comment: {exc}", file=sys.stderr) + + +def main() -> int: + if not CHANGED_FILES: + print("No changed eval files to lint.") + return 0 + + api_key = os.environ.get("ANTHROPIC_API_KEY") + if not api_key: + print("Error: ANTHROPIC_API_KEY is not set.", file=sys.stderr) + return 1 + + client = anthropic.Anthropic(api_key=api_key) + skill_context = load_skill_content() + + all_pass = True + comment_lines: list[str] = ["## Governance Compliance Lint\n"] + + for file_str in CHANGED_FILES: + eval_path = Path(file_str) + if not eval_path.exists(): + continue + + print(f"\nLinting {eval_path}...") + blocks = extract_governance_blocks(eval_path) + + if not blocks: + print(f" No governance blocks found — skipping.") + comment_lines.append(f"**{eval_path}**: no `governance:` block — skipped.\n") + continue + + file_pass = True + file_lines: list[str] = [] + for entry in blocks: + location = entry["location"] + block = entry["block"] + print(f" Linting {location}...") + report = lint_block(client, skill_context, location, block) + + if report.get("pass"): + print(f" ✓ pass") + file_lines.append(f" - `{location}`: ✅ pass") + else: + file_pass = False + all_pass = False + violations = report.get("violations", []) + print(f" ✗ {len(violations)} violation(s)") + file_lines.append(f" - `{location}`: ❌ {len(violations)} violation(s)") + for v in violations: + msg = v.get("message", "") + sug = v.get("suggestion", "") + print(f" [{v.get('rule')}] {v.get('key')}: {msg}") + file_lines.append(f" - **{v.get('rule')}** `{v.get('key')}`: {msg}") + if sug: + file_lines.append(f" *Suggestion:* {sug}") + + status = "✅" if file_pass else "❌" + comment_lines.append(f"**{eval_path}** {status}") + comment_lines.extend(file_lines) + comment_lines.append("") + + if all_pass: + comment_lines.append("\n✅ All governance blocks passed.") + else: + comment_lines.append("\n❌ Some governance blocks have violations. See details above.") + + comment_body = "\n".join(comment_lines) + post_pr_comment(comment_body) + print("\n" + comment_body) + + return 0 if all_pass else 1 + + +if __name__ == "__main__": + sys.exit(main())