Phase 2: submission-gate, risk tiers, and PR status state machine (#4184)

.github/risk-tiers.yml

@@ -0,0 +1,152 @@
+# Merge risk tiers.
+#
+# Automation (eng/submission-gate.mjs) applies exactly one of `merge-risk:low`,
+# `merge-risk:medium`, or `merge-risk:high` to every open PR and the
+# `submission-gate` check enforces the approvals each tier requires.
+# See docs/maintainers/submission-gate.md.
+#
+# Evaluation order:
+#   1. high   if any changed file matches `high.paths`, any added line matches a
+#             `high.capabilities` trigger, a `high.labels` label is present, or the
+#             changes can't be fully scanned (GitHub truncated the file list, or a
+#             text file in a capability's scope has no diff to scan).
+#   2. low    if every changed file matches `low.paths`, or the PR only modifies
+#             existing resource files (no added/removed/renamed files) and the total
+#             change is at most `low.small_update.max_changed_lines`.
+#   3. medium otherwise.
+#
+# This file is a review-policy file: changing it places a PR in merge-risk:high.
+
+high:
+  description: Privileged execution, automation, or review-policy change
+  paths:
+    # Review policy and automation
+    - ".github/**"
+    - "CODEOWNERS"
+    - "docs/CODEOWNERS"
+    # Build, release, and repository scripts
+    - "eng/**"
+    - "scripts/**"
+    - "package.json"
+    - "package-lock.json"
+    # MCP server configuration
+    - "**/mcp.json"
+    - "**/.mcp.json"
+    # Bundled executable scripts (hooks, skills, and plugins). Agentic workflow
+    # sources (`workflows/**`) and hook metadata (`hooks/**`) are content; a hook
+    # command in hooks.json is caught by the `mcp-server-command` capability.
+    - "**/*.sh"
+    - "**/*.bash"
+    - "**/*.ps1"
+    - "**/*.psm1"
+    - "**/*.bat"
+    - "**/*.cmd"
+    - "**/*.py"
+    - "skills/**/scripts/**"
+    - "plugins/**/scripts/**"
+    # Plugin marketplace sources for externally hosted code
+    - "plugins/external.json"
+  # Generated output that lives under a high-risk path but is safe to regenerate.
+  exclude_paths:
+    - ".github/plugin/marketplace.json"
+  # Regexes matched against added lines of the diff. `files` limits where each applies.
+  capabilities:
+    - id: process-execution
+      description: Spawns processes or evaluates code
+      files: ["extensions/**", "plugins/**", "skills/**", "website/**"]
+      pattern: "child_process|\\bexecSync\\(|\\bexecFile(Sync)?\\(|\\bspawn(Sync)?\\(|\\beval\\(|new Function\\(|node:vm|require\\(['\"]vm['\"]\\)"
+    - id: remote-script-execution
+      description: Pipes a downloaded script into a shell
+      pattern: "(curl|wget)[^\\n|]*\\|\\s*(sudo\\s+)?(ba|z)?sh\\b|(iwr|irm|Invoke-WebRequest|Invoke-RestMethod)[^\\n|]*\\|\\s*(iex|Invoke-Expression)|Invoke-Expression"
+    - id: mcp-server-command
+      description: Declares an MCP server or hook command
+      files: ["plugins/**/*.json", "extensions/**/*.json", "skills/**/*.json", "hooks/**/*.json"]
+      pattern: "\"(mcpServers|command|bash|powershell)\"\\s*:"
+  # Files whose missing diff is acceptable: binaries GitHub never diffs, and generated
+  # documentation whose sources are scanned. Other files in a capability's scope without
+  # a diff (binary or too large) fail closed to high because their added lines can't be scanned.
+  unscanned_paths:
+    - "docs/**"
+    - "README.md"
+    - ".github/plugin/marketplace.json"
+    - "**/*.png"
+    - "**/*.jpg"
+    - "**/*.jpeg"
+    - "**/*.gif"
+    - "**/*.webp"
+    - "**/*.ico"
+    - "**/*.pdf"
+    - "**/*.woff"
+    - "**/*.woff2"
+    - "**/*.ttf"
+    - "**/*.otf"
+    - "**/*.mp4"
+    - "**/*.webm"
+    - "**/*.zip"
+  labels:
+    # Applied by the Contributor Reputation Check writer.
+    - "needs-review:HIGH"
+  approvals:
+    required: 2
+    require_core: true
+
+medium:
+  description: New or substantially changed resource without privileged execution
+  approvals:
+    required: 1
+    require_domain: true
+
+low:
+  description: Documentation, metadata, generated output, or a small update to an existing resource
+  paths:
+    - "docs/**"
+    - "README.md"
+    - "CONTRIBUTING.md"
+    - "CODE_OF_CONDUCT.md"
+    - "SECURITY.md"
+    - "SUPPORT.md"
+    - "LICENSE"
+    - ".all-contributorsrc"
+    - ".github/plugin/marketplace.json"
+    - "**/*.png"
+    - "**/*.jpg"
+    - "**/*.jpeg"
+    - "**/*.gif"
+    - "**/*.svg"
+    - "**/*.webp"
+  small_update:
+    max_changed_lines: 40
+    # Existing resources that may be updated in a small change and stay low risk.
+    resource_paths:
+      - "agents/**"
+      - "instructions/**"
+      - "skills/**"
+      - "plugins/**"
+      - "extensions/**"
+  approvals:
+    # One approval from an owner of the changed resource (its domain pool, or the core
+    # pools for files outside every domain). Until pools are staffed, any approver with
+    # write access counts.
+    required: 1
+    require_owner: true
+
+# Areas used to pick the domain reviewer pool from .github/review-routing.yml.
+# `pools` are routing pool keys. When a matching pool has no reviewers yet, any
+# approver with write access satisfies the domain requirement.
+domains:
+  canvas:
+    paths: ["extensions/**"]
+    pools: ["canvas"]
+  plugin:
+    paths: ["plugins/**"]
+    pools: ["plugin"]
+  content:
+    paths: ["agents/**", "instructions/**", "skills/**", "workflows/**", "hooks/**"]
+    pools: ["content"]
+# Files outside every domain (repository automation such as `.github/workflows/**`,
+# `eng/**`, review policy, and docs) are owned by the core pools.
+
+# Routing pools whose members count as core maintainers for high-risk approvals.
+# Until those pools are staffed, users with admin or maintain permission on the
+# repository count instead.
+core_pools: ["core-maintainers"]

.github/submission-gate.yml

@@ -0,0 +1,234 @@
+# Submission gate configuration.
+#
+# The submission gate aggregates the checks below on the PR head commit into one
+# required `submission-gate` check. The read-only Submission Gate workflow waits for
+# them; the trusted Submission Gate Writer publishes the check.
+# Logic lives in eng/submission-gate.mjs. See docs/maintainers/submission-gate.md.
+#
+# This file is a review-policy file: changing it places a PR in merge-risk:high.
+#
+# Check fields:
+#   id           Stable identifier used in the status comment.
+#   title        Human-readable name.
+#   workflow     Workflow file (under .github/workflows/) that reports the check.
+#                Omit and set `check_name` for checks matched by check-run name only.
+#   check_name   Match a check run by name instead of by workflow (any workflow).
+#   branches     Base branches the check runs for (mirrors the workflow trigger).
+#   paths        Path globs that make the check applicable (mirrors the workflow's
+#                `paths` filter). Omit to apply to every PR. Kept in sync with the
+#                workflow triggers by eng/submission-gate.test.mjs.
+#   required     true: failures block the gate. false: failures are shown as warnings.
+#   optional     true: skip silently when the check never reports (pluggable slot).
+#   allow_skip   true: a run its workflow skipped counts as skipped. Otherwise a skipped
+#                required check is an infrastructure failure, because it validated nothing.
+#   failure_kind auto (default): classify a failed run by its failing step.
+#                infrastructure: every failure of this check is an infrastructure failure.
+#   contribution_steps  Step-name patterns (regex) that mean the contribution failed.
+#   hint         Fix guidance shown for contribution failures.
+
+wait:
+  # Maximum time the gate polls for pending checks before reporting them as incomplete.
+  timeout_minutes: 40
+  interval_seconds: 30
+  # A check that has not appeared after this long (while everything else finished)
+  # is reported as "did not report".
+  report_grace_minutes: 8
+
+# Failed steps that match these patterns are infrastructure failures (runner, network,
+# dependency install, artifact transfer) rather than problems with the contribution.
+infrastructure_steps:
+  - "^Set up job$"
+  - "^Complete job$"
+  - "^Post "
+  - "(Checkout|checkout)"
+  - "^Setup (Node|Python)"
+  - "^Set up (Node|Python)"
+  - "(Install|install) (dependencies|gh-aw)"
+  - "^Fetch AGT"
+  - "(Upload|Download) .*artifact"
+
+checks:
+  - id: line-endings
+    title: Line endings
+    workflow: check-line-endings.yml
+    branches: [main]
+    required: true
+    contribution_steps: ["CRLF"]
+    hint: Run `bash eng/fix-line-endings.sh` and commit the result.
+
+  - id: spelling
+    title: Spelling
+    workflow: codespell.yml
+    branches: [main]
+    required: true
+    contribution_steps: ["^Check spelling with codespell$"]
+    hint: Fix the misspellings reported by codespell in the job log.
+
+  - id: readme
+    title: Generated README consistency
+    workflow: validate-readme.yml
+    branches: [main]
+    paths:
+      - "instructions/**"
+      - "prompts/**"
+      - "agents/**"
+      - "plugins/**"
+      - "workflows/**"
+      - "*.js"
+      - "README.md"
+      - "docs/**"
+      - "skills/**"
+      - "eng/update-readme.mjs"
+      - "eng/generate-marketplace.mjs"
+      - "eng/validate-plugins.mjs"
+      - ".github/workflows/validate-readme.yml"
+    required: true
+    contribution_steps: ["^Validate plugins$", "^Update README", "^Fail workflow if files need updating$"]
+    hint: Run `npm start` locally and commit the regenerated files.
+
+  - id: plugin-validation
+    title: Plugin and extension validation
+    workflow: validate-plugins.yml
+    branches: [main]
+    paths:
+      - "plugins/**"
+      - "extensions/**"
+      - "eng/validate-plugins.mjs"
+      - ".github/workflows/validate-plugins.yml"
+    required: true
+    contribution_steps: ["^Validate plugins and extensions$"]
+    hint: Run `npm run plugin:validate` locally and fix the reported errors.
+
+  - id: canvas-extension-validation
+    title: Canvas extension validation
+    workflow: validate-canvas-extensions.yml
+    branches: [main]
+    paths:
+      - "extensions/**"
+      - "eng/validate-plugins.mjs"
+      - ".github/workflows/validate-canvas-extensions.yml"
+    required: true
+    contribution_steps: ["^Validate changed extensions$"]
+    hint: Run `npm run plugin:validate` locally and fix the reported errors.
+
+  - id: plugin-structure
+    title: Plugin structure
+    workflow: check-plugin-structure.yml
+    branches: [main]
+    paths:
+      - "plugins/**"
+      - ".github/workflows/check-plugin-structure.yml"
+    required: true
+    contribution_steps: ["materialized files"]
+    hint: Remove materialized or symlinked files from the plugin directory.
+
+  - id: skill-validation
+    title: Skill validation
+    workflow: validate-skills.yml
+    branches: [main]
+    paths:
+      - "skills/**"
+      - "eng/validate-skills.mjs"
+      - "eng/yaml-parser.mjs"
+      - ".github/workflows/validate-skills.yml"
+    required: true
+    contribution_steps: ["^Validate skills$"]
+    hint: Run `npm run skill:validate` locally and fix the reported errors.
+
+  - id: skill-lint
+    title: Skill lint (vally)
+    workflow: skill-check.yml
+    branches: [main]
+    paths:
+      - "skills/**"
+      - "agents/**"
+      - "plugins/**/skills/**"
+      - "plugins/**/agents/**"
+      - ".github/workflows/skill-check.yml"
+    required: true
+    failure_kind: infrastructure
+
+  - id: submission-gate-validation
+    title: Submission gate tests
+    workflow: validate-submission-gate.yml
+    branches: [main]
+    paths:
+      - ".github/submission-gate.yml"
+      - ".github/risk-tiers.yml"
+      - ".github/workflows/**"
+      - "eng/submission-gate.mjs"
+      - "eng/submission-gate.test.mjs"
+    required: true
+    contribution_steps: ["^Test submission gate$"]
+    hint: Run `node --test eng/submission-gate.test.mjs` and keep `.github/submission-gate.yml` in sync with workflow triggers.
+
+  - id: agentic-workflow-validation
+    title: Agentic workflow validation
+    workflow: validate-agentic-workflows-pr.yml
+    branches: [main]
+    paths:
+      - "workflows/**"
+    required: true
+    contribution_steps: ["^Check for forbidden files$", "^Compile workflow files$"]
+    hint: Only add `.md` sources under `workflows/` and make sure `gh aw compile --validate` passes.
+
+  - id: risk-scan
+    title: Risk scan
+    workflow: pr-risk-scan.yml
+    branches: [main]
+    paths:
+      - "skills/**"
+      - "agents/**"
+      - "workflows/**"
+      - "plugins/**"
+      - "hooks/**"
+      - "instructions/**"
+      - "eng/pr-risk-scan.mjs"
+      - ".github/workflows/pr-risk-scan.yml"
+    required: true
+    failure_kind: infrastructure
+
+  - id: contributor-reputation
+    title: Contributor reputation
+    workflow: contributor-check.yml
+    required: true
+    # Its jobs skip for bot authors (Dependabot, github-actions, Copilot coding agent).
+    allow_skip: true
+    failure_kind: infrastructure
+
+  # AI-assisted advisory reviews. Their findings are posted as comments and never fail
+  # the run, so a failed run is always an infrastructure failure (for example the
+  # intermittent Copilot inference 401). They are non-blocking by design.
+  - id: duplicate-scan
+    title: Duplicate resource scan
+    workflow: pr-duplicate-check.lock.yml
+    required: false
+    failure_kind: infrastructure
+
+  - id: quality-signal
+    title: PR quality signal
+    workflow: pr-quality-signal.lock.yml
+    required: false
+    failure_kind: infrastructure
+
+  # Pluggable slot for the canvas/plugin materialization and install smoke test.
+  # Any workflow can satisfy it by reporting a job named `canvas-smoke-test`.
+  - id: canvas-smoke-test
+    title: Canvas/plugin smoke test
+    check_name: canvas-smoke-test
+    branches: [main]
+    paths:
+      - "extensions/**"
+      - "plugins/**"
+    required: true
+    optional: true
+    contribution_steps: ["smoke", "(Materialize|materialize|Materialization|materialization)", "(Install|install) plugin"]
+    hint: See the smoke-test job log for the failing plugin or extension.
+
+commands:
+  request_review:
+    label: needs-reviewer
+    # Reviewer routing workflow dispatched after the label is added (labels added with
+    # GITHUB_TOKEN do not trigger `labeled` workflows). Ignored if the file is absent.
+    dispatch_workflow: review-routing.yml
+    dispatch_ref: main

.github/workflows/check-plugin-structure.yml

@@ -5,6 +5,7 @@ on:
     branches: [main]
     paths:
       - "plugins/**"
+      - ".github/workflows/check-plugin-structure.yml"
 
 permissions:
   contents: read