Phase 3: canvas smoke-test evidence and weekly review metrics (#4184)

.github/review-metrics.yml

@@ -0,0 +1,61 @@
+# Weekly review operating metrics (github/awesome-copilot#4184, phase 3).
+#
+# Read by eng/review-metrics.mjs, which .github/workflows/review-metrics.yml
+# runs every Monday. See docs/maintainers/canvas-evidence-and-metrics.md.
+
+# Days of activity summarized in each report.
+window_days: 7
+
+# Review targets, in business days (weekends are skipped).
+targets_business_days: [2, 4]
+
+# Base branch for contribution PRs.
+base_branch: main
+
+# State labels (maintained by the submission gate). Open PRs without any of
+# these are reported as "unlabeled".
+state_labels:
+  - awaiting-automation
+  - requires-submitter-fixes
+  - ready-for-review
+  - review-in-progress
+  - approved
+
+# Risk tier labels. Open PRs without any of these are "unclassified".
+risk_labels:
+  - merge-risk:low
+  - merge-risk:medium
+  - merge-risk:high
+
+# External plugin submission issues are tracked alongside PRs.
+external_plugin_label: external-plugin
+external_plugin_state_labels:
+  - awaiting-review
+  - requires-submitter-fixes
+  - ready-for-review
+  - awaiting-approval
+
+# Review-automation workflows whose failed/errored runs count toward the
+# automation failure rate. Files that do not exist yet are reported as
+# "not found" and skipped, so later phases can be listed ahead of time.
+automation_workflows:
+  - submission-gate.yml
+  - submission-gate-writer.yml
+  - review-routing.yml
+  - canvas-smoke-test.yml
+  - canvas-smoke-test-comment.yml
+  - pr-risk-scan.yml
+  - pr-risk-scan-comment.yml
+  - label-pr-intent.yml
+  - label-pr-intent-writer.yml
+  - check-plugin-structure.yml
+  - external-plugin-intake.yml
+  - external-plugin-quality-gates.yml
+  - external-plugin-pr-quality-gates.yml
+  - external-plugin-pr-quality-gates-writer.yml
+  - external-plugin-command-router.yml
+
+# Tracking issue that receives the weekly report.
+tracking_issue:
+  title: Review operating metrics
+  label: review-metrics

.github/workflows/canvas-smoke-test-comment.yml

@@ -0,0 +1,157 @@
+name: Canvas Smoke Test — Comment
+
+# Writer half of the canvas-smoke-test reader/writer split. The reader
+# (canvas-smoke-test.yml) runs on untrusted PR code with a read-only
+# token; this workflow only reads its artifact and never checks out PR code.
+
+on:
+  workflow_run:
+    workflows: ["Canvas Smoke Test"]
+    types: [completed]
+
+permissions:
+  actions: read
+  issues: write
+  pull-requests: write
+
+jobs:
+  comment:
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.event == 'pull_request'
+    steps:
+      - name: Download review artifact
+        id: download
+        continue-on-error: true
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: canvas-smoke-test-results
+          run-id: ${{ github.event.workflow_run.id }}
+          github-token: ${{ github.token }}
+
+      - name: Upsert PR comment
+        uses: actions/github-script@f28e40c7f34bde8b3046d885e986cb6290c5673b # v7.1.0
+        with:
+          script: |
+            const fs = require('fs');
+            const marker = '<!-- canvas-smoke-test -->';
+
+            if (!fs.existsSync('report.md')) {
+              core.info('No canvas smoke test report artifact found. Skipping.');
+              return;
+            }
+
+            let status = 'unknown';
+            try {
+              status = JSON.parse(fs.readFileSync('results.json', 'utf8')).status || 'unknown';
+            } catch {
+              core.warning('results.json is missing or invalid.');
+            }
+
+            const workflowRun = context.payload.workflow_run;
+            const artifactPr = fs.existsSync('pr-number.txt') ? fs.readFileSync('pr-number.txt', 'utf8').trim() : '';
+            const prNumber = /^[1-9][0-9]*$/.test(artifactPr) ? Number(artifactPr) : workflowRun.pull_requests?.[0]?.number;
+            if (!prNumber) {
+              core.warning('Could not determine PR number. Skipping.');
+              return;
+            }
+
+            // The artifact is untrusted: bind the target PR to the triggering run
+            // (base repo, head repo/ref/SHA, and a unique association) before writing.
+            const { data: pr } = await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: prNumber,
+            });
+            if (pr.state !== 'open') {
+              core.info(`Skipping non-open PR #${prNumber}.`);
+              return;
+            }
+            const expectedBaseRepository = `${context.repo.owner}/${context.repo.repo}`.toLowerCase();
+            const runHeadRepository = String(workflowRun.head_repository?.full_name || '');
+            const runHeadRepositoryParts = runHeadRepository.split('/');
+            const runHeadRef = String(workflowRun.head_branch || '');
+            if (String(pr.base?.repo?.full_name || '').toLowerCase() !== expectedBaseRepository) {
+              core.setFailed(`PR #${prNumber} does not target this repository.`);
+              return;
+            }
+            if (pr.head.sha !== workflowRun.head_sha) {
+              core.info(`PR #${prNumber} head ${pr.head.sha} does not match run head ${workflowRun.head_sha}. Skipping stale or mismatched report.`);
+              return;
+            }
+            if (
+              runHeadRepositoryParts.length !== 2 ||
+              !runHeadRepositoryParts[0] ||
+              !runHeadRepositoryParts[1] ||
+              !runHeadRef ||
+              String(pr.head?.repo?.full_name || '').toLowerCase() !== runHeadRepository.toLowerCase() ||
+              String(pr.head?.ref || '') !== runHeadRef
+            ) {
+              core.setFailed(`PR #${prNumber} head repository/ref did not match workflow_run.`);
+              return;
+            }
+            const workflowRunPullRequests = Array.isArray(workflowRun.pull_requests) ? workflowRun.pull_requests : [];
+            if (workflowRunPullRequests.length > 0) {
+              if (!workflowRunPullRequests.some((pullRequest) => pullRequest.number === prNumber)) {
+                core.setFailed(`PR #${prNumber} was not present in workflow_run.pull_requests.`);
+                return;
+              }
+            } else {
+              const candidatePullRequests = await github.paginate(github.rest.pulls.list, {
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                state: 'open',
+                head: `${runHeadRepositoryParts[0]}:${runHeadRef}`,
+                per_page: 100,
+              });
+              const trustedMatches = candidatePullRequests.filter((candidate) =>
+                candidate.head?.sha === workflowRun.head_sha &&
+                String(candidate.head?.ref || '') === runHeadRef &&
+                String(candidate.head?.repo?.full_name || '').toLowerCase() === runHeadRepository.toLowerCase() &&
+                String(candidate.base?.repo?.full_name || '').toLowerCase() === expectedBaseRepository
+              );
+              if (trustedMatches.length !== 1 || trustedMatches[0].number !== prNumber) {
+                core.setFailed(`PR #${prNumber} could not be uniquely associated with workflow_run.`);
+                return;
+              }
+            }
+
+            let body = fs.readFileSync('report.md', 'utf8').replace(/@/g, '@\u200b');
+            const maxLength = 65000;
+            if (body.length > maxLength) {
+              body = `${body.slice(0, maxLength)}\n\n_...(truncated)..._`;
+            }
+            if (!body.includes(marker)) {
+              body = `${marker}\n${body}`;
+            }
+
+            const comments = await github.paginate(github.rest.issues.listComments, {
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber,
+              per_page: 100,
+            });
+            const existing = comments.find((comment) =>
+              comment.user?.type === 'Bot' && comment.body?.includes(marker));
+
+            if (status === 'skipped' && !existing) {
+              core.info('No canvas changes and no previous comment. Nothing to post.');
+              return;
+            }
+
+            if (existing) {
+              await github.rest.issues.updateComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                comment_id: existing.id,
+                body,
+              });
+              core.info(`Updated canvas smoke test comment ${existing.id}`);
+            } else {
+              await github.rest.issues.createComment({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                issue_number: prNumber,
+                body,
+              });
+              core.info('Created canvas smoke test comment');
+            }

.github/workflows/canvas-smoke-test.yml

@@ -0,0 +1,111 @@
+name: Canvas Smoke Test
+
+# Reader half of the canvas-smoke-test reader/writer split. Runs untrusted PR
+# content with a read-only token and no secrets; extension code is parsed and
+# compiled but never executed. canvas-smoke-test-comment.yml posts the report.
+
+on:
+  pull_request:
+    branches: [main]
+    types: [opened, synchronize, reopened]
+    paths:
+      - "extensions/**"
+      - "plugins/**"
+      - ".github/workflows/canvas-smoke-test.yml"
+      - "eng/canvas-smoke-test.mjs"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: canvas-smoke-test-${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
+jobs:
+  canvas-smoke-test:
+    name: canvas-smoke-test
+    runs-on: ubuntu-latest
+    env:
+      PR_NUMBER: ${{ github.event.pull_request.number }}
+      HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+      HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
+      BASE_SHA: ${{ github.event.pull_request.base.sha }}
+      CANVAS_PREVIEW_MIN_WIDTH: ${{ vars.CANVAS_PREVIEW_MIN_WIDTH }}
+      CANVAS_PREVIEW_MIN_HEIGHT: ${{ vars.CANVAS_PREVIEW_MIN_HEIGHT }}
+      CANVAS_RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4.3.1
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+
+      - name: Setup Node.js
+        uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0
+        with:
+          node-version: "22"
+          cache: "npm"
+
+      - name: Detect canvas targets
+        id: detect
+        run: |
+          set -euo pipefail
+          git diff --name-only "${BASE_SHA}...HEAD" > "${RUNNER_TEMP}/changed-files.txt"
+          node ./eng/canvas-smoke-test.mjs \
+            --changed-files "${RUNNER_TEMP}/changed-files.txt" \
+            --base-ref "${BASE_SHA}" \
+            --detect-only
+
+      - name: Write skipped report
+        if: steps.detect.outputs.canvas != 'true'
+        run: |
+          set -euo pipefail
+          mkdir -p canvas-smoke-results
+          printf '<!-- canvas-smoke-test -->\n## 🧩 Canvas smoke test\n\n⏭️ **Skipped** — no canvas extension or extension-bearing plugin paths changed.\n' > canvas-smoke-results/report.md
+          printf '{"schema_version":"canvas-smoke-test/v1","status":"skipped"}\n' > canvas-smoke-results/results.json
+          echo "${PR_NUMBER}" > canvas-smoke-results/pr-number.txt
+          echo "${HEAD_SHA}" > canvas-smoke-results/head-sha.txt
+          cat canvas-smoke-results/report.md >> "$GITHUB_STEP_SUMMARY"
+
+      - name: Install dependencies
+        if: steps.detect.outputs.canvas == 'true'
+        run: npm ci --ignore-scripts
+
+      - name: Install GitHub Copilot CLI
+        if: steps.detect.outputs.canvas == 'true'
+        continue-on-error: true
+        run: npm install -g @github/copilot
+
+      - name: Run canvas smoke test
+        if: steps.detect.outputs.canvas == 'true'
+        run: |
+          set -uo pipefail
+          export CANVAS_PREVIEW_BASE_URL="https://raw.githubusercontent.com/${HEAD_REPO}/${HEAD_SHA}/"
+          node ./eng/canvas-smoke-test.mjs \
+            --changed-files "${RUNNER_TEMP}/changed-files.txt" \
+            --base-ref "${BASE_SHA}" \
+            --install require \
+            --output-dir canvas-smoke-results > /dev/null
+          exit_code=$?
+          if [ ! -f canvas-smoke-results/report.md ]; then
+            mkdir -p canvas-smoke-results
+            printf '<!-- canvas-smoke-test -->\n## 🧩 Canvas smoke test\n\n⚠️ **Infrastructure error** — the checker crashed. See the workflow logs.\n' > canvas-smoke-results/report.md
+            echo '{"schema_version":"canvas-smoke-test/v1","status":"infra_error"}' > canvas-smoke-results/results.json
+          fi
+          echo "${PR_NUMBER}" > canvas-smoke-results/pr-number.txt
+          echo "${HEAD_SHA}" > canvas-smoke-results/head-sha.txt
+          cat canvas-smoke-results/report.md >> "$GITHUB_STEP_SUMMARY"
+          case "$exit_code" in
+            0) ;;
+            1) echo "::error::Canvas smoke test found contribution issues. See the job summary." ;;
+            *) echo "::error::Canvas smoke test hit an infrastructure error (not a contribution failure)." ;;
+          esac
+          exit "$exit_code"
+
+      - name: Upload review artifact
+        if: always() && hashFiles('canvas-smoke-results/report.md') != ''
+        uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7
+        with:
+          name: canvas-smoke-test-results
+          path: canvas-smoke-results/
+          retention-days: 7