Skip to content

Commit 9448564

Browse files
edburnsedburns
committed
Reject incomplete shepherd CI checks
Treat only passing, skipped, and the named remove-before-merge failure as acceptable in the independent post-stage gate. Pending, cancelled, failed, and unknown buckets now fail closed in both Bash and PowerShell. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Copilot-Session: 292755c3-37b5-4a61-8f01-6d2d399b05cf
1 parent 92ad4b9 commit 9448564

7 files changed

Lines changed: 123 additions & 12 deletions

File tree

‎plugins/shepherd-task/figure-02-shepherd-task.md‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ sequenceDiagram
4040
end
4141
4242
ST->>GH: Ensure PR base equals campaign base
43-
ST->>GH: Reject non-exempt failed CI checks
43+
ST->>GH: Reject pending, cancelled, unknown, or non-exempt failed CI checks
4444
ST->>GH: Reject unresolved review threads
4545
4646
alt PR is not merged
@@ -65,3 +65,6 @@ prior Stage 40 attempt already made the PR ready, the runner resumes Stage 40
6565
only after validating a successful Stage 30 transcript for that exact task and
6666
PR in the supplied run directory. The stage skills perform the deeper issue,
6767
SHA, CI, review, and lesson gates shown in Figures 03 and 04.
68+
The outer CI postcondition independently requires every reported check bucket
69+
to be terminal and acceptable; JSON-mode exit status alone is not treated as
70+
proof that pending checks completed.

‎plugins/shepherd-task/scripts/shepherd-task.ps1‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -242,18 +242,18 @@ function Find-LinkedPR {
242242
return $null
243243
}
244244

245-
# --- Helper: Verify all CI checks pass (excluding expected failure) ---
245+
# --- Helper: Verify all CI checks are terminal and acceptable (excluding expected failure) ---
246246
function Test-CIPassing {
247247
param([string]$PRNumber)
248248

249-
$failures = @(gh pr checks $PRNumber -R $Repo --json name,state,bucket `
250-
--jq '.[] | select(.bucket == "fail") | select(.name != "No remove-before-merge directories") | .name' 2>$null)
249+
$blockingChecks = @(gh pr checks $PRNumber -R $Repo --json name,state,bucket `
250+
--jq '.[] | select((.bucket == "pass" or .bucket == "skipping" or (.bucket == "fail" and .name == "No remove-before-merge directories")) | not) | "\(.name): \(.bucket)"' 2>$null)
251251
$ghExitCode = $LASTEXITCODE
252252
if ($ghExitCode -ne 0) {
253253
throw "Unable to query CI checks for PR #$PRNumber."
254254
}
255255

256-
return $failures.Count -eq 0
256+
return $blockingChecks.Count -eq 0
257257
}
258258

259259
# --- Helper: Check for unresolved review state ---

‎plugins/shepherd-task/scripts/shepherd-task.sh‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -168,15 +168,15 @@ find_linked_pr() {
168168
return 1
169169
}
170170

171-
# Verify all CI checks pass (excluding expected failure).
171+
# Verify all CI checks are terminal and acceptable (excluding expected failure).
172172
ci_passing() {
173173
local pr_number="$1"
174-
local failures
175-
failures=$(gh pr checks "$pr_number" -R "$REPO" --json name,state,bucket \
176-
--jq '.[] | select(.bucket == "fail") | select(.name != "No remove-before-merge directories") | .name' 2>/dev/null) ||
174+
local blocking_checks
175+
blocking_checks=$(gh pr checks "$pr_number" -R "$REPO" --json name,state,bucket \
176+
--jq '.[] | select((.bucket == "pass" or .bucket == "skipping" or (.bucket == "fail" and .name == "No remove-before-merge directories")) | not) | "\(.name): \(.bucket)"' 2>/dev/null) ||
177177
return 1
178178

179-
[[ -z "$failures" ]]
179+
[[ -z "$blocking_checks" ]]
180180
}
181181

182182
# Check for unresolved bot review comments.

‎plugins/shepherd-task/test/cargotracker-add-change-arrival-deadline-feature-treatment-control/12-session-outcome-contract.ps1‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,33 @@ The GitHub CLI and Copilot CLI both returned process exit code 0.
9999
-SharePath $successfulStage30 -Stage 30 -TaskIssue 14 -PRNumber 0
100100
}
101101

102+
$blockingCheckFilter = '.[] | select((.bucket == "pass" or .bucket == "skipping" or (.bucket == "fail" and .name == "No remove-before-merge directories")) | not) | "\(.name): \(.bucket)"'
103+
$ciChecksJson = @'
104+
[
105+
{"name":"passing","state":"SUCCESS","bucket":"pass"},
106+
{"name":"skipped","state":"SKIPPED","bucket":"skipping"},
107+
{"name":"No remove-before-merge directories","state":"FAILURE","bucket":"fail"},
108+
{"name":"pending","state":"PENDING","bucket":"pending"},
109+
{"name":"cancelled","state":"CANCELLED","bucket":"cancel"},
110+
{"name":"failed","state":"FAILURE","bucket":"fail"},
111+
{"name":"future-state","state":"UNKNOWN","bucket":"unknown"}
112+
]
113+
'@
114+
$blockingChecks = @($ciChecksJson | jq -r $blockingCheckFilter)
115+
$jqExitCode = $LASTEXITCODE
116+
if ($jqExitCode -ne 0) {
117+
throw "Unable to evaluate the CI gate fixture; jq exited $jqExitCode."
118+
}
119+
$expectedBlockingChecks = @(
120+
'pending: pending',
121+
'cancelled: cancel',
122+
'failed: fail',
123+
'future-state: unknown'
124+
)
125+
if (($blockingChecks -join "`n") -ne ($expectedBlockingChecks -join "`n")) {
126+
throw "CI gate accepted a nonterminal or unsuccessful check bucket: $($blockingChecks -join ', ')"
127+
}
128+
102129
$orchestrator = [System.IO.File]::ReadAllText($orchestratorPath)
103130
foreach ($required in @(
104131
'resuming Phase 1',
@@ -118,12 +145,18 @@ The GitHub CLI and Copilot CLI both returned process exit code 0.
118145
'-SharePath $candidate.FullName',
119146
'gh api graphql --paginate --slurp',
120147
"reviewDecision -eq 'CHANGES_REQUESTED'",
121-
'$reviewDecision -ne ''CHANGES_REQUESTED'''
148+
'$reviewDecision -ne ''CHANGES_REQUESTED''',
149+
$blockingCheckFilter
122150
)) {
123151
if (-not $orchestrator.Contains($required)) {
124152
throw "PowerShell orchestrator is missing semantic outcome contract text: $required"
125153
}
126154
}
155+
if ($orchestrator.Contains(
156+
'.[] | select(.bucket == "fail") | select(.name != "No remove-before-merge directories")'
157+
)) {
158+
throw 'PowerShell orchestrator still accepts pending checks through the failure-only filter.'
159+
}
127160
if ($orchestrator.Contains('test(`"#$TaskIssue`")')) {
128161
throw 'PowerShell orchestrator still uses a prefix-colliding issue body search.'
129162
}

‎plugins/shepherd-task/test/cargotracker-add-change-arrival-deadline-feature-treatment-control/12-session-outcome-contract.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,22 @@ title_branch_matches=$(jq -r '.[] | select(((.title // "") | test("(^|[^0-9])14(
5555
exit 1
5656
}
5757

58+
blocking_check_filter='.[] | select((.bucket == "pass" or .bucket == "skipping" or (.bucket == "fail" and .name == "No remove-before-merge directories")) | not) | "\(.name): \(.bucket)"'
59+
ci_checks_json='[
60+
{"name":"passing","state":"SUCCESS","bucket":"pass"},
61+
{"name":"skipped","state":"SKIPPED","bucket":"skipping"},
62+
{"name":"No remove-before-merge directories","state":"FAILURE","bucket":"fail"},
63+
{"name":"pending","state":"PENDING","bucket":"pending"},
64+
{"name":"cancelled","state":"CANCELLED","bucket":"cancel"},
65+
{"name":"failed","state":"FAILURE","bucket":"fail"},
66+
{"name":"future-state","state":"UNKNOWN","bucket":"unknown"}
67+
]'
68+
blocking_checks=$(jq -r "$blocking_check_filter" <<<"$ci_checks_json" | tr -d '\r')
69+
[[ "$blocking_checks" == $'pending: pending\ncancelled: cancel\nfailed: fail\nfuture-state: unknown' ]] || {
70+
echo "CI gate accepted a nonterminal or unsuccessful check bucket: $blocking_checks" >&2
71+
exit 1
72+
}
73+
5874
grep -Fq 'resuming Phase 1' "$orchestrator"
5975
grep -Fq 'find_linked_pr MERGED' "$orchestrator"
6076
grep -Fq 'closingIssuesReferences' "$orchestrator"
@@ -71,6 +87,11 @@ grep -Fq 'has no successful Stage 30 transcript for that PR' "$orchestrator"
7187
grep -Fq '"$candidate" 30 "$TASK_ISSUE" "$PR_NUMBER"' "$orchestrator"
7288
grep -Fq 'gh api graphql --paginate --slurp' "$orchestrator"
7389
grep -Fq '"$review_decision" != "CHANGES_REQUESTED"' "$orchestrator"
90+
grep -Fq "$blocking_check_filter" "$orchestrator"
91+
if grep -Fq '.[] | select(.bucket == "fail") | select(.name != "No remove-before-merge directories")' "$orchestrator"; then
92+
echo 'Bash orchestrator still accepts pending checks through the failure-only filter.' >&2
93+
exit 1
94+
fi
7495
if grep -Fq 'find_linked_pr OPEN) || true' "$orchestrator"; then
7596
echo 'Bash orchestrator still suppresses linked-PR discovery errors.' >&2
7697
exit 1

‎plugins/shepherd-task/test/cargotracker-add-change-arrival-deadline-feature/12-session-outcome-contract.ps1‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,33 @@ The GitHub CLI and Copilot CLI both returned process exit code 0.
9999
-SharePath $successfulStage30 -Stage 30 -TaskIssue 14 -PRNumber 0
100100
}
101101

102+
$blockingCheckFilter = '.[] | select((.bucket == "pass" or .bucket == "skipping" or (.bucket == "fail" and .name == "No remove-before-merge directories")) | not) | "\(.name): \(.bucket)"'
103+
$ciChecksJson = @'
104+
[
105+
{"name":"passing","state":"SUCCESS","bucket":"pass"},
106+
{"name":"skipped","state":"SKIPPED","bucket":"skipping"},
107+
{"name":"No remove-before-merge directories","state":"FAILURE","bucket":"fail"},
108+
{"name":"pending","state":"PENDING","bucket":"pending"},
109+
{"name":"cancelled","state":"CANCELLED","bucket":"cancel"},
110+
{"name":"failed","state":"FAILURE","bucket":"fail"},
111+
{"name":"future-state","state":"UNKNOWN","bucket":"unknown"}
112+
]
113+
'@
114+
$blockingChecks = @($ciChecksJson | jq -r $blockingCheckFilter)
115+
$jqExitCode = $LASTEXITCODE
116+
if ($jqExitCode -ne 0) {
117+
throw "Unable to evaluate the CI gate fixture; jq exited $jqExitCode."
118+
}
119+
$expectedBlockingChecks = @(
120+
'pending: pending',
121+
'cancelled: cancel',
122+
'failed: fail',
123+
'future-state: unknown'
124+
)
125+
if (($blockingChecks -join "`n") -ne ($expectedBlockingChecks -join "`n")) {
126+
throw "CI gate accepted a nonterminal or unsuccessful check bucket: $($blockingChecks -join ', ')"
127+
}
128+
102129
$orchestrator = [System.IO.File]::ReadAllText($orchestratorPath)
103130
foreach ($required in @(
104131
'resuming Phase 1',
@@ -118,12 +145,18 @@ The GitHub CLI and Copilot CLI both returned process exit code 0.
118145
'-SharePath $candidate.FullName',
119146
'gh api graphql --paginate --slurp',
120147
"reviewDecision -eq 'CHANGES_REQUESTED'",
121-
'$reviewDecision -ne ''CHANGES_REQUESTED'''
148+
'$reviewDecision -ne ''CHANGES_REQUESTED''',
149+
$blockingCheckFilter
122150
)) {
123151
if (-not $orchestrator.Contains($required)) {
124152
throw "PowerShell orchestrator is missing semantic outcome contract text: $required"
125153
}
126154
}
155+
if ($orchestrator.Contains(
156+
'.[] | select(.bucket == "fail") | select(.name != "No remove-before-merge directories")'
157+
)) {
158+
throw 'PowerShell orchestrator still accepts pending checks through the failure-only filter.'
159+
}
127160
if ($orchestrator.Contains('test(`"#$TaskIssue`")')) {
128161
throw 'PowerShell orchestrator still uses a prefix-colliding issue body search.'
129162
}

‎plugins/shepherd-task/test/cargotracker-add-change-arrival-deadline-feature/12-session-outcome-contract.sh‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,22 @@ title_branch_matches=$(jq -r '.[] | select(((.title // "") | test("(^|[^0-9])14(
5555
exit 1
5656
}
5757

58+
blocking_check_filter='.[] | select((.bucket == "pass" or .bucket == "skipping" or (.bucket == "fail" and .name == "No remove-before-merge directories")) | not) | "\(.name): \(.bucket)"'
59+
ci_checks_json='[
60+
{"name":"passing","state":"SUCCESS","bucket":"pass"},
61+
{"name":"skipped","state":"SKIPPED","bucket":"skipping"},
62+
{"name":"No remove-before-merge directories","state":"FAILURE","bucket":"fail"},
63+
{"name":"pending","state":"PENDING","bucket":"pending"},
64+
{"name":"cancelled","state":"CANCELLED","bucket":"cancel"},
65+
{"name":"failed","state":"FAILURE","bucket":"fail"},
66+
{"name":"future-state","state":"UNKNOWN","bucket":"unknown"}
67+
]'
68+
blocking_checks=$(jq -r "$blocking_check_filter" <<<"$ci_checks_json" | tr -d '\r')
69+
[[ "$blocking_checks" == $'pending: pending\ncancelled: cancel\nfailed: fail\nfuture-state: unknown' ]] || {
70+
echo "CI gate accepted a nonterminal or unsuccessful check bucket: $blocking_checks" >&2
71+
exit 1
72+
}
73+
5874
grep -Fq 'resuming Phase 1' "$orchestrator"
5975
grep -Fq 'find_linked_pr MERGED' "$orchestrator"
6076
grep -Fq 'closingIssuesReferences' "$orchestrator"
@@ -71,6 +87,11 @@ grep -Fq 'has no successful Stage 30 transcript for that PR' "$orchestrator"
7187
grep -Fq '"$candidate" 30 "$TASK_ISSUE" "$PR_NUMBER"' "$orchestrator"
7288
grep -Fq 'gh api graphql --paginate --slurp' "$orchestrator"
7389
grep -Fq '"$review_decision" != "CHANGES_REQUESTED"' "$orchestrator"
90+
grep -Fq "$blocking_check_filter" "$orchestrator"
91+
if grep -Fq '.[] | select(.bucket == "fail") | select(.name != "No remove-before-merge directories")' "$orchestrator"; then
92+
echo 'Bash orchestrator still accepts pending checks through the failure-only filter.' >&2
93+
exit 1
94+
fi
7495
if grep -Fq 'find_linked_pr OPEN) || true' "$orchestrator"; then
7596
echo 'Bash orchestrator still suppresses linked-PR discovery errors.' >&2
7697
exit 1

0 commit comments

Comments
 (0)