Skip to content

Commit 56d201f

Browse files
shakkernerdshakkernerd
committed
fix: retry workflow sanity checkout fetches
1 parent 9bb68b5 commit 56d201f

2 files changed

Lines changed: 82 additions & 12 deletions

File tree

‎.github/workflows/workflow-sanity.yml‎

Lines changed: 57 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,25 @@ jobs:
3434
git init "$GITHUB_WORKSPACE"
3535
git -C "$GITHUB_WORKSPACE" config gc.auto 0
3636
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
37-
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
38-
-c protocol.version=2 \
39-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
40-
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout"
37+
fetch_checkout_ref() {
38+
local fetch_status
39+
for attempt in 1 2 3; do
40+
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
41+
-c protocol.version=2 \
42+
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
43+
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout" && return 0
44+
fetch_status="$?"
45+
if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then
46+
return "$fetch_status"
47+
fi
48+
if [ "$attempt" = "3" ]; then
49+
return "$fetch_status"
50+
fi
51+
echo "::warning::checkout fetch for '$CHECKOUT_SHA' timed out on attempt $attempt; retrying"
52+
sleep 5
53+
done
54+
}
55+
fetch_checkout_ref
4156
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
4257
4358
- name: Fail on tabs in workflow files
@@ -78,10 +93,25 @@ jobs:
7893
git init "$GITHUB_WORKSPACE"
7994
git -C "$GITHUB_WORKSPACE" config gc.auto 0
8095
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
81-
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
82-
-c protocol.version=2 \
83-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
84-
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout"
96+
fetch_checkout_ref() {
97+
local fetch_status
98+
for attempt in 1 2 3; do
99+
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
100+
-c protocol.version=2 \
101+
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
102+
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout" && return 0
103+
fetch_status="$?"
104+
if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then
105+
return "$fetch_status"
106+
fi
107+
if [ "$attempt" = "3" ]; then
108+
return "$fetch_status"
109+
fi
110+
echo "::warning::checkout fetch for '$CHECKOUT_SHA' timed out on attempt $attempt; retrying"
111+
sleep 5
112+
done
113+
}
114+
fetch_checkout_ref
85115
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
86116
87117
- name: Setup Python
@@ -190,10 +220,25 @@ jobs:
190220
git init "$GITHUB_WORKSPACE"
191221
git -C "$GITHUB_WORKSPACE" config gc.auto 0
192222
git -C "$GITHUB_WORKSPACE" remote add origin "https://github.com/${CHECKOUT_REPO}.git"
193-
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
194-
-c protocol.version=2 \
195-
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
196-
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout"
223+
fetch_checkout_ref() {
224+
local fetch_status
225+
for attempt in 1 2 3; do
226+
timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE" \
227+
-c protocol.version=2 \
228+
fetch --no-tags --prune --no-recurse-submodules --depth=1 origin \
229+
"+${CHECKOUT_SHA}:refs/remotes/origin/checkout" && return 0
230+
fetch_status="$?"
231+
if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then
232+
return "$fetch_status"
233+
fi
234+
if [ "$attempt" = "3" ]; then
235+
return "$fetch_status"
236+
fi
237+
echo "::warning::checkout fetch for '$CHECKOUT_SHA' timed out on attempt $attempt; retrying"
238+
sleep 5
239+
done
240+
}
241+
fetch_checkout_ref
197242
git -C "$GITHUB_WORKSPACE" checkout --detach refs/remotes/origin/checkout
198243
199244
- name: Setup Node environment

‎test/scripts/ci-workflow-guards.test.ts‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,10 @@ function readCiWorkflow() {
77
return parse(readFileSync(".github/workflows/ci.yml", "utf8"));
88
}
99

10+
function readWorkflowSanityWorkflow() {
11+
return parse(readFileSync(".github/workflows/workflow-sanity.yml", "utf8"));
12+
}
13+
1014
function readCriticalQualityWorkflow() {
1115
return readFileSync(".github/workflows/codeql-critical-quality.yml", "utf8");
1216
}
@@ -74,6 +78,27 @@ describe("ci workflow guards", () => {
7478
}
7579
});
7680

81+
it("retries workflow sanity checkout fetch timeouts", () => {
82+
const workflow = readWorkflowSanityWorkflow();
83+
84+
for (const jobName of ["no-tabs", "actionlint", "generated-doc-baselines"]) {
85+
const checkoutStep = workflow.jobs[jobName].steps.find((step) => step.name === "Checkout");
86+
87+
expect(checkoutStep.run, jobName).toContain("fetch_checkout_ref()");
88+
expect(checkoutStep.run, jobName).toContain("for attempt in 1 2 3");
89+
expect(checkoutStep.run, jobName).toContain(
90+
'timeout --signal=TERM --kill-after=10s 30s git -C "$GITHUB_WORKSPACE"',
91+
);
92+
expect(checkoutStep.run, jobName).toContain(
93+
'if [ "$fetch_status" != "124" ] && [ "$fetch_status" != "137" ]; then',
94+
);
95+
expect(checkoutStep.run, jobName).toContain("timed out on attempt $attempt; retrying");
96+
expect(checkoutStep.run, jobName).toContain(
97+
"fetch --no-tags --prune --no-recurse-submodules --depth=1 origin",
98+
);
99+
}
100+
});
101+
77102
it("bounds platform checkout fetches without GNU timeout", () => {
78103
const workflow = readCiWorkflow();
79104

0 commit comments

Comments
 (0)