Skip to content

Commit 70967c7

Browse files
SunrisepeakSunrisepeak
committed
fix(mirror): never kill slow uploads (outer timeout 300<inner 600 bug), parallel hosts, shared-deadline verify
v0.0.90 postmortem — the 20min+ mirror was three stacked issues: - the outer 'timeout 300' SIGKILLed every cross-border PUT slower than 5min at ~60% and restarted it from byte zero (gtc's whole-file PUT has its own 600s timeout; the wrapper must exceed it → 900). Same anti-pattern as 0.0.86/0.0.89's delete-on-verify-timeout: cutting the slow thing instead of waiting for it. - GH and GitCode mirrors serialized despite being independent → now run in parallel (wall ≈ max, not sum; the gitcode leg is the slow one). - per-asset 120s verify waits stacked serially → one shared ~150s propagation clock sweeps all pending assets. Expected normal path: ≈ one cross-border transfer time (2-4min).
1 parent 116f532 commit 70967c7

1 file changed

Lines changed: 67 additions & 59 deletions

File tree

‎.github/tools/mirror_res.sh‎

Lines changed: 67 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -62,88 +62,96 @@ probe() { # host_path asset → 0 iff the object serves bytes
6262
[[ "$code" == 200 || "$code" == 206 ]]
6363
}
6464

65-
# Wait for an asset to serve, with real propagation patience (fresh uploads
66-
# take ~20-60s to reach the download CDN). ~2min ceiling, 5s cadence.
67-
wait_asset() { # url → 0 iff 200/206 within the window
68-
for _ in $(seq 1 24); do
69-
probe "$1" && return 0
70-
sleep 5
65+
# Shared-deadline batch verify: ONE ~2min propagation clock sweeps ALL
66+
# pending assets (per-asset serial 120s waits stacked up to 16min in the
67+
# v0.0.90 incident). Echoes the still-missing set; empty output = all good.
68+
verify_batch() { # base_url asset... → prints assets still not serving
69+
local base="$1"; shift
70+
local deadline=$((SECONDS + 150))
71+
local pending=("$@")
72+
while ((${#pending[@]})) && ((SECONDS < deadline)); do
73+
local still=()
74+
for a in "${pending[@]}"; do
75+
probe "${base}/${a}" || still+=("$a")
76+
done
77+
pending=("${still[@]}")
78+
((${#pending[@]})) && sleep 5
7179
done
72-
return 1
80+
((${#pending[@]})) && echo "${pending[*]}"
81+
return 0
7382
}
7483

75-
# ── GitHub ──────────────────────────────────────────────────────────
76-
# A4 hardening: `gh release upload` was observed both HANGING (>1h on one
77-
# asset, 2026-07-08) and leaving a phantom asset that later 404s (0.0.75/76).
78-
# Shape (v0.0.89 lesson): upload ALL assets first, then verify with patience —
79-
# propagation overlaps instead of serializing per asset. NEVER delete on a
80-
# verify timeout: --clobber re-upload already replaces, and the eager
81-
# 404→delete loop repeatedly deleted GOOD uploads whose propagation was
82-
# merely slower than the wait window (0.0.86 incident, recurred at 18s in
83-
# v0.0.89 — 11 delete+reupload cycles across 8 assets).
84-
GH_ENABLED=0
85-
if [[ -n "${XLINGS_RES_TOKEN:-}" ]] || gh auth status >/dev/null 2>&1; then
86-
GH_ENABLED=1
87-
info "GitHub $GH_DST tag $VER"
88-
GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release view "$VER" -R "$GH_DST" >/dev/null 2>&1 \
89-
|| GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release create "$VER" -R "$GH_DST" --title "$VER" --notes "$PROJ $VER (mirror of $SRC_REPO)"
84+
# One host's mirror: upload everything not yet serving, then batch-verify
85+
# with propagation patience, up to 3 rounds.
86+
#
87+
# Hard-won rules (0.0.86 / 0.0.89 / 0.0.90 postmortems):
88+
# - NEVER delete on a verify timeout — the eager 404→delete loop repeatedly
89+
# deleted GOOD uploads whose propagation was merely slow.
90+
# - NEVER kill a slow-but-progressing upload: the outer `timeout` MUST
91+
# exceed gtc's inner PUT timeout (600s). v0.0.90 wrapped uploads in
92+
# `timeout 300`, so every cross-border PUT >5min was SIGKILLed at 60%%
93+
# and restarted from byte zero — the 20min job ceiling fell to this.
94+
# - gtc's exit code lies both ways (obs_callback flakiness); the download
95+
# probe is the only source of truth.
96+
mirror_host() { # kind(gh|gtc) base_url
97+
local kind="$1" base="$2" try a
98+
local pending failed
9099
for try in 1 2 3; do
91-
# Phase 1: upload everything not yet serving (idempotent re-runs skip).
92100
pending=()
93101
for a in "${ASSETS[@]}"; do
94-
if probe "https://github.com/${GH_DST}/releases/download/${VER}/${a}"; then
95-
[[ $try == 1 ]] && info "gh $a already mirrored, skipping"
102+
if probe "${base}/${a}"; then
103+
[[ $try == 1 ]] && info "$kind $a already mirrored, skipping"
96104
continue
97105
fi
98106
pending+=("$a")
99-
GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout 300 gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber || true
100-
done
101-
[[ ${#pending[@]} == 0 ]] && break
102-
# Phase 2: verify the batch with propagation patience.
103-
failed=()
104-
for a in "${pending[@]}"; do
105-
wait_asset "https://github.com/${GH_DST}/releases/download/${VER}/${a}" \
106-
|| failed+=("$a")
107+
if [[ "$kind" == gh ]]; then
108+
GH_TOKEN="${XLINGS_RES_TOKEN:-}" timeout 900 \
109+
gh release upload "$VER" "$DL/$a" -R "$GH_DST" --clobber || true
110+
else
111+
timeout 900 gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" \
112+
>/dev/null 2>&1 || true
113+
fi
107114
done
108-
[[ ${#failed[@]} == 0 ]] && break
109-
echo "[mirror] gh not serving after patience (try $try): ${failed[*]} — re-uploading (no delete)"
115+
[[ ${#pending[@]} == 0 ]] && return 0
116+
failed=$(verify_batch "$base" "${pending[@]}")
117+
[[ -z "$failed" ]] && return 0
118+
echo "[mirror] $kind not serving after patience (try $try): $failed — re-uploading (no delete)"
110119
done
120+
return 0 # the completeness gate below is the real pass/fail
121+
}
122+
123+
# ── Both hosts IN PARALLEL: they are fully independent, and the gitcode
124+
# leg is cross-border-slow — serializing them doubled wall time for nothing.
125+
GH_ENABLED=0
126+
if [[ -n "${XLINGS_RES_TOKEN:-}" ]] || gh auth status >/dev/null 2>&1; then
127+
GH_ENABLED=1
128+
info "GitHub $GH_DST tag $VER"
129+
GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release view "$VER" -R "$GH_DST" >/dev/null 2>&1 \
130+
|| GH_TOKEN="${XLINGS_RES_TOKEN:-}" gh release create "$VER" -R "$GH_DST" --title "$VER" --notes "$PROJ $VER (mirror of $SRC_REPO)"
111131
else
112132
info "no github auth; skipping github mirror"
113133
fi
114-
115-
# ── GitCode (gtc — multi-file upload can 502 and drop files) ─────────
116-
# Same two-phase shape. gtc can report errors on uploads that actually
117-
# succeeded (obs_callback flakiness) — the probe, not gtc's exit code, is
118-
# the source of truth.
119134
GTC_ENABLED=0
120135
if [[ -n "${GITCODE_TOKEN:-}" ]] && command -v gtc >/dev/null 2>&1; then
121136
GTC_ENABLED=1
122137
info "GitCode $GTC_DST tag $VER"
123138
gtc release create "$GTC_DST" --tag "$VER" --name "$VER" 2>/dev/null || true
124-
for try in 1 2 3; do
125-
pending=()
126-
for a in "${ASSETS[@]}"; do
127-
if probe "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}"; then
128-
[[ $try == 1 ]] && info "gtc $a already mirrored, skipping"
129-
continue
130-
fi
131-
pending+=("$a")
132-
timeout 300 gtc release upload "$GTC_DST" "$DL/$a" --tag "$VER" >/dev/null 2>&1 || true
133-
done
134-
[[ ${#pending[@]} == 0 ]] && break
135-
failed=()
136-
for a in "${pending[@]}"; do
137-
wait_asset "https://gitcode.com/${GTC_DST}/releases/download/${VER}/${a}" \
138-
|| failed+=("$a")
139-
done
140-
[[ ${#failed[@]} == 0 ]] && break
141-
echo "[mirror] gtc not serving after patience (try $try): ${failed[*]} — re-uploading"
142-
done
143139
else
144140
info "no GITCODE_TOKEN/gtc; skipping gitcode mirror"
145141
fi
146142

143+
GH_PID=""; GTC_PID=""
144+
if [[ "$GH_ENABLED" == 1 ]]; then
145+
mirror_host gh "https://github.com/${GH_DST}/releases/download/${VER}" &
146+
GH_PID=$!
147+
fi
148+
if [[ "$GTC_ENABLED" == 1 ]]; then
149+
mirror_host gtc "https://gitcode.com/${GTC_DST}/releases/download/${VER}" &
150+
GTC_PID=$!
151+
fi
152+
[[ -n "$GH_PID" ]] && wait "$GH_PID"
153+
[[ -n "$GTC_PID" ]] && wait "$GTC_PID"
154+
147155
# ── Completeness gate: verify every asset on every ENABLED host ──
148156
# A4: this is the mirror's definition of done. The caller must treat a
149157
# non-zero exit as a hard failure (release.yml does since the 0.0.85

0 commit comments

Comments
 (0)