From 6453fa8a759e9944a835ea4f98f54c928fdd85d2 Mon Sep 17 00:00:00 2001
From: Eric Hare <ericrhare@gmail.com>
Date: Tue, 23 Jun 2026 16:45:34 -0700
Subject: [PATCH] ci: throttle bundle publishes to PyPI

---
 .github/workflows/release.yml         | 39 ++++++++++++++++++++++-----
 .github/workflows/release_bundles.yml | 39 ++++++++++++++++++++++-----
 2 files changed, 64 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
index ade6471a4c..4820bbef29 100644
--- a/.github/workflows/release.yml
+++ b/.github/workflows/release.yml
@@ -1013,19 +1013,44 @@ jobs:
             exit 0
           fi
           failed=0
+          pypi_publish_delay_seconds=60
+          pypi_publish_max_attempts=5
+          wheel_count=${#wheels[@]}
+          wheel_number=0
           for wheel in "${wheels[@]}"; do
+            wheel_number=$((wheel_number + 1))
             echo "Publishing $wheel"
-            # Tolerate "already exists" so reruns -- or a bundle whose version is
-            # unchanged and already on PyPI -- do not fail the job. Matches the
-            # duplicate-tolerant publish in release_bundles.yml.
-            if ! err=$(uv publish "$wheel" 2>&1); then
+            attempt=1
+            published=0
+            while true; do
+              # Tolerate "already exists" so reruns -- or a bundle whose version is
+              # unchanged and already on PyPI -- do not fail the job. Matches the
+              # duplicate-tolerant publish in release_bundles.yml.
+              if err=$(uv publish "$wheel" 2>&1); then
+                echo "$err"
+                published=1
+                break
+              fi
               echo "$err"
               if echo "$err" | grep -qiE 'already exists|file already exists|HTTP 400|duplicate'; then
                 echo "Skipping $wheel: already published."
-              else
-                echo "Publish failed for $wheel"
-                failed=1
+                break
               fi
+              if echo "$err" | grep -qiE 'HTTP 429|429 Too Many Requests|Too many new projects created'; then
+                if [ "$attempt" -lt "$pypi_publish_max_attempts" ]; then
+                  echo "PyPI rate limited $wheel; sleeping ${pypi_publish_delay_seconds}s before retry $((attempt + 1))/${pypi_publish_max_attempts}."
+                  sleep "$pypi_publish_delay_seconds"
+                  attempt=$((attempt + 1))
+                  continue
+                fi
+              fi
+              echo "Publish failed for $wheel"
+              failed=1
+              break
+            done
+            if [ "$published" = "1" ] && [ "$wheel_number" -lt "$wheel_count" ]; then
+              echo "Sleeping ${pypi_publish_delay_seconds}s before the next PyPI upload."
+              sleep "$pypi_publish_delay_seconds"
             fi
           done
           if [ "$failed" = "1" ]; then
diff --git a/.github/workflows/release_bundles.yml b/.github/workflows/release_bundles.yml
index 17a89edb4a..ccad3b2bdb 100644
--- a/.github/workflows/release_bundles.yml
+++ b/.github/workflows/release_bundles.yml
@@ -203,19 +203,44 @@ jobs:
             exit 0
           fi
           failed=0
+          pypi_publish_delay_seconds=60
+          pypi_publish_max_attempts=5
+          wheel_count=${#wheels[@]}
+          wheel_number=0
           for wheel in "${wheels[@]}"; do
+            wheel_number=$((wheel_number + 1))
             echo "Publishing $wheel"
-            # Capture stderr so we can tolerate "already exists" without failing
-            # the whole run — re-running this workflow without bumping a bundle
-            # version should be a no-op for that bundle.
-            if ! err=$(uv publish "$wheel" 2>&1); then
+            attempt=1
+            published=0
+            while true; do
+              # Capture stderr so we can tolerate "already exists" without failing
+              # the whole run — re-running this workflow without bumping a bundle
+              # version should be a no-op for that bundle.
+              if err=$(uv publish "$wheel" 2>&1); then
+                echo "$err"
+                published=1
+                break
+              fi
               echo "$err"
               if echo "$err" | grep -qiE 'already exists|file already exists|HTTP 400|duplicate'; then
                 echo "Skipping $wheel: already published."
-              else
-                echo "Publish failed for $wheel"
-                failed=1
+                break
               fi
+              if echo "$err" | grep -qiE 'HTTP 429|429 Too Many Requests|Too many new projects created'; then
+                if [ "$attempt" -lt "$pypi_publish_max_attempts" ]; then
+                  echo "PyPI rate limited $wheel; sleeping ${pypi_publish_delay_seconds}s before retry $((attempt + 1))/${pypi_publish_max_attempts}."
+                  sleep "$pypi_publish_delay_seconds"
+                  attempt=$((attempt + 1))
+                  continue
+                fi
+              fi
+              echo "Publish failed for $wheel"
+              failed=1
+              break
+            done
+            if [ "$published" = "1" ] && [ "$wheel_number" -lt "$wheel_count" ]; then
+              echo "Sleeping ${pypi_publish_delay_seconds}s before the next PyPI upload."
+              sleep "$pypi_publish_delay_seconds"
             fi
           done
           if [ "$failed" = "1" ]; then