From 01e64b4e9aeb084de13314f8a35e8a5c9170190a Mon Sep 17 00:00:00 2001
From: ckohen <chaikohen@gmail.com>
Date: Sun, 12 Jan 2025 10:05:35 -0800
Subject: [PATCH] fix(actions): respect ratelimits on split docs upload
 (#10697)

* fix(actions): respect ratelimits on split docs upload

* fix: set failed on missing uploads
---
 packages/actions/package.json                 |  1 +
 .../src/uploadSplitDocumentation/index.ts     | 41 +++++++++++++++----
 pnpm-lock.yaml                                | 20 ++++++++-
 3 files changed, 53 insertions(+), 9 deletions(-)

diff --git a/packages/actions/package.json b/packages/actions/package.json
index bd811d475..8b1a202df 100644
--- a/packages/actions/package.json
+++ b/packages/actions/package.json
@@ -48,6 +48,7 @@
 		"@vercel/postgres": "^0.9.0",
 		"meilisearch": "^0.38.0",
 		"p-limit": "^6.2.0",
+		"p-queue": "^8.0.1",
 		"tslib": "^2.8.1",
 		"undici": "6.21.0"
 	},
diff --git a/packages/actions/src/uploadSplitDocumentation/index.ts b/packages/actions/src/uploadSplitDocumentation/index.ts
index ea2b5887b..06cfb4cff 100644
--- a/packages/actions/src/uploadSplitDocumentation/index.ts
+++ b/packages/actions/src/uploadSplitDocumentation/index.ts
@@ -1,16 +1,18 @@
 import { readFile } from 'node:fs/promises';
 import { basename, dirname, relative, sep } from 'node:path';
 import { cwd } from 'node:process';
-import { getInput } from '@actions/core';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { setFailed, getInput } from '@actions/core';
 import { create } from '@actions/glob';
 import { put } from '@vercel/blob';
-import pLimit from 'p-limit';
+import PQueue from 'p-queue';
 
 const pkg = getInput('package') || '*';
 const version = getInput('version') || 'main';
 
-const limit = pLimit(10);
+const queue = new PQueue({ concurrency: 10, interval: 60_000, intervalCap: 1_000 });
 const promises = [];
+const failedUploads: string[] = [];
 
 const globber = await create(`packages/${pkg}/docs/${pkg}/split/*.api.json`);
 console.log('Glob: ', await globber.glob());
@@ -20,13 +22,33 @@ for await (const file of globber.globGenerator()) {
 	try {
 		promises.push(
 			// eslint-disable-next-line @typescript-eslint/no-loop-func
-			limit(async () => {
+			queue.add(async () => {
 				console.log(`Uploading ${file} with ${version} from ${pkgName}...`);
 				const name = basename(file).replace('main.', '');
-				await put(`rewrite/${pkgName}/${version}.${name}`, data, {
-					access: 'public',
-					addRandomSuffix: false,
-				});
+				async function upload(retries = 0) {
+					try {
+						await put(`rewrite/${pkgName}/${version}.${name}`, data, {
+							access: 'public',
+							addRandomSuffix: false,
+						});
+					} catch (error) {
+						if (retries > 3) {
+							console.error(`Could not upload ${file} after 3 retries`, error);
+							failedUploads.push(name);
+							return;
+						}
+
+						if (typeof error === 'object' && error && 'retryAfter' in error && typeof error.retryAfter === 'number') {
+							await sleep(error.retryAfter * 1_000);
+							return upload(retries + 1);
+						} else {
+							console.error(`Could not upload ${file}`, error);
+							failedUploads.push(name);
+						}
+					}
+				}
+
+				await upload();
 			}),
 		);
 	} catch (error) {
@@ -36,6 +58,9 @@ for await (const file of globber.globGenerator()) {
 
 try {
 	await Promise.all(promises);
+	if (failedUploads.length) {
+		setFailed(`Failed to upload ${failedUploads.length} files: ${failedUploads.join(', ')}`);
+	}
 } catch (error) {
 	console.log(error);
 }
diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml
index 993fb92e7..9e4767b27 100644
--- a/pnpm-lock.yaml
+++ b/pnpm-lock.yaml
@@ -404,6 +404,9 @@ importers:
       p-limit:
         specifier: ^6.2.0
         version: 6.2.0
+      p-queue:
+        specifier: ^8.0.1
+        version: 8.0.1
       tslib:
         specifier: ^2.8.1
         version: 2.8.1
@@ -10650,10 +10653,18 @@ packages:
     resolution: {integrity: sha512-VkndIv2fIB99swvQoA65bm+fsmt6UNdGeIB0oxBs+WhAhdh08QA04JXpI7rbB9r08/nkbysKoya9rtDERYOYMA==}
     engines: {node: '>=18'}
 
+  p-queue@8.0.1:
+    resolution: {integrity: sha512-NXzu9aQJTAzbBqOt2hwsR63ea7yvxJc0PwN/zobNAudYfb1B7R08SzB4TsLeSbUCuG467NhnoT0oO6w1qRO+BA==}
+    engines: {node: '>=18'}
+
   p-timeout@5.1.0:
     resolution: {integrity: sha512-auFDyzzzGZZZdHz3BtET9VEz0SE/uMEAx7uWfGPucfzEwwe/xH0iVeZibQmANYE/hp9T2+UUZT5m+BKyrDp3Ew==}
     engines: {node: '>=12'}
 
+  p-timeout@6.1.4:
+    resolution: {integrity: sha512-MyIV3ZA/PmyBN/ud8vV9XzwTrNtR4jFrObymZYnZqMmW0zA8Z17vnT0rBgFE/TlohB+YCHqXMgZzb3Csp49vqg==}
+    engines: {node: '>=14.16'}
+
   p-try@2.2.0:
     resolution: {integrity: sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==}
     engines: {node: '>=6'}
@@ -17790,7 +17801,7 @@ snapshots:
 
   '@types/conventional-commits-parser@5.0.1':
     dependencies:
-      '@types/node': 22.10.2
+      '@types/node': 20.17.10
 
   '@types/cookiejar@2.1.5': {}
 
@@ -25153,8 +25164,15 @@ snapshots:
 
   p-map@7.0.3: {}
 
+  p-queue@8.0.1:
+    dependencies:
+      eventemitter3: 5.0.1
+      p-timeout: 6.1.4
+
   p-timeout@5.1.0: {}
 
+  p-timeout@6.1.4: {}
+
   p-try@2.2.0: {}
 
   pac-proxy-agent@7.1.0: