chore(ci): backport actions updates (#10853)

.github/workflows/cleanup-cache.yml

@@ -1,4 +1,4 @@
-# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
+# https://docs.github.com/actions/using-workflows/caching-dependencies-to-speed-up-workflows#force-deleting-cache-entries
 name: Cleanup caches
 on:
   pull_request:

.github/workflows/documentation.yml

@@ -103,7 +103,15 @@ jobs:
         if: ${{ env.REF_TYPE == 'tag' && (!inputs.ref || inputs.ref == 'main') }}
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
         uses: ./packages/actions/src/uploadDocumentation
         with:
           package: ${{ steps.extract-tag.outputs.package }}
@@ -113,7 +121,15 @@ jobs:
         if: ${{ env.REF_TYPE == 'tag' && inputs.ref && inputs.ref != 'main' }}
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
         uses: ./main/packages/actions/src/uploadDocumentation
         with:
           package: ${{ steps.extract-tag.outputs.package }}
@@ -123,6 +139,10 @@ jobs:
         if: ${{ env.REF_TYPE == 'tag' && (!inputs.ref || inputs.ref == 'main') }}
         env:
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
         uses: ./packages/actions/src/uploadSplitDocumentation
         with:
           package: ${{ steps.extract-tag.outputs.package }}
@@ -132,6 +152,10 @@ jobs:
         if: ${{ env.REF_TYPE == 'tag' && inputs.ref && inputs.ref != 'main' }}
         env:
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
         uses: ./main/packages/actions/src/uploadSplitDocumentation
         with:
           package: ${{ steps.extract-tag.outputs.package }}
@@ -155,26 +179,50 @@ jobs:
         if: ${{ env.REF_TYPE == 'branch' && (!inputs.ref || inputs.ref == 'main') }}
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
         uses: ./packages/actions/src/uploadDocumentation
 
       - name: Upload documentation to database
         if: ${{ env.REF_TYPE == 'branch' && inputs.ref && inputs.ref != 'main' }}
         env:
           DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
+          CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
+          CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
+          CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
         uses: ./main/packages/actions/src/uploadDocumentation
 
       - name: Upload split documentation to blob storage
         if: ${{ env.REF_TYPE == 'branch' && (!inputs.ref || inputs.ref == 'main') }}
         env:
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
         uses: ./packages/actions/src/uploadSplitDocumentation
 
       - name: Upload split documentation to blob storage
         if: ${{ env.REF_TYPE == 'branch' && inputs.ref && inputs.ref != 'main' }}
         env:
           BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
+          CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
+          CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
+          CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
+          CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
         uses: ./main/packages/actions/src/uploadSplitDocumentation
 
       - name: Move docs to correct directory

packages/actions/package.json

@@ -41,28 +41,32 @@
 	"homepage": "https://discord.js.org",
 	"funding": "https://github.com/discordjs/discord.js?sponsor",
 	"dependencies": {
-		"@actions/core": "^1.10.1",
+		"@actions/core": "^1.11.1",
 		"@actions/glob": "^0.5.0",
+		"@aws-sdk/client-s3": "^3.787.0",
 		"@discordjs/scripts": "workspace:^",
-		"@vercel/blob": "^0.23.4",
+		"@vercel/blob": "^0.27.3",
 		"@vercel/postgres": "^0.9.0",
+		"cloudflare": "^4.2.0",
 		"meilisearch": "^0.38.0",
-		"p-limit": "^6.1.0",
-		"tslib": "^2.6.3",
-		"undici": "6.21.1"
+		"p-limit": "^6.2.0",
+		"p-queue": "^8.1.0",
+		"tslib": "^2.8.1",
+		"undici": "7.8.0"
 	},
 	"devDependencies": {
-		"@types/node": "^18.19.45",
-		"@vitest/coverage-v8": "^2.0.5",
+		"@types/node": "^22.14.0",
+		"@vitest/coverage-v8": "^3.1.1",
 		"cross-env": "^7.0.3",
 		"eslint": "^8.57.0",
 		"eslint-config-neon": "^0.1.62",
 		"eslint-formatter-pretty": "^6.0.1",
-		"prettier": "^3.3.3",
-		"tsup": "^8.2.4",
-		"turbo": "^2.0.14",
-		"typescript": "~5.5.4",
-		"vitest": "^2.0.5"
+		"prettier": "^3.5.3",
+		"terser": "^5.37.0",
+		"tsup": "^8.4.0",
+		"turbo": "^2.5.0",
+		"typescript": "~5.8.3",
+		"vitest": "^3.1.1"
 	},
 	"engines": {
 		"node": ">=18"

packages/actions/src/pnpmCache/action.yml

@@ -9,7 +9,7 @@ runs:
       with:
         swap-size-gb: 10
 
-    - uses: pnpm/action-setup@v4.0.0
+    - uses: pnpm/action-setup@v4.1.0
       name: Install pnpm
       with:
         run_install: false

packages/actions/src/uploadDocumentation/index.ts

@@ -1,13 +1,26 @@
+/* eslint-disable @typescript-eslint/no-loop-func */
 import { readFile } from 'node:fs/promises';
 import process from 'node:process';
 import { getInput, setFailed } from '@actions/core';
 import { create } from '@actions/glob';
+import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
 import { put } from '@vercel/blob';
 import { createPool } from '@vercel/postgres';
+import Cloudflare from 'cloudflare';
 import pLimit from 'p-limit';
 
-if (!process.env.DATABASE_URL) {
-	setFailed('DATABASE_URL is not set');
+if (
+	!process.env.DATABASE_URL ||
+	!process.env.CF_R2_DOCS_URL ||
+	!process.env.CF_R2_DOCS_ACCESS_KEY_ID ||
+	!process.env.CF_R2_DOCS_SECRET_ACCESS_KEY ||
+	!process.env.CF_R2_DOCS_BUCKET ||
+	!process.env.CF_R2_DOCS_BUCKET_URL ||
+	!process.env.CF_D1_DOCS_API_KEY ||
+	!process.env.CF_D1_DOCS_ID ||
+	!process.env.CF_ACCOUNT_ID
+) {
+	setFailed('Missing environment variables');
 }
 
 const pkg = getInput('package') || '*';
@@ -17,6 +30,21 @@ const pool = createPool({
 	connectionString: process.env.DATABASE_URL,
 });
 
+const S3 = new S3Client({
+	region: 'auto',
+	endpoint: process.env.CF_R2_DOCS_URL!,
+	credentials: {
+		accessKeyId: process.env.CF_R2_DOCS_ACCESS_KEY_ID!,
+		secretAccessKey: process.env.CF_R2_DOCS_SECRET_ACCESS_KEY!,
+	},
+	requestChecksumCalculation: 'WHEN_REQUIRED',
+	responseChecksumValidation: 'WHEN_REQUIRED',
+});
+
+const client = new Cloudflare({
+	apiToken: process.env.CF_D1_DOCS_API_KEY,
+});
+
 const limit = pLimit(10);
 const promises = [];
 
@@ -26,12 +54,14 @@ for await (const file of globber.globGenerator()) {
 	const data = await readFile(file, 'utf8');
 	try {
 		promises.push(
-			// eslint-disable-next-line @typescript-eslint/no-loop-func
 			limit(async () => {
 				console.log(`Uploading ${file} with ${version}...`);
 				const json = JSON.parse(data);
 				const name = json.name ?? json.n;
-				const { url } = await put(`${name.replace('@discordjs/', '')}/${version}.json`, data, {
+
+				const key = `${name.replace('@discordjs/', '')}/${version}.json`;
+
+				const { url } = await put(key, data, {
 					access: 'public',
 					addRandomSuffix: false,
 				});
@@ -39,6 +69,19 @@ for await (const file of globber.globGenerator()) {
 					'@discordjs/',
 					'',
 				)}, ${version}, ${url}) on conflict (name, version) do update set url = EXCLUDED.url`;
+
+				await S3.send(
+					new PutObjectCommand({
+						Bucket: process.env.CF_R2_DOCS_BUCKET,
+						Key: key,
+						Body: data,
+					}),
+				);
+				await client.d1.database.raw(process.env.CF_D1_DOCS_ID!, {
+					account_id: process.env.CF_ACCOUNT_ID!,
+					sql: `insert into documentation (name, version, url) values (?, ?, ?) on conflict (name, version) do update set url = excluded.url;`,
+					params: [name.replace('@discordjs/', ''), version, process.env.CF_R2_DOCS_BUCKET_URL + '/' + key],
+				});
 			}),
 		);
 	} catch (error) {

packages/actions/src/uploadSplitDocumentation/index.ts

@@ -1,32 +1,82 @@
+/* eslint-disable @typescript-eslint/no-loop-func */
 import { readFile } from 'node:fs/promises';
 import { basename, dirname, relative, sep } from 'node:path';
-import { cwd } from 'node:process';
-import { getInput } from '@actions/core';
+import process from 'node:process';
+import { setTimeout as sleep } from 'node:timers/promises';
+import { setFailed, getInput } from '@actions/core';
 import { create } from '@actions/glob';
+import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
 import { put } from '@vercel/blob';
-import pLimit from 'p-limit';
+import PQueue from 'p-queue';
+
+if (
+	!process.env.CF_R2_DOCS_URL ||
+	!process.env.CF_R2_DOCS_ACCESS_KEY_ID ||
+	!process.env.CF_R2_DOCS_SECRET_ACCESS_KEY ||
+	!process.env.CF_R2_DOCS_BUCKET
+) {
+	setFailed('Missing environment variables');
+}
 
 const pkg = getInput('package') || '*';
 const version = getInput('version') || 'main';
 
-const limit = pLimit(10);
+const queue = new PQueue({ concurrency: 10, interval: 60_000, intervalCap: 1_000 });
 const promises = [];
+const failedUploads: string[] = [];
+
+const S3 = new S3Client({
+	region: 'auto',
+	endpoint: process.env.CF_R2_DOCS_URL!,
+	credentials: {
+		accessKeyId: process.env.CF_R2_DOCS_ACCESS_KEY_ID!,
+		secretAccessKey: process.env.CF_R2_DOCS_SECRET_ACCESS_KEY!,
+	},
+	requestChecksumCalculation: 'WHEN_REQUIRED',
+	responseChecksumValidation: 'WHEN_REQUIRED',
+});
 
 const globber = await create(`packages/${pkg}/docs/${pkg}/split/*.api.json`);
 console.log('Glob: ', await globber.glob());
 for await (const file of globber.globGenerator()) {
 	const data = await readFile(file, 'utf8');
-	const pkgName = dirname(relative(cwd(), file)).split(sep)[1];
+	const pkgName = dirname(relative(process.cwd(), file)).split(sep)[1];
 	try {
 		promises.push(
-			// eslint-disable-next-line @typescript-eslint/no-loop-func
-			limit(async () => {
+			queue.add(async () => {
 				console.log(`Uploading ${file} with ${version} from ${pkgName}...`);
 				const name = basename(file).replace('main.', '');
-				await put(`rewrite/${pkgName}/${version}.${name}`, data, {
-					access: 'public',
-					addRandomSuffix: false,
-				});
+				async function upload(retries = 0) {
+					try {
+						await put(`rewrite/${pkgName}/${version}.${name}`, data, {
+							access: 'public',
+							addRandomSuffix: false,
+						});
+						await S3.send(
+							new PutObjectCommand({
+								Bucket: process.env.CF_R2_DOCS_BUCKET,
+								Key: `${pkgName}/${version}.${name}`,
+								Body: data,
+							}),
+						);
+					} catch (error) {
+						if (retries > 3) {
+							console.error(`Could not upload ${file} after 3 retries`, error);
+							failedUploads.push(name);
+							return;
+						}
+
+						if (typeof error === 'object' && error && 'retryAfter' in error && typeof error.retryAfter === 'number') {
+							await sleep(error.retryAfter * 1_000);
+							return upload(retries + 1);
+						} else {
+							console.error(`Could not upload ${file}`, error);
+							failedUploads.push(name);
+						}
+					}
+				}
+
+				await upload();
 			}),
 		);
 	} catch (error) {
@@ -36,6 +86,9 @@ for await (const file of globber.globGenerator()) {
 
 try {
 	await Promise.all(promises);
+	if (failedUploads.length) {
+		setFailed(`Failed to upload ${failedUploads.length} files: ${failedUploads.join(', ')}`);
+	}
 } catch (error) {
 	console.log(error);
 }

packages/actions/tsconfig.test.json

@@ -0,0 +1,10 @@
+{
+	"$schema": "https://json.schemastore.org/tsconfig.json",
+	"extends": "./tsconfig.json",
+	"compilerOptions": {
+		"noEmit": true,
+		"skipLibCheck": true
+	},
+	"include": ["__tests__/**/*.ts"],
+	"exclude": ["node_modules"]
+}