Nomchy/discord.js
1
0
Fork 0
mirror of https://github.com/discordjs/discord.js.git synced 2026-03-09 16:13:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

ci: upload docs to cf

Browse Source
This commit is contained in:
iCrawl
2025-04-11 00:55:57 +02:00
parent 74110b3e7d
commit e22a7bb505
5 changed files with 1374 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
.github/workflows/documentation.yml vendored
View File

@@ -105,7 +105,15 @@ jobs:
if: ${{ env.REF_TYPE == 'tag' && (!inputs.ref || inputs.ref == 'main') }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
uses: ./packages/actions/src/uploadDocumentation
with:
package: ${{ steps.extract-tag.outputs.package }}
@@ -115,7 +123,15 @@ jobs:
if: ${{ env.REF_TYPE == 'tag' && inputs.ref && inputs.ref != 'main' }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
uses: ./main/packages/actions/src/uploadDocumentation
with:
package: ${{ steps.extract-tag.outputs.package }}
@@ -125,6 +141,10 @@ jobs:
if: ${{ env.REF_TYPE == 'tag' && (!inputs.ref || inputs.ref == 'main') }}
env:
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
uses: ./packages/actions/src/uploadSplitDocumentation
with:
package: ${{ steps.extract-tag.outputs.package }}
@@ -134,6 +154,10 @@ jobs:
if: ${{ env.REF_TYPE == 'tag' && inputs.ref && inputs.ref != 'main' }}
env:
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
uses: ./main/packages/actions/src/uploadSplitDocumentation
with:
package: ${{ steps.extract-tag.outputs.package }}
@@ -157,26 +181,50 @@ jobs:
if: ${{ env.REF_TYPE == 'branch' && (!inputs.ref || inputs.ref == 'main') }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
uses: ./packages/actions/src/uploadDocumentation
- name: Upload documentation to database
if: ${{ env.REF_TYPE == 'branch' && inputs.ref && inputs.ref != 'main' }}
env:
DATABASE_URL: ${{ secrets.DATABASE_URL }}
CF_D1_DOCS_API_KEY: ${{ secrets.CF_D1_DOCS_API_KEY }}
CF_D1_DOCS_ID: ${{ secrets.CF_D1_DOCS_ID }}
CF_ACCOUNT_ID: ${{ secrets.CF_ACCOUNT_ID }}
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
CF_R2_DOCS_BUCKET_URL: ${{ secrets.CF_R2_DOCS_BUCKET_URL }}
uses: ./main/packages/actions/src/uploadDocumentation
- name: Upload split documentation to blob storage
if: ${{ env.REF_TYPE == 'branch' && (!inputs.ref || inputs.ref == 'main') }}
env:
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
uses: ./packages/actions/src/uploadSplitDocumentation
- name: Upload split documentation to blob storage
if: ${{ env.REF_TYPE == 'branch' && inputs.ref && inputs.ref != 'main' }}
env:
BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
CF_R2_DOCS_URL: ${{ secrets.CF_R2_DOCS_URL }}
CF_R2_DOCS_ACCESS_KEY_ID: ${{ secrets.CF_R2_DOCS_ACCESS_KEY_ID }}
CF_R2_DOCS_SECRET_ACCESS_KEY: ${{ secrets.CF_R2_DOCS_SECRET_ACCESS_KEY }}
CF_R2_DOCS_BUCKET: ${{ secrets.CF_R2_DOCS_BUCKET }}
uses: ./main/packages/actions/src/uploadSplitDocumentation
- name: Move docs to correct directory

2
packages/actions/package.json
View File

@@ -43,9 +43,11 @@
"dependencies": {
"@actions/core": "^1.11.1",
"@actions/glob": "^0.5.0",
"@aws-sdk/client-s3": "^3.787.0",
"@discordjs/scripts": "workspace:^",
"@vercel/blob": "^0.27.3",
"@vercel/postgres": "^0.9.0",
"cloudflare": "^4.2.0",
"meilisearch": "^0.38.0",
"p-limit": "^6.2.0",
"p-queue": "^8.1.0",

51
packages/actions/src/uploadDocumentation/index.ts
View File

@@ -2,12 +2,24 @@ import { readFile } from 'node:fs/promises';
import process from 'node:process';
import { getInput, setFailed } from '@actions/core';
import { create } from '@actions/glob';
import { PutObjectCommand, S3Client } from '@aws-sdk/client-s3';
import { put } from '@vercel/blob';
import { createPool } from '@vercel/postgres';
import Cloudflare from 'cloudflare';
import pLimit from 'p-limit';
if (!process.env.DATABASE_URL) {
setFailed('DATABASE_URL is not set');
if (
!process.env.DATABASE_URL ||
!process.env.CF_R2_DOCS_URL ||
!process.env.CF_R2_DOCS_ACCESS_KEY_ID ||
!process.env.CF_R2_DOCS_SECRET_ACCESS_KEY ||
!process.env.CF_R2_DOCS_BUCKET ||
!process.env.CF_R2_DOCS_BUCKET_URL ||
!process.env.CF_D1_DOCS_API_KEY ||
!process.env.CF_D1_DOCS_ID ||
!process.env.CF_ACCOUNT_ID
) {
setFailed('Missing environment variables');
}
const pkg = getInput('package') || '*';
@@ -17,6 +29,21 @@ const pool = createPool({
connectionString: process.env.DATABASE_URL,
});
const S3 = new S3Client({
region: 'auto',
endpoint: process.env.CF_R2_DOCS_URL!,
credentials: {
accessKeyId: process.env.CF_R2_DOCS_ACCESS_KEY_ID!,
secretAccessKey: process.env.CF_R2_DOCS_SECRET_ACCESS_KEY!,
},
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED',
});
const client = new Cloudflare({
apiToken: process.env.CF_D1_DOCS_API_KEY,
});
const limit = pLimit(10);
const promises = [];
@@ -30,7 +57,10 @@ for await (const file of globber.globGenerator()) {
console.log(`Uploading ${file} with ${version}...`);
const json = JSON.parse(data);
const name = json.name ?? json.n;
const { url } = await put(`${name.replace('@discordjs/', '')}/${version}.json`, data, {
const key = `${name.replace('@discordjs/', '')}/${version}.json`;
const { url } = await put(key, data, {
access: 'public',
addRandomSuffix: false,
});
@@ -38,6 +68,21 @@ for await (const file of globber.globGenerator()) {
'@discordjs/',
'',
)}, ${version}, ${url}) on conflict (name, version) do update set url = EXCLUDED.url`;
await S3.send(
new PutObjectCommand({
Bucket: process.env.CF_R2_DOCS_BUCKET,
Key: key,
Body: data,
}),
);
await client.d1.database.raw(process.env.CF_D1_DOCS_ID!, {
account_id: process.env.CF_ACCOUNT_ID!,
sql: `insert into documentation (name, version, url) values (${name.replace(
'@discordjs/',
'',
)}, ${version}, ${process.env.CF_R2_DOCS_BUCKET_URL + '/' + key}) on conflict (name, version) do update set url = excluded.url`,
});
}),
);
} catch (error) {

32
packages/actions/src/uploadSplitDocumentation/index.ts
View File

@@ -1,12 +1,22 @@
import { readFile } from 'node:fs/promises';
import { basename, dirname, relative, sep } from 'node:path';
import { cwd } from 'node:process';
import process from 'node:process';
import { setTimeout as sleep } from 'node:timers/promises';
import { setFailed, getInput } from '@actions/core';
import { create } from '@actions/glob';
import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3';
import { put } from '@vercel/blob';
import PQueue from 'p-queue';
if (
!process.env.CF_R2_DOCS_URL ||
!process.env.CF_R2_DOCS_ACCESS_KEY_ID ||
!process.env.CF_R2_DOCS_SECRET_ACCESS_KEY ||
!process.env.CF_R2_DOCS_BUCKET
) {
setFailed('Missing environment variables');
}
const pkg = getInput('package') || '*';
const version = getInput('version') || 'main';
@@ -14,11 +24,22 @@ const queue = new PQueue({ concurrency: 10, interval: 60_000, intervalCap: 1_000
const promises = [];
const failedUploads: string[] = [];
const S3 = new S3Client({
region: 'auto',
endpoint: process.env.CF_R2_DOCS_URL!,
credentials: {
accessKeyId: process.env.CF_R2_DOCS_ACCESS_KEY_ID!,
secretAccessKey: process.env.CF_R2_DOCS_SECRET_ACCESS_KEY!,
},
requestChecksumCalculation: 'WHEN_REQUIRED',
responseChecksumValidation: 'WHEN_REQUIRED',
});
const globber = await create(`packages/${pkg}/docs/${pkg}/split/*.api.json`);
console.log('Glob: ', await globber.glob());
for await (const file of globber.globGenerator()) {
const data = await readFile(file, 'utf8');
const pkgName = dirname(relative(cwd(), file)).split(sep)[1];
const pkgName = dirname(relative(process.cwd(), file)).split(sep)[1];
try {
promises.push(
queue.add(async () => {
@@ -30,6 +51,13 @@ for await (const file of globber.globGenerator()) {
access: 'public',
addRandomSuffix: false,
});
await S3.send(
new PutObjectCommand({
Bucket: process.env.CF_R2_DOCS_BUCKET,
Key: `${pkgName}/${version}.${name}`,
Body: data,
}),
);
} catch (error) {
if (retries > 3) {
console.error(`Could not upload ${file} after 3 retries`, error);

1246
pnpm-lock.yaml generated
View File

File diff suppressed because it is too large Load Diff