diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index 429cdbfe9..c3aeba2ed 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -5,7 +5,13 @@ on: - 'main' paths: - 'packages/*/src/**' + - '!packages/actions/**' + - '!packages/api-extractor-utils/**' - '!packages/create-discord-bot/**' + - '!packages/docgen/**' + - '!packages/proxy-container/**' + - '!packages/scripts/**' + - '!packages/ui/**' tags: - '**' workflow_dispatch: diff --git a/.github/workflows/search-indicies.yml b/.github/workflows/search-indicies.yml new file mode 100644 index 000000000..f5471b445 --- /dev/null +++ b/.github/workflows/search-indicies.yml @@ -0,0 +1,48 @@ +name: Search indicies +on: + push: + branches: + - 'main' + paths: + - 'packages/*/src/**' + - '!packages/actions/**' + - '!packages/api-extractor-utils/**' + - '!packages/create-discord-bot/**' + - '!packages/docgen/**' + - '!packages/proxy-container/**' + - '!packages/scripts/**' + - '!packages/ui/**' + tags: + - '**' + workflow_dispatch: +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.ref }} + cancel-in-progress: true +jobs: + build: + name: Build & upload search indicies + runs-on: ubuntu-latest + env: + TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }} + TURBO_TEAM: ${{ secrets.TURBO_TEAM }} + if: github.repository_owner == 'discordjs' + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Install node.js v18 + uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Install dependencies + uses: ./packages/actions/src/pnpmCache + + - name: Build dependencies + run: pnpm run build + + - name: Upload search indicies to meilisearch + env: + DATABASE_URL: ${{ secrets.DATABASE_URL }} + SEARCH_API_KEY: ${{ secrets.SEARCH_API_KEY }} + uses: ./packages/actions/src/uploadSearchIndicies diff --git a/apps/website/src/app/docAPI.ts b/apps/website/src/app/docAPI.ts index 6208d3673..c010caf0b 100644 --- a/apps/website/src/app/docAPI.ts +++ b/apps/website/src/app/docAPI.ts @@ -20,7 +20,7 @@ export async function fetchVersions(packageName: string): Promise { ]); // @ts-expect-error: https://github.com/planetscale/database-js/issues/71 - return rows[0].data; + return rows.map((row) => row.version); } export async function fetchModelJSON(packageName: string, version: string): Promise { diff --git a/packages/actions/package.json b/packages/actions/package.json index cedae6442..c7e5e210c 100644 --- a/packages/actions/package.json +++ b/packages/actions/package.json @@ -42,7 +42,9 @@ "dependencies": { "@actions/core": "^1.10.1", "@actions/glob": "^0.4.0", + "@discordjs/scripts": "workspace:^", "@planetscale/database": "^1.11.0", + "meilisearch": "^0.35.0", "tslib": "^2.6.2", "undici": "5.27.2" }, diff --git a/packages/actions/src/uploadSearchIndicies/action.yml b/packages/actions/src/uploadSearchIndicies/action.yml new file mode 100644 index 000000000..513226204 --- /dev/null +++ b/packages/actions/src/uploadSearchIndicies/action.yml @@ -0,0 +1,5 @@ +name: 'Upload search indicies' +description: 'Uploads the search indicies to a meilisearch database' +runs: + using: node16 + main: ../../dist/uploadSearchIndicies/index.js diff --git a/packages/actions/src/uploadSearchIndicies/index.ts b/packages/actions/src/uploadSearchIndicies/index.ts new file mode 100644 index 000000000..369f2b0bd --- /dev/null +++ b/packages/actions/src/uploadSearchIndicies/index.ts @@ -0,0 +1,64 @@ +import process from 'node:process'; +import { setFailed } from '@actions/core'; +import { generateAllIndices } from '@discordjs/scripts'; +import { connect } from '@planetscale/database'; +import MeiliSearch from 'meilisearch'; +import { fetch } from 'undici'; + +if (!process.env.DATABASE_URL) { + setFailed('DATABASE_URL is not set'); +} + +if (!process.env.SEARCH_API) { + setFailed('SEARCH_API is not set'); +} + +if (!process.env.SEARCH_API_KEY) { + setFailed('SEARCH_API_KEY is not set'); +} + +const sql = connect({ + fetch, + url: process.env.DATABASE_URL!, +}); + +const client = new MeiliSearch({ + host: process.env.SEARCH_API!, + apiKey: process.env.SEARCH_API_KEY!, +}); + +try { + console.log('Generating all indices...'); + const indicies = await generateAllIndices({ + fetchPackageVersions: async (pkg) => { + console.log(`Fetching versions for ${pkg}...`); + const { rows } = await sql.execute('select version from documentation where name = ?', [pkg]); + + // @ts-expect-error: https://github.com/planetscale/database-js/issues/71 + return rows.map((row) => row.version); + }, + fetchPackageVersionDocs: async (pkg, version) => { + console.log(`Fetching data for ${pkg} ${version}...`); + const { rows } = await sql.execute('select data from documentation where name = ? and version = ?', [ + pkg, + version, + ]); + + // @ts-expect-error: https://github.com/planetscale/database-js/issues/71 + return rows[0].data; + }, + writeToFile: false, + }); + console.log('Generated all indices.'); + + console.log('Uploading indices...'); + for (const index of indicies) { + console.log(`Uploading ${index.index}...`); + await client.index(index.index).addDocuments([index]); + } + + console.log('Uploaded all indices.'); +} catch (error) { + const err = error as Error; + setFailed(err.message); +} diff --git a/packages/scripts/src/generateIndex.ts b/packages/scripts/src/generateIndex.ts index 247da764c..eb70eddd5 100644 --- a/packages/scripts/src/generateIndex.ts +++ b/packages/scripts/src/generateIndex.ts @@ -135,9 +135,11 @@ export function visitNodes(item: ApiItem, tag: string) { return members; } -export async function generateIndex(model: ApiModel, packageName: string, tag = 'main') { - const members = visitNodes(model.tryGetPackageByName(packageName)!.entryPoints[0]!, tag); - +export async function writeIndexToFileSystem( + members: ReturnType, + packageName: string, + tag = 'main', +) { const dir = 'searchIndex'; try { @@ -147,24 +149,44 @@ export async function generateIndex(model: ApiModel, packageName: string, tag = } await writeFile( - join(cwd(), 'public', dir, `${packageName}-${tag}-index.json`), + join(cwd(), 'public', dir, `${packageName}-${tag.replaceAll('.', '-')}-index.json`), JSON.stringify(members, undefined, 2), ); } -export async function generateAllIndices() { +export async function fetchVersions(pkg: string) { + const response = await request(`https://docs.discordjs.dev/api/info?package=${pkg}`); + return response.body.json() as Promise; +} + +export async function fetchVersionDocs(pkg: string, version: string) { + const response = await request(`https://docs.discordjs.dev/docs/${pkg}/${version}.api.json`); + return response.body.json() as Promise>; +} + +export async function generateAllIndices({ + fetchPackageVersions = fetchVersions, + fetchPackageVersionDocs = fetchVersionDocs, + writeToFile = true, +}) { + const indicies: Record[] = []; + for (const pkg of PACKAGES) { - const response = await request(`https://docs.discordjs.dev/api/info?package=${pkg}`); - const versions = (await response.body.json()) as any; + const versions = await fetchPackageVersions(pkg); for (const version of versions) { idx = 0; - const versionRes = await request(`https://docs.discordjs.dev/docs/${pkg}/${version}.api.json`); - const data = await versionRes.body.json(); - + const data = await fetchPackageVersionDocs(pkg, version); const model = addPackageToModel(new ApiModel(), data); - await generateIndex(model, pkg, version); + const members = visitNodes(model.tryGetPackageByName(pkg)!.entryPoints[0]!, version); + if (writeToFile) { + await writeIndexToFileSystem(members, pkg, version); + } else { + indicies.push({ index: `${pkg}-${version.replaceAll('.', '-')}`, data: members }); + } } } + + return indicies; } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 313afb8c6..60e1da9e3 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -316,7 +316,7 @@ importers: version: 14.0.0(react-dom@18.2.0)(react@18.2.0) '@testing-library/user-event': specifier: ^14.5.1 - version: 14.5.1 + version: 14.5.1(@testing-library/dom@9.3.3) '@types/node': specifier: 18.18.8 version: 18.18.8 @@ -337,7 +337,7 @@ importers: version: 0.57.2 '@vitejs/plugin-react': specifier: ^4.1.1 - version: 4.1.1 + version: 4.1.1(vite@4.5.0) '@vitest/coverage-v8': specifier: ^0.34.6 version: 0.34.6(vitest@0.34.6) @@ -389,9 +389,15 @@ importers: '@actions/glob': specifier: ^0.4.0 version: 0.4.0 + '@discordjs/scripts': + specifier: workspace:^ + version: link:../scripts '@planetscale/database': specifier: ^1.11.0 version: 1.11.0 + meilisearch: + specifier: ^0.35.0 + version: 0.35.0 tslib: specifier: ^2.6.2 version: 2.6.2 @@ -6808,13 +6814,6 @@ packages: react-dom: 18.2.0(react@18.2.0) dev: true - /@testing-library/user-event@14.5.1: - resolution: {integrity: sha512-UCcUKrUYGj7ClomOo2SpNVvx4/fkd/2BbIHDCle8A0ax+P3bU7yJwDBDrS6ZwdTMARWTGODX1hEsCcO+7beJjg==} - engines: {node: '>=12', npm: '>=6'} - peerDependencies: - '@testing-library/dom': '>=7.21.4' - dev: true - /@testing-library/user-event@14.5.1(@testing-library/dom@9.3.3): resolution: {integrity: sha512-UCcUKrUYGj7ClomOo2SpNVvx4/fkd/2BbIHDCle8A0ax+P3bU7yJwDBDrS6ZwdTMARWTGODX1hEsCcO+7beJjg==} engines: {node: '>=12', npm: '>=6'} @@ -8041,21 +8040,6 @@ packages: - supports-color dev: true - /@vitejs/plugin-react@4.1.1: - resolution: {integrity: sha512-Jie2HERK+uh27e+ORXXwEP5h0Y2lS9T2PRGbfebiHGlwzDO0dEnd2aNtOR/qjBlPb1YgxwAONeblL1xqLikLag==} - engines: {node: ^14.18.0 || >=16.0.0} - peerDependencies: - vite: ^4.2.0 - dependencies: - '@babel/core': 7.23.2 - '@babel/plugin-transform-react-jsx-self': 7.22.5(@babel/core@7.23.2) - '@babel/plugin-transform-react-jsx-source': 7.22.5(@babel/core@7.23.2) - '@types/babel__core': 7.20.3 - react-refresh: 0.14.0 - transitivePeerDependencies: - - supports-color - dev: true - /@vitejs/plugin-react@4.1.1(vite@4.5.0): resolution: {integrity: sha512-Jie2HERK+uh27e+ORXXwEP5h0Y2lS9T2PRGbfebiHGlwzDO0dEnd2aNtOR/qjBlPb1YgxwAONeblL1xqLikLag==} engines: {node: ^14.18.0 || >=16.0.0} @@ -10776,7 +10760,7 @@ packages: dependencies: semver: 7.5.4 shelljs: 0.8.5 - typescript: 5.4.0-dev.20231106 + typescript: 5.4.0-dev.20231107 dev: true /dts-critic@3.3.11(typescript@5.2.2): @@ -20470,8 +20454,8 @@ packages: engines: {node: '>=14.17'} hasBin: true - /typescript@5.4.0-dev.20231106: - resolution: {integrity: sha512-Uxjjt7meOHOSXpQJkiObTPUXfQKqSkdcMQOV+ZAjo3U1w/D/1Qah5Pdg0x/jt/fIA6laCNWLogS+QiaT5HOohw==} + /typescript@5.4.0-dev.20231107: + resolution: {integrity: sha512-4bQlfsyFjzZMo1bSLEkFJEcvuQ64pgE0S/BpdWu16lYsQOfJZtntZIyWdED3h+e96IMtVgrC6sxzfPZhj4J/dw==} engines: {node: '>=14.17'} hasBin: true dev: true @@ -21082,28 +21066,6 @@ packages: unist-util-stringify-position: 3.0.3 vfile-message: 3.1.4 - /vite-node@0.34.6(@types/node@18.18.8): - resolution: {integrity: sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==} - engines: {node: '>=v14.18.0'} - hasBin: true - dependencies: - cac: 6.7.14 - debug: 4.3.4 - mlly: 1.4.2 - pathe: 1.1.1 - picocolors: 1.0.0 - vite: 4.5.0(@types/node@18.18.8) - transitivePeerDependencies: - - '@types/node' - - less - - lightningcss - - sass - - stylus - - sugarss - - supports-color - - terser - dev: true - /vite-node@0.34.6(@types/node@18.18.8)(terser@5.24.0): resolution: {integrity: sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==} engines: {node: '>=v14.18.0'} @@ -21186,42 +21148,6 @@ packages: fsevents: 2.3.3 dev: true - /vite@4.5.0(@types/node@18.18.8): - resolution: {integrity: sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==} - engines: {node: ^14.18.0 || >=16.0.0} - hasBin: true - peerDependencies: - '@types/node': '>= 14' - less: '*' - lightningcss: ^1.21.0 - sass: '*' - stylus: '*' - sugarss: '*' - terser: ^5.4.0 - peerDependenciesMeta: - '@types/node': - optional: true - less: - optional: true - lightningcss: - optional: true - sass: - optional: true - stylus: - optional: true - sugarss: - optional: true - terser: - optional: true - dependencies: - '@types/node': 18.18.8 - esbuild: 0.18.20 - postcss: 8.4.31 - rollup: 3.29.4 - optionalDependencies: - fsevents: 2.3.3 - dev: true - /vite@4.5.0(@types/node@18.18.8)(terser@5.24.0): resolution: {integrity: sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==} engines: {node: ^14.18.0 || >=16.0.0} @@ -21312,8 +21238,8 @@ packages: strip-literal: 1.3.0 tinybench: 2.5.1 tinypool: 0.7.0 - vite: 4.5.0(@types/node@18.18.8) - vite-node: 0.34.6(@types/node@18.18.8) + vite: 4.5.0(@types/node@18.18.8)(terser@5.24.0) + vite-node: 0.34.6(@types/node@18.18.8)(terser@5.24.0) why-is-node-running: 2.2.2 transitivePeerDependencies: - less