ci: update search with github actions

.github/workflows/documentation.yml

@@ -5,7 +5,13 @@ on:
       - 'main'
     paths:
       - 'packages/*/src/**'
+      - '!packages/actions/**'
+      - '!packages/api-extractor-utils/**'
       - '!packages/create-discord-bot/**'
+      - '!packages/docgen/**'
+      - '!packages/proxy-container/**'
+      - '!packages/scripts/**'
+      - '!packages/ui/**'
     tags:
       - '**'
   workflow_dispatch:

.github/workflows/search-indicies.yml

@@ -0,0 +1,48 @@
+name: Search indicies
+on:
+  push:
+    branches:
+      - 'main'
+    paths:
+      - 'packages/*/src/**'
+      - '!packages/actions/**'
+      - '!packages/api-extractor-utils/**'
+      - '!packages/create-discord-bot/**'
+      - '!packages/docgen/**'
+      - '!packages/proxy-container/**'
+      - '!packages/scripts/**'
+      - '!packages/ui/**'
+    tags:
+      - '**'
+  workflow_dispatch:
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
+  cancel-in-progress: true
+jobs:
+  build:
+    name: Build & upload search indicies
+    runs-on: ubuntu-latest
+    env:
+      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
+      TURBO_TEAM: ${{ secrets.TURBO_TEAM }}
+    if: github.repository_owner == 'discordjs'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Install node.js v18
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+
+      - name: Install dependencies
+        uses: ./packages/actions/src/pnpmCache
+
+      - name: Build dependencies
+        run: pnpm run build
+
+      - name: Upload search indicies to meilisearch
+        env:
+          DATABASE_URL: ${{ secrets.DATABASE_URL }}
+          SEARCH_API_KEY: ${{ secrets.SEARCH_API_KEY }}
+        uses: ./packages/actions/src/uploadSearchIndicies

apps/website/src/app/docAPI.ts

@@ -20,7 +20,7 @@ export async function fetchVersions(packageName: string): Promise<string[]> {
 	]);
 
 	// @ts-expect-error: https://github.com/planetscale/database-js/issues/71
-	return rows[0].data;
+	return rows.map((row) => row.version);
 }
 
 export async function fetchModelJSON(packageName: string, version: string): Promise<unknown> {

packages/actions/package.json

@@ -42,7 +42,9 @@
 	"dependencies": {
 		"@actions/core": "^1.10.1",
 		"@actions/glob": "^0.4.0",
+		"@discordjs/scripts": "workspace:^",
 		"@planetscale/database": "^1.11.0",
+		"meilisearch": "^0.35.0",
 		"tslib": "^2.6.2",
 		"undici": "5.27.2"
 	},

packages/actions/src/uploadSearchIndicies/action.yml

@@ -0,0 +1,5 @@
+name: 'Upload search indicies'
+description: 'Uploads the search indicies to a meilisearch database'
+runs:
+  using: node16
+  main: ../../dist/uploadSearchIndicies/index.js

packages/actions/src/uploadSearchIndicies/index.ts

@@ -0,0 +1,64 @@
+import process from 'node:process';
+import { setFailed } from '@actions/core';
+import { generateAllIndices } from '@discordjs/scripts';
+import { connect } from '@planetscale/database';
+import MeiliSearch from 'meilisearch';
+import { fetch } from 'undici';
+
+if (!process.env.DATABASE_URL) {
+	setFailed('DATABASE_URL is not set');
+}
+
+if (!process.env.SEARCH_API) {
+	setFailed('SEARCH_API is not set');
+}
+
+if (!process.env.SEARCH_API_KEY) {
+	setFailed('SEARCH_API_KEY is not set');
+}
+
+const sql = connect({
+	fetch,
+	url: process.env.DATABASE_URL!,
+});
+
+const client = new MeiliSearch({
+	host: process.env.SEARCH_API!,
+	apiKey: process.env.SEARCH_API_KEY!,
+});
+
+try {
+	console.log('Generating all indices...');
+	const indicies = await generateAllIndices({
+		fetchPackageVersions: async (pkg) => {
+			console.log(`Fetching versions for ${pkg}...`);
+			const { rows } = await sql.execute('select version from documentation where name = ?', [pkg]);
+
+			// @ts-expect-error: https://github.com/planetscale/database-js/issues/71
+			return rows.map((row) => row.version);
+		},
+		fetchPackageVersionDocs: async (pkg, version) => {
+			console.log(`Fetching data for ${pkg} ${version}...`);
+			const { rows } = await sql.execute('select data from documentation where name = ? and version = ?', [
+				pkg,
+				version,
+			]);
+
+			// @ts-expect-error: https://github.com/planetscale/database-js/issues/71
+			return rows[0].data;
+		},
+		writeToFile: false,
+	});
+	console.log('Generated all indices.');
+
+	console.log('Uploading indices...');
+	for (const index of indicies) {
+		console.log(`Uploading ${index.index}...`);
+		await client.index(index.index).addDocuments([index]);
+	}
+
+	console.log('Uploaded all indices.');
+} catch (error) {
+	const err = error as Error;
+	setFailed(err.message);
+}

packages/scripts/src/generateIndex.ts

@@ -135,9 +135,11 @@ export function visitNodes(item: ApiItem, tag: string) {
 	return members;
 }
 
-export async function generateIndex(model: ApiModel, packageName: string, tag = 'main') {
+export async function writeIndexToFileSystem(
-	const members = visitNodes(model.tryGetPackageByName(packageName)!.entryPoints[0]!, tag);
+	members: ReturnType<typeof visitNodes>,
-
+	packageName: string,
+	tag = 'main',
+) {
 	const dir = 'searchIndex';
 
 	try {
@@ -147,24 +149,44 @@ export async function generateIndex(model: ApiModel, packageName: string, tag =
 	}
 
 	await writeFile(
-		join(cwd(), 'public', dir, `${packageName}-${tag}-index.json`),
+		join(cwd(), 'public', dir, `${packageName}-${tag.replaceAll('.', '-')}-index.json`),
 		JSON.stringify(members, undefined, 2),
 	);
 }
 
-export async function generateAllIndices() {
+export async function fetchVersions(pkg: string) {
+	const response = await request(`https://docs.discordjs.dev/api/info?package=${pkg}`);
+	return response.body.json() as Promise<string[]>;
+}
+
+export async function fetchVersionDocs(pkg: string, version: string) {
+	const response = await request(`https://docs.discordjs.dev/docs/${pkg}/${version}.api.json`);
+	return response.body.json() as Promise<Record<any, any>>;
+}
+
+export async function generateAllIndices({
+	fetchPackageVersions = fetchVersions,
+	fetchPackageVersionDocs = fetchVersionDocs,
+	writeToFile = true,
+}) {
+	const indicies: Record<any, any>[] = [];
+
 	for (const pkg of PACKAGES) {
-		const response = await request(`https://docs.discordjs.dev/api/info?package=${pkg}`);
+		const versions = await fetchPackageVersions(pkg);
-		const versions = (await response.body.json()) as any;
 
 		for (const version of versions) {
 			idx = 0;
 
-			const versionRes = await request(`https://docs.discordjs.dev/docs/${pkg}/${version}.api.json`);
+			const data = await fetchPackageVersionDocs(pkg, version);
-			const data = await versionRes.body.json();
-
 			const model = addPackageToModel(new ApiModel(), data);
-			await generateIndex(model, pkg, version);
+			const members = visitNodes(model.tryGetPackageByName(pkg)!.entryPoints[0]!, version);
+			if (writeToFile) {
+				await writeIndexToFileSystem(members, pkg, version);
+			} else {
+				indicies.push({ index: `${pkg}-${version.replaceAll('.', '-')}`, data: members });
+			}
 		}
 	}
+
+	return indicies;
 }

pnpm-lock.yaml

@@ -316,7 +316,7 @@ importers:
         version: 14.0.0(react-dom@18.2.0)(react@18.2.0)
       '@testing-library/user-event':
         specifier: ^14.5.1
-        version: 14.5.1
+        version: 14.5.1(@testing-library/dom@9.3.3)
       '@types/node':
         specifier: 18.18.8
         version: 18.18.8
@@ -337,7 +337,7 @@ importers:
         version: 0.57.2
       '@vitejs/plugin-react':
         specifier: ^4.1.1
-        version: 4.1.1
+        version: 4.1.1(vite@4.5.0)
       '@vitest/coverage-v8':
         specifier: ^0.34.6
         version: 0.34.6(vitest@0.34.6)
@@ -389,9 +389,15 @@ importers:
       '@actions/glob':
         specifier: ^0.4.0
         version: 0.4.0
+      '@discordjs/scripts':
+        specifier: workspace:^
+        version: link:../scripts
       '@planetscale/database':
         specifier: ^1.11.0
         version: 1.11.0
+      meilisearch:
+        specifier: ^0.35.0
+        version: 0.35.0
       tslib:
         specifier: ^2.6.2
         version: 2.6.2
@@ -6808,13 +6814,6 @@ packages:
       react-dom: 18.2.0(react@18.2.0)
     dev: true
 
-  /@testing-library/user-event@14.5.1:
-    resolution: {integrity: sha512-UCcUKrUYGj7ClomOo2SpNVvx4/fkd/2BbIHDCle8A0ax+P3bU7yJwDBDrS6ZwdTMARWTGODX1hEsCcO+7beJjg==}
-    engines: {node: '>=12', npm: '>=6'}
-    peerDependencies:
-      '@testing-library/dom': '>=7.21.4'
-    dev: true
-
   /@testing-library/user-event@14.5.1(@testing-library/dom@9.3.3):
     resolution: {integrity: sha512-UCcUKrUYGj7ClomOo2SpNVvx4/fkd/2BbIHDCle8A0ax+P3bU7yJwDBDrS6ZwdTMARWTGODX1hEsCcO+7beJjg==}
     engines: {node: '>=12', npm: '>=6'}
@@ -8041,21 +8040,6 @@ packages:
       - supports-color
     dev: true
 
-  /@vitejs/plugin-react@4.1.1:
-    resolution: {integrity: sha512-Jie2HERK+uh27e+ORXXwEP5h0Y2lS9T2PRGbfebiHGlwzDO0dEnd2aNtOR/qjBlPb1YgxwAONeblL1xqLikLag==}
-    engines: {node: ^14.18.0 || >=16.0.0}
-    peerDependencies:
-      vite: ^4.2.0
-    dependencies:
-      '@babel/core': 7.23.2
-      '@babel/plugin-transform-react-jsx-self': 7.22.5(@babel/core@7.23.2)
-      '@babel/plugin-transform-react-jsx-source': 7.22.5(@babel/core@7.23.2)
-      '@types/babel__core': 7.20.3
-      react-refresh: 0.14.0
-    transitivePeerDependencies:
-      - supports-color
-    dev: true
-
   /@vitejs/plugin-react@4.1.1(vite@4.5.0):
     resolution: {integrity: sha512-Jie2HERK+uh27e+ORXXwEP5h0Y2lS9T2PRGbfebiHGlwzDO0dEnd2aNtOR/qjBlPb1YgxwAONeblL1xqLikLag==}
     engines: {node: ^14.18.0 || >=16.0.0}
@@ -10776,7 +10760,7 @@ packages:
     dependencies:
       semver: 7.5.4
       shelljs: 0.8.5
-      typescript: 5.4.0-dev.20231106
+      typescript: 5.4.0-dev.20231107
     dev: true
 
   /dts-critic@3.3.11(typescript@5.2.2):
@@ -20470,8 +20454,8 @@ packages:
     engines: {node: '>=14.17'}
     hasBin: true
 
-  /typescript@5.4.0-dev.20231106:
+  /typescript@5.4.0-dev.20231107:
-    resolution: {integrity: sha512-Uxjjt7meOHOSXpQJkiObTPUXfQKqSkdcMQOV+ZAjo3U1w/D/1Qah5Pdg0x/jt/fIA6laCNWLogS+QiaT5HOohw==}
+    resolution: {integrity: sha512-4bQlfsyFjzZMo1bSLEkFJEcvuQ64pgE0S/BpdWu16lYsQOfJZtntZIyWdED3h+e96IMtVgrC6sxzfPZhj4J/dw==}
     engines: {node: '>=14.17'}
     hasBin: true
     dev: true
@@ -21082,28 +21066,6 @@ packages:
       unist-util-stringify-position: 3.0.3
       vfile-message: 3.1.4
 
-  /vite-node@0.34.6(@types/node@18.18.8):
-    resolution: {integrity: sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==}
-    engines: {node: '>=v14.18.0'}
-    hasBin: true
-    dependencies:
-      cac: 6.7.14
-      debug: 4.3.4
-      mlly: 1.4.2
-      pathe: 1.1.1
-      picocolors: 1.0.0
-      vite: 4.5.0(@types/node@18.18.8)
-    transitivePeerDependencies:
-      - '@types/node'
-      - less
-      - lightningcss
-      - sass
-      - stylus
-      - sugarss
-      - supports-color
-      - terser
-    dev: true
-
   /vite-node@0.34.6(@types/node@18.18.8)(terser@5.24.0):
     resolution: {integrity: sha512-nlBMJ9x6n7/Amaz6F3zJ97EBwR2FkzhBRxF5e+jE6LA3yi6Wtc2lyTij1OnDMIr34v5g/tVQtsVAzhT0jc5ygA==}
     engines: {node: '>=v14.18.0'}
@@ -21186,42 +21148,6 @@ packages:
       fsevents: 2.3.3
     dev: true
 
-  /vite@4.5.0(@types/node@18.18.8):
-    resolution: {integrity: sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==}
-    engines: {node: ^14.18.0 || >=16.0.0}
-    hasBin: true
-    peerDependencies:
-      '@types/node': '>= 14'
-      less: '*'
-      lightningcss: ^1.21.0
-      sass: '*'
-      stylus: '*'
-      sugarss: '*'
-      terser: ^5.4.0
-    peerDependenciesMeta:
-      '@types/node':
-        optional: true
-      less:
-        optional: true
-      lightningcss:
-        optional: true
-      sass:
-        optional: true
-      stylus:
-        optional: true
-      sugarss:
-        optional: true
-      terser:
-        optional: true
-    dependencies:
-      '@types/node': 18.18.8
-      esbuild: 0.18.20
-      postcss: 8.4.31
-      rollup: 3.29.4
-    optionalDependencies:
-      fsevents: 2.3.3
-    dev: true
-
   /vite@4.5.0(@types/node@18.18.8)(terser@5.24.0):
     resolution: {integrity: sha512-ulr8rNLA6rkyFAlVWw2q5YJ91v098AFQ2R0PRFwPzREXOUJQPtFUG0t+/ZikhaOCDqFoDhN6/v8Sq0o4araFAw==}
     engines: {node: ^14.18.0 || >=16.0.0}
@@ -21312,8 +21238,8 @@ packages:
       strip-literal: 1.3.0
       tinybench: 2.5.1
       tinypool: 0.7.0
-      vite: 4.5.0(@types/node@18.18.8)
+      vite: 4.5.0(@types/node@18.18.8)(terser@5.24.0)
-      vite-node: 0.34.6(@types/node@18.18.8)
+      vite-node: 0.34.6(@types/node@18.18.8)(terser@5.24.0)
       why-is-node-running: 2.2.2
     transitivePeerDependencies:
       - less