ci: update search with github actions

packages/actions/package.json

@@ -42,7 +42,9 @@
 	"dependencies": {
 		"@actions/core": "^1.10.1",
 		"@actions/glob": "^0.4.0",
+		"@discordjs/scripts": "workspace:^",
 		"@planetscale/database": "^1.11.0",
+		"meilisearch": "^0.35.0",
 		"tslib": "^2.6.2",
 		"undici": "5.27.2"
 	},

packages/actions/src/uploadSearchIndicies/action.yml

@@ -0,0 +1,5 @@
+name: 'Upload search indicies'
+description: 'Uploads the search indicies to a meilisearch database'
+runs:
+  using: node16
+  main: ../../dist/uploadSearchIndicies/index.js

packages/actions/src/uploadSearchIndicies/index.ts

@@ -0,0 +1,64 @@
+import process from 'node:process';
+import { setFailed } from '@actions/core';
+import { generateAllIndices } from '@discordjs/scripts';
+import { connect } from '@planetscale/database';
+import MeiliSearch from 'meilisearch';
+import { fetch } from 'undici';
+
+if (!process.env.DATABASE_URL) {
+	setFailed('DATABASE_URL is not set');
+}
+
+if (!process.env.SEARCH_API) {
+	setFailed('SEARCH_API is not set');
+}
+
+if (!process.env.SEARCH_API_KEY) {
+	setFailed('SEARCH_API_KEY is not set');
+}
+
+const sql = connect({
+	fetch,
+	url: process.env.DATABASE_URL!,
+});
+
+const client = new MeiliSearch({
+	host: process.env.SEARCH_API!,
+	apiKey: process.env.SEARCH_API_KEY!,
+});
+
+try {
+	console.log('Generating all indices...');
+	const indicies = await generateAllIndices({
+		fetchPackageVersions: async (pkg) => {
+			console.log(`Fetching versions for ${pkg}...`);
+			const { rows } = await sql.execute('select version from documentation where name = ?', [pkg]);
+
+			// @ts-expect-error: https://github.com/planetscale/database-js/issues/71
+			return rows.map((row) => row.version);
+		},
+		fetchPackageVersionDocs: async (pkg, version) => {
+			console.log(`Fetching data for ${pkg} ${version}...`);
+			const { rows } = await sql.execute('select data from documentation where name = ? and version = ?', [
+				pkg,
+				version,
+			]);
+
+			// @ts-expect-error: https://github.com/planetscale/database-js/issues/71
+			return rows[0].data;
+		},
+		writeToFile: false,
+	});
+	console.log('Generated all indices.');
+
+	console.log('Uploading indices...');
+	for (const index of indicies) {
+		console.log(`Uploading ${index.index}...`);
+		await client.index(index.index).addDocuments([index]);
+	}
+
+	console.log('Uploaded all indices.');
+} catch (error) {
+	const err = error as Error;
+	setFailed(err.message);
+}

packages/scripts/src/generateIndex.ts

@@ -135,9 +135,11 @@ export function visitNodes(item: ApiItem, tag: string) {
 	return members;
 }
 
-export async function generateIndex(model: ApiModel, packageName: string, tag = 'main') {
-	const members = visitNodes(model.tryGetPackageByName(packageName)!.entryPoints[0]!, tag);
-
+export async function writeIndexToFileSystem(
+	members: ReturnType<typeof visitNodes>,
+	packageName: string,
+	tag = 'main',
+) {
 	const dir = 'searchIndex';
 
 	try {
@@ -147,24 +149,44 @@ export async function generateIndex(model: ApiModel, packageName: string, tag =
 	}
 
 	await writeFile(
-		join(cwd(), 'public', dir, `${packageName}-${tag}-index.json`),
+		join(cwd(), 'public', dir, `${packageName}-${tag.replaceAll('.', '-')}-index.json`),
 		JSON.stringify(members, undefined, 2),
 	);
 }
 
-export async function generateAllIndices() {
+export async function fetchVersions(pkg: string) {
+	const response = await request(`https://docs.discordjs.dev/api/info?package=${pkg}`);
+	return response.body.json() as Promise<string[]>;
+}
+
+export async function fetchVersionDocs(pkg: string, version: string) {
+	const response = await request(`https://docs.discordjs.dev/docs/${pkg}/${version}.api.json`);
+	return response.body.json() as Promise<Record<any, any>>;
+}
+
+export async function generateAllIndices({
+	fetchPackageVersions = fetchVersions,
+	fetchPackageVersionDocs = fetchVersionDocs,
+	writeToFile = true,
+}) {
+	const indicies: Record<any, any>[] = [];
+
 	for (const pkg of PACKAGES) {
-		const response = await request(`https://docs.discordjs.dev/api/info?package=${pkg}`);
-		const versions = (await response.body.json()) as any;
+		const versions = await fetchPackageVersions(pkg);
 
 		for (const version of versions) {
 			idx = 0;
 
-			const versionRes = await request(`https://docs.discordjs.dev/docs/${pkg}/${version}.api.json`);
-			const data = await versionRes.body.json();
-
+			const data = await fetchPackageVersionDocs(pkg, version);
 			const model = addPackageToModel(new ApiModel(), data);
-			await generateIndex(model, pkg, version);
+			const members = visitNodes(model.tryGetPackageByName(pkg)!.entryPoints[0]!, version);
+			if (writeToFile) {
+				await writeIndexToFileSystem(members, pkg, version);
+			} else {
+				indicies.push({ index: `${pkg}-${version.replaceAll('.', '-')}`, data: members });
+			}
 		}
 	}
+
+	return indicies;
 }