From a4777aa9b0aafc50b3da3faad7db6b339584da5c Mon Sep 17 00:00:00 2001 From: iCrawl Date: Sat, 3 Sep 2022 00:42:34 +0200 Subject: [PATCH] feat: add naive client-based search --- packages/scripts/.gitignore | 2 +- packages/scripts/package.json | 4 +- packages/scripts/src/generateIndex.ts | 75 +++++++++++++++++-- packages/website/package.json | 1 + .../website/src/components/SidebarLayout.tsx | 1 + packages/website/src/pages/_app.tsx | 15 +++- packages/website/src/pages/docs/[...slug].tsx | 27 +++++++ packages/website/src/util/search.ts | 6 ++ yarn.lock | 10 +++ 9 files changed, 130 insertions(+), 11 deletions(-) create mode 100644 packages/website/src/util/search.ts diff --git a/packages/scripts/.gitignore b/packages/scripts/.gitignore index 47873baa9..77327af27 100644 --- a/packages/scripts/.gitignore +++ b/packages/scripts/.gitignore @@ -24,4 +24,4 @@ docs/**/* .tmp/ coverage/ -searchIndex/ \ No newline at end of file +searchIndex/ diff --git a/packages/scripts/package.json b/packages/scripts/package.json index f744da2f6..ab9b0a238 100644 --- a/packages/scripts/package.json +++ b/packages/scripts/package.json @@ -46,8 +46,10 @@ "@discordjs/api-extractor-utils": "workspace:^", "@microsoft/api-extractor-model": "7.23.3", "@microsoft/tsdoc": "^0.14.1", + "@microsoft/tsdoc-config": "0.16.1", "commander": "^9.4.0", - "tslib": "^2.4.0" + "tslib": "^2.4.0", + "undici": "^5.10.0" }, "devDependencies": { "@types/node": "^16.11.56", diff --git a/packages/scripts/src/generateIndex.ts b/packages/scripts/src/generateIndex.ts index 95b6e197f..37fd33fb5 100644 --- a/packages/scripts/src/generateIndex.ts +++ b/packages/scripts/src/generateIndex.ts @@ -1,8 +1,24 @@ -import { stat, mkdir, writeFile } from 'node:fs/promises'; +import { stat, mkdir, writeFile, readFile } from 'node:fs/promises'; import { join } from 'node:path'; +import { cwd } from 'node:process'; import { generatePath } from '@discordjs/api-extractor-utils'; -import { ApiDeclaredItem, ApiItemContainerMixin, type ApiItem, type ApiModel } from '@microsoft/api-extractor-model'; -import { DocNodeKind, type DocCodeSpan, type DocNode, type DocParagraph, type DocPlainText } from '@microsoft/tsdoc'; +import { + ApiDeclaredItem, + ApiItemContainerMixin, + ApiItem, + ApiModel, + type ApiPackage, +} from '@microsoft/api-extractor-model'; +import { + DocNodeKind, + type DocCodeSpan, + type DocNode, + type DocParagraph, + type DocPlainText, + TSDocConfiguration, +} from '@microsoft/tsdoc'; +import { TSDocConfigFile } from '@microsoft/tsdoc-config'; +// import { request } from 'undici'; export interface MemberJSON { kind: string; @@ -11,6 +27,25 @@ export interface MemberJSON { summary: string | null; } +export const PACKAGES = ['builders', 'collection', 'proxy', 'rest', 'voice', 'ws']; + +export function createApiModel(data: any) { + const model = new ApiModel(); + const tsdocConfiguration = new TSDocConfiguration(); + const tsdocConfigFile = TSDocConfigFile.loadFromObject(data.metadata.tsdocConfig); + tsdocConfigFile.configureParser(tsdocConfiguration); + + const apiPackage = ApiItem.deserialize(data, { + apiJsonFilename: '', + toolPackage: data.metadata.toolPackage, + toolVersion: data.metadata.toolVersion, + versionToDeserialize: data.metadata.schemaVersion, + tsdocConfiguration, + }) as ApiPackage; + model.addMember(apiPackage); + return model; +} + /** * Attempts to resolve the summary text for the given item. * @@ -82,14 +117,38 @@ export function visitNodes(item: ApiItem, tag: string) { return members; } -export async function generateIndex(model: ApiModel, packageName: string, tag: string) { - const members = visitNodes(model, tag); +export async function generateIndex(model: ApiModel, packageName: string, tag = 'main') { + const members = visitNodes(model.tryGetPackageByName(packageName)!.entryPoints[0]!, tag); const dir = 'searchIndex'; - if (!(await stat(dir)).isDirectory()) { - await mkdir(dir); + try { + (await stat(join(cwd(), dir))).isDirectory(); + } catch { + await mkdir(join(cwd(), dir)); } - await writeFile(join('searchIndex', `${packageName}-${tag}-doc-index.json`), JSON.stringify(members, undefined, 2)); + await writeFile( + join(cwd(), 'searchIndex', `${packageName}-${tag}-index.json`), + JSON.stringify(members, undefined, 2), + ); +} + +export async function generateAllIndicies() { + for (const pkg of PACKAGES) { + const res = await readFile(join(cwd(), '..', pkg, 'docs', 'docs.api.json'), 'utf8'); + const data = JSON.parse(res); + + // TODO: finish picking versions to generate + // const response = await request(`https://docs.discordjs.dev/api/info?package=${pkg}`); + // const versions = await response.body.json(); + + // for (const version of versions) { + // const model = createApiModel(data); + // await generateIndex(model, pkg, version); + // } + + const model = createApiModel(data); + await generateIndex(model, pkg); + } } diff --git a/packages/website/package.json b/packages/website/package.json index 649c01a6a..d1b2b9dc6 100644 --- a/packages/website/package.json +++ b/packages/website/package.json @@ -51,6 +51,7 @@ "@microsoft/tsdoc": "0.14.1", "@microsoft/tsdoc-config": "0.16.1", "@vscode/codicons": "^0.0.32", + "minisearch": "^5.0.0", "next": "^12.2.5", "next-mdx-remote": "^4.1.0", "react": "^18.2.0", diff --git a/packages/website/src/components/SidebarLayout.tsx b/packages/website/src/components/SidebarLayout.tsx index 96d755000..4449665b4 100644 --- a/packages/website/src/components/SidebarLayout.tsx +++ b/packages/website/src/components/SidebarLayout.tsx @@ -47,6 +47,7 @@ export interface SidebarLayoutProps { data: { member: ReturnType; members: ReturnType; + searchIndex: any[]; source: MDXRemoteSerializeResult; }; packageName: string; diff --git a/packages/website/src/pages/_app.tsx b/packages/website/src/pages/_app.tsx index cfee63028..0451c156d 100644 --- a/packages/website/src/pages/_app.tsx +++ b/packages/website/src/pages/_app.tsx @@ -8,6 +8,7 @@ import { VscPackage } from 'react-icons/vsc'; import { RouterTransition } from '../components/RouterTransition'; import '../styles/unocss.css'; import '../styles/main.css'; +import { miniSearch } from '~/util/search'; const actions: (router: NextRouter) => SpotlightAction[] = (router: NextRouter) => [ { @@ -111,7 +112,19 @@ export default function MyApp({ Component, pageProps }: AppProps) { withNormalizeCSS withGlobalStyles > - + { + if (!query) { + return actions; + } + + const search = miniSearch.search(query); + return actions.filter((action) => search.some((res) => res.name === action.title)); + }} + > diff --git a/packages/website/src/pages/docs/[...slug].tsx b/packages/website/src/pages/docs/[...slug].tsx index 61be19086..0622da9ae 100644 --- a/packages/website/src/pages/docs/[...slug].tsx +++ b/packages/website/src/pages/docs/[...slug].tsx @@ -14,12 +14,14 @@ import { } from '@discordjs/api-extractor-utils'; import { ActionIcon, Affix, Box, LoadingOverlay, Transition } from '@mantine/core'; import { useMediaQuery, useWindowScroll } from '@mantine/hooks'; +import { registerSpotlightActions } from '@mantine/spotlight'; import { ApiFunction, ApiItemKind, type ApiPackage } from '@microsoft/api-extractor-model'; import Head from 'next/head'; import { useRouter } from 'next/router'; import type { GetStaticPaths, GetStaticProps } from 'next/types'; import { MDXRemote } from 'next-mdx-remote'; import { serialize } from 'next-mdx-remote/serialize'; +import { useEffect } from 'react'; import { VscChevronUp } from 'react-icons/vsc'; import rehypeIgnore from 'rehype-ignore'; import rehypePrettyCode from 'rehype-pretty-code'; @@ -37,6 +39,7 @@ import { MemberProvider } from '~/contexts/member'; import { createApiModel } from '~/util/api-model.server'; import { findMember, findMemberByKey } from '~/util/model.server'; import { PACKAGES } from '~/util/packages'; +import { miniSearch } from '~/util/search'; export const getStaticPaths: GetStaticPaths = async () => { const pkgs = ( @@ -137,9 +140,16 @@ export const getStaticProps: GetStaticProps = async ({ params }) => { }); let data; + let searchIndex = []; if (process.env.NEXT_PUBLIC_LOCAL_DEV) { const res = await readFile(join(cwd(), '..', packageName, 'docs', 'docs.api.json'), 'utf8'); data = JSON.parse(res); + + const response = await readFile( + join(cwd(), '..', 'scripts', 'searchIndex', `${packageName}-main-index.json`), + 'utf8', + ); + searchIndex = JSON.parse(response); } else { const res = await fetch(`https://docs.discordjs.dev/docs/${packageName}/${branchName}.api.json`); data = await res.json(); @@ -163,6 +173,7 @@ export const getStaticProps: GetStaticProps = async ({ params }) => { member: memberName && containerKey ? findMemberByKey(model, packageName, containerKey, branchName) ?? null : null, source: mdxSource, + searchIndex, }, }, revalidate: 3_600, @@ -204,6 +215,22 @@ export default function SlugPage(props: Partial { + if (props.data?.searchIndex) { + const searchIndex = props.data?.searchIndex.map((idx, index) => ({ id: index, ...idx })) ?? []; + miniSearch.addAll(searchIndex); + + registerSpotlightActions( + searchIndex.map((idx) => ({ + title: idx.name, + description: idx.summary ?? '', + onTrigger: () => void router.push(idx.path), + })), + ); + } + // eslint-disable-next-line react-hooks/exhaustive-deps + }, []); + const name = `discord.js${props.data?.member?.name ? ` | ${props.data.member.name}` : ''}`; if (router.isFallback) { diff --git a/packages/website/src/util/search.ts b/packages/website/src/util/search.ts new file mode 100644 index 000000000..c48895b9d --- /dev/null +++ b/packages/website/src/util/search.ts @@ -0,0 +1,6 @@ +import MiniSearch from 'minisearch'; + +export const miniSearch = new MiniSearch({ + fields: ['name', 'summary'], + storeFields: ['name', 'summary', 'path'], +}); diff --git a/yarn.lock b/yarn.lock index 05a4c01c6..1fd3a170b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1913,6 +1913,7 @@ __metadata: "@discordjs/api-extractor-utils": "workspace:^" "@microsoft/api-extractor-model": 7.23.3 "@microsoft/tsdoc": ^0.14.1 + "@microsoft/tsdoc-config": 0.16.1 "@types/node": ^16.11.56 "@vitest/coverage-c8": ^0.22.1 commander: ^9.4.0 @@ -1922,6 +1923,7 @@ __metadata: tslib: ^2.4.0 tsup: ^6.2.3 typescript: ^4.8.2 + undici: ^5.10.0 vitest: ^0.22.1 languageName: unknown linkType: soft @@ -1986,6 +1988,7 @@ __metadata: eslint: ^8.23.0 eslint-config-neon: ^0.1.31 happy-dom: ^6.0.4 + minisearch: ^5.0.0 next: ^12.2.5 next-mdx-remote: ^4.1.0 prettier: ^2.7.1 @@ -12870,6 +12873,13 @@ __metadata: languageName: node linkType: hard +"minisearch@npm:^5.0.0": + version: 5.0.0 + resolution: "minisearch@npm:5.0.0" + checksum: e716c877706be6b4977b1e4fe70baa006c392857bdd7cf13f483483fcc7c8cbb34f852616449da9de41bed4b0b2e034dde80305ee3ed7c10f8c4845a813a91ec + languageName: node + linkType: hard + "minizlib@npm:^2.1.1, minizlib@npm:^2.1.2": version: 2.1.2 resolution: "minizlib@npm:2.1.2"