diff --git a/.vscode/settings.json b/.vscode/settings.json index c55efcff8..fd3abdf5f 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -5,5 +5,8 @@ "editor.codeActionsOnSave": { "source.fixAll": true, "source.organizeImports": false + }, + "files.exclude": { + "**/node_modules": true } } diff --git a/packages/discord.js/package.json b/packages/discord.js/package.json index 21d430616..df8e832de 100644 --- a/packages/discord.js/package.json +++ b/packages/discord.js/package.json @@ -56,7 +56,7 @@ "fast-deep-equal": "^3.1.3", "lodash.snakecase": "^4.1.1", "tslib": "^2.3.1", - "undici": "^4.16.0", + "undici": "^5.2.0", "ws": "^8.5.0" }, "devDependencies": { diff --git a/packages/rest/__tests__/CDN.test.ts b/packages/rest/__tests__/CDN.test.ts index 48dcb848c..b5c3d101f 100644 --- a/packages/rest/__tests__/CDN.test.ts +++ b/packages/rest/__tests__/CDN.test.ts @@ -66,6 +66,10 @@ test('guildMemberAvatar dynamic-not-animated', () => { expect(cdn.guildMemberAvatar(id, id, hash)).toBe(`${base}/guilds/${id}/users/${id}/avatars/${hash}.webp`); }); +test('guildScheduledEventCover default', () => { + expect(cdn.guildScheduledEventCover(id, hash)).toBe(`${base}/guild-events/${id}/${hash}.webp`); +}); + test('icon default', () => { expect(cdn.icon(id, hash)).toBe(`${base}/icons/${id}/${hash}.webp`); }); diff --git a/packages/rest/__tests__/REST.test.ts b/packages/rest/__tests__/REST.test.ts index d6ea30e08..18d2a46b2 100644 --- a/packages/rest/__tests__/REST.test.ts +++ b/packages/rest/__tests__/REST.test.ts @@ -1,202 +1,224 @@ import { DiscordSnowflake } from '@sapphire/snowflake'; import { Routes, Snowflake } from 'discord-api-types/v10'; -import nock from 'nock'; -import { Response } from 'node-fetch'; -import { REST, DefaultRestOptions, APIRequest } from '../src'; +import { File, FormData, MockAgent, setGlobalDispatcher } from 'undici'; +import type { Interceptable, MockInterceptor } from 'undici/types/mock-interceptor'; +import { genPath } from './util'; +import { REST } from '../src'; const newSnowflake: Snowflake = DiscordSnowflake.generate().toString(); const api = new REST().setToken('A-Very-Fake-Token'); -nock(`${DefaultRestOptions.api}/v${DefaultRestOptions.version}`) - .get('/simpleGet') - .reply(200, { test: true }) - .delete('/simpleDelete') - .reply(200, { test: true }) - .patch('/simplePatch') - .reply(200, { test: true }) - .put('/simplePut') - .reply(200, { test: true }) - .post('/simplePost') - .reply(200, { test: true }) - .get('/getQuery') - .query({ foo: 'bar', hello: 'world' }) - .reply(200, { test: true }) - .get('/getAuth') - .times(3) - .reply(200, function handler() { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - return { auth: this.req.headers.authorization?.[0] ?? null }; - }) - .get('/getReason') - .times(3) - .reply(200, function handler() { - // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition - return { reason: this.req.headers['x-audit-log-reason']?.[0] ?? null }; - }) - .post('/urlEncoded') - .reply(200, (_, body) => body) - .post('/postEcho') - .reply(200, (_, body) => body) - .post('/postFile') - .times(5) - .reply(200, (_, body) => ({ - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call - body: body - .replace(/\r\n/g, '\n') - .replace(/-+\d+-*\n?/g, '') - .trim(), - })) - .delete('/channels/339942739275677727/messages/392063687801700356') - .reply(200, { test: true }) - .delete(`/channels/339942739275677727/messages/${newSnowflake}`) - .reply(200, { test: true }) - .get('/request') - .times(2) - .reply(200, { test: true }); +// @discordjs/rest uses the `content-type` header to detect whether to parse +// the response as JSON or as an ArrayBuffer. +const responseOptions: MockInterceptor.MockResponseOptions = { + headers: { + 'content-type': 'application/json', + }, +}; + +let mockAgent: MockAgent; +let mockPool: Interceptable; + +beforeEach(() => { + mockAgent = new MockAgent(); + mockAgent.disableNetConnect(); // prevent actual requests to Discord + setGlobalDispatcher(mockAgent); // enabled the mock client to intercept requests + + mockPool = mockAgent.get('https://discord.com'); +}); + +afterEach(async () => { + await mockAgent.close(); +}); test('simple GET', async () => { + mockPool + .intercept({ + path: genPath('/simpleGet'), + method: 'GET', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect(await api.get('/simpleGet')).toStrictEqual({ test: true }); }); test('simple DELETE', async () => { + mockPool + .intercept({ + path: genPath('/simpleDelete'), + method: 'DELETE', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect(await api.delete('/simpleDelete')).toStrictEqual({ test: true }); }); test('simple PATCH', async () => { + mockPool + .intercept({ + path: genPath('/simplePatch'), + method: 'PATCH', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect(await api.patch('/simplePatch')).toStrictEqual({ test: true }); }); test('simple PUT', async () => { + mockPool + .intercept({ + path: genPath('/simplePut'), + method: 'PUT', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect(await api.put('/simplePut')).toStrictEqual({ test: true }); }); test('simple POST', async () => { + mockPool + .intercept({ + path: genPath('/simplePost'), + method: 'POST', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect(await api.post('/simplePost')).toStrictEqual({ test: true }); }); +test('simple PUT', async () => { + mockPool + .intercept({ + path: genPath('/simplePut'), + method: 'PUT', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + + expect(await api.put('/simplePut')).toStrictEqual({ test: true }); +}); + test('getQuery', async () => { + const query = new URLSearchParams([ + ['foo', 'bar'], + ['hello', 'world'], + ]); + + mockPool + .intercept({ + path: `${genPath('/getQuery')}?${query.toString()}`, + method: 'GET', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect( await api.get('/getQuery', { - query: new URLSearchParams([ - ['foo', 'bar'], - ['hello', 'world'], - ]), + query: query, }), ).toStrictEqual({ test: true }); }); -test('getAuth default', async () => { +test('getAuth', async () => { + mockPool + .intercept({ + path: genPath('/getAuth'), + method: 'GET', + }) + .reply((t) => ({ + data: { auth: (t.headers as unknown as Record)['Authorization'] ?? null }, + statusCode: 200, + responseOptions, + })) + .times(3); + + // default expect(await api.get('/getAuth')).toStrictEqual({ auth: 'Bot A-Very-Fake-Token' }); + + // unauthorized + expect( + await api.get('/getAuth', { + auth: false, + }), + ).toStrictEqual({ auth: null }); + + // authorized + expect( + await api.get('/getAuth', { + auth: true, + }), + ).toStrictEqual({ auth: 'Bot A-Very-Fake-Token' }); }); -test('getAuth unauthorized', async () => { - expect(await api.get('/getAuth', { auth: false })).toStrictEqual({ auth: null }); -}); +test('getReason', async () => { + mockPool + .intercept({ + path: genPath('/getReason'), + method: 'GET', + }) + .reply((t) => ({ + data: { reason: (t.headers as unknown as Record)['X-Audit-Log-Reason'] ?? null }, + statusCode: 200, + responseOptions, + })) + .times(3); -test('getAuth authorized', async () => { - expect(await api.get('/getAuth', { auth: true })).toStrictEqual({ auth: 'Bot A-Very-Fake-Token' }); -}); - -test('getReason default', async () => { + // default expect(await api.get('/getReason')).toStrictEqual({ reason: null }); -}); -test('getReason plain text', async () => { - expect(await api.get('/getReason', { reason: 'Hello' })).toStrictEqual({ reason: 'Hello' }); -}); - -test('getReason encoded', async () => { - expect(await api.get('/getReason', { reason: '😄' })).toStrictEqual({ reason: '%F0%9F%98%84' }); -}); - -test('postFile empty', async () => { - expect(await api.post('/postFile', { files: [] })).toStrictEqual({ - body: '', - }); -}); - -test('postFile file (string)', async () => { + // plain text expect( - await api.post('/postFile', { - files: [{ name: 'out.txt', data: 'Hello' }], + await api.get('/getReason', { + reason: 'Hello', }), - ).toStrictEqual({ - body: [ - 'Content-Disposition: form-data; name="files[0]"; filename="out.txt"', - 'Content-Type: text/plain', - '', - 'Hello', - ].join('\n'), - }); -}); + ).toStrictEqual({ reason: 'Hello' }); -test('postFile file and JSON', async () => { + // encoded expect( - await api.post('/postFile', { - files: [{ name: 'out.txt', data: Buffer.from('Hello') }], - body: { foo: 'bar' }, + await api.get('/getReason', { + reason: '😄', }), - ).toStrictEqual({ - body: [ - 'Content-Disposition: form-data; name="files[0]"; filename="out.txt"', - 'Content-Type: text/plain', - '', - 'Hello', - 'Content-Disposition: form-data; name="payload_json"', - '', - '{"foo":"bar"}', - ].join('\n'), - }); -}); - -test('postFile files and JSON', async () => { - expect( - await api.post('/postFile', { - files: [ - { name: 'out.txt', data: Buffer.from('Hello') }, - { name: 'out.txt', data: Buffer.from('Hi') }, - ], - body: { files: [{ id: 0, description: 'test' }] }, - }), - ).toStrictEqual({ - body: [ - 'Content-Disposition: form-data; name="files[0]"; filename="out.txt"', - 'Content-Type: text/plain', - '', - 'Hello', - 'Content-Disposition: form-data; name="files[1]"; filename="out.txt"', - 'Content-Type: text/plain', - '', - 'Hi', - 'Content-Disposition: form-data; name="payload_json"', - '', - '{"files":[{"id":0,"description":"test"}]}', - ].join('\n'), - }); -}); - -test('postFile sticker and JSON', async () => { - expect( - await api.post('/postFile', { - files: [{ key: 'file', name: 'sticker.png', data: Buffer.from('Sticker') }], - body: { foo: 'bar' }, - appendToFormData: true, - }), - ).toStrictEqual({ - body: [ - 'Content-Disposition: form-data; name="file"; filename="sticker.png"', - 'Content-Type: image/png', - '', - 'Sticker', - 'Content-Disposition: form-data; name="foo"', - '', - 'bar', - ].join('\n'), - }); + ).toStrictEqual({ reason: '%F0%9F%98%84' }); }); test('urlEncoded', async () => { + mockPool + .intercept({ + path: genPath('/urlEncoded'), + method: 'POST', + }) + .reply((t) => ({ + data: t.body!, + statusCode: 200, + })); + const body = new URLSearchParams([ ['client_id', '1234567890123545678'], ['client_secret', 'totally-valid-secret'], @@ -204,6 +226,7 @@ test('urlEncoded', async () => { ['grant_type', 'authorization_code'], ['code', 'very-invalid-code'], ]); + expect( new Uint8Array( (await api.post('/urlEncoded', { @@ -216,55 +239,156 @@ test('urlEncoded', async () => { }); test('postEcho', async () => { + mockPool + .intercept({ + path: genPath('/postEcho'), + method: 'POST', + }) + .reply((t) => ({ + data: t.body!, + statusCode: 200, + responseOptions, + })); + expect(await api.post('/postEcho', { body: { foo: 'bar' } })).toStrictEqual({ foo: 'bar' }); }); test('Old Message Delete Edge-Case: Old message', async () => { + mockPool + .intercept({ + path: genPath('/channels/339942739275677727/messages/392063687801700356'), + method: 'DELETE', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect(await api.delete(Routes.channelMessage('339942739275677727', '392063687801700356'))).toStrictEqual({ test: true, }); }); -test('Old Message Delete Edge-Case: New message', async () => { +test('Old Message Delete Edge-Case: Old message', async () => { + mockPool + .intercept({ + path: genPath(`/channels/339942739275677727/messages/${newSnowflake}`), + method: 'DELETE', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + expect(await api.delete(Routes.channelMessage('339942739275677727', newSnowflake))).toStrictEqual({ test: true }); }); -test('Request and Response Events', async () => { - const requestListener = jest.fn(); - const responseListener = jest.fn(); +test('postFile', async () => { + const mockData = { + statusCode: 200, + data: 'Hello', + }; - api.on('request', requestListener); - api.on('response', responseListener); + mockPool + .intercept({ + path: genPath('/postFileEmptyArray'), + method: 'POST', + }) + .reply(({ body }) => { + expect(body).toBeNull(); + return mockData; + }); - await api.get('/request'); + // postFile empty + await api.post('/postFileEmptyArray', { files: [] }); - expect(requestListener).toHaveBeenCalledTimes(1); - expect(responseListener).toHaveBeenCalledTimes(1); - expect(requestListener).toHaveBeenLastCalledWith<[APIRequest]>( - expect.objectContaining({ - method: 'get', - path: '/request', - route: '/request', - data: { files: undefined, body: undefined, auth: true }, - retries: 0, - }) as APIRequest, - ); - expect(responseListener).toHaveBeenLastCalledWith<[APIRequest, Response]>( - expect.objectContaining({ - method: 'get', - path: '/request', - route: '/request', - data: { files: undefined, body: undefined, auth: true }, - retries: 0, - }) as APIRequest, - expect.objectContaining({ status: 200, statusText: 'OK' }) as Response, - ); + mockPool + .intercept({ + path: genPath('/postFileStringData'), + method: 'POST', + }) + .reply(({ body }) => { + const fd = body as FormData; - api.off('request', requestListener); - api.off('response', responseListener); + expect(fd.get('files[0]')).toBeInstanceOf(File); + expect(fd.get('files[0]')).toHaveProperty('size', 5); // 'Hello' - await api.get('/request'); + return mockData; + }); - expect(requestListener).toHaveBeenCalledTimes(1); - expect(responseListener).toHaveBeenCalledTimes(1); + // postFile file (string) + await api.post('/postFileStringData', { + files: [{ name: 'out.txt', data: 'Hello' }], + }); + + mockPool + .intercept({ + path: genPath('/postFileBufferWithJson'), + method: 'POST', + }) + .reply(({ body }) => { + const fd = body as FormData; + + expect(fd.get('files[0]')).toBeInstanceOf(File); + expect(fd.get('files[0]')).toHaveProperty('size', 5); // Buffer.from('Hello') + expect(fd.get('payload_json')).toStrictEqual(JSON.stringify({ foo: 'bar' })); + + return mockData; + }); + + // postFile file and JSON + await api.post('/postFileBufferWithJson', { + files: [{ name: 'out.txt', data: Buffer.from('Hello') }], + body: { foo: 'bar' }, + }); + + mockPool + .intercept({ + path: genPath('/postFilesAndJson'), + method: 'POST', + }) + .reply(({ body }) => { + const fd = body as FormData; + + expect(fd.get('files[0]')).toBeInstanceOf(File); + expect(fd.get('files[1]')).toBeInstanceOf(File); + expect(fd.get('files[0]')).toHaveProperty('size', 5); // Buffer.from('Hello') + expect(fd.get('files[1]')).toHaveProperty('size', 2); // Buffer.from('Hi') + expect(fd.get('payload_json')).toStrictEqual(JSON.stringify({ files: [{ id: 0, description: 'test' }] })); + + return mockData; + }); + + // postFile files and JSON + await api.post('/postFilesAndJson', { + files: [ + { name: 'out.txt', data: Buffer.from('Hello') }, + { name: 'out.txt', data: Buffer.from('Hi') }, + ], + body: { files: [{ id: 0, description: 'test' }] }, + }); + + mockPool + .intercept({ + path: genPath('/postFileStickerAndJson'), + method: 'POST', + }) + .reply(({ body }) => { + const fd = body as FormData; + + expect(fd.get('file')).toBeInstanceOf(File); + expect(fd.get('file')).toHaveProperty('size', 7); // Buffer.from('Sticker') + expect(fd.get('foo')).toStrictEqual('bar'); + + return mockData; + }); + + // postFile sticker and JSON + await api.post('/postFileStickerAndJson', { + files: [{ key: 'file', name: 'sticker.png', data: Buffer.from('Sticker') }], + body: { foo: 'bar' }, + appendToFormData: true, + }); }); diff --git a/packages/rest/__tests__/RequestHandler.test.ts b/packages/rest/__tests__/RequestHandler.test.ts index 14baf68a2..98b229517 100644 --- a/packages/rest/__tests__/RequestHandler.test.ts +++ b/packages/rest/__tests__/RequestHandler.test.ts @@ -1,10 +1,38 @@ -import nock from 'nock'; -import { DefaultRestOptions, DiscordAPIError, HTTPError, RateLimitError, REST, RESTEvents } from '../src'; +import { MockAgent, setGlobalDispatcher } from 'undici'; +import type { Interceptable, MockInterceptor } from 'undici/types/mock-interceptor'; +import { genPath } from './util'; +import { DiscordAPIError, HTTPError, RateLimitError, REST, RESTEvents } from '../src'; + +let mockAgent: MockAgent; +let mockPool: Interceptable; const api = new REST({ timeout: 2000, offset: 5 }).setToken('A-Very-Fake-Token'); const invalidAuthApi = new REST({ timeout: 2000 }).setToken('Definitely-Not-A-Fake-Token'); const rateLimitErrorApi = new REST({ rejectOnRateLimit: ['/channels'] }).setToken('Obviously-Not-A-Fake-Token'); +beforeEach(() => { + mockAgent = new MockAgent(); + mockAgent.disableNetConnect(); + setGlobalDispatcher(mockAgent); + + mockPool = mockAgent.get('https://discord.com'); + api.setAgent(mockAgent); + invalidAuthApi.setAgent(mockAgent); + rateLimitErrorApi.setAgent(mockAgent); +}); + +afterEach(async () => { + await mockAgent.close(); +}); + +// @discordjs/rest uses the `content-type` header to detect whether to parse +// the response as JSON or as an ArrayBuffer. +const responseOptions: MockInterceptor.MockResponseOptions = { + headers: { + 'content-type': 'application/json', + }, +}; + let resetAfter = 0; let sublimitResetAfter = 0; let retryAfter = 0; @@ -13,7 +41,10 @@ let sublimitHits = 0; let serverOutage = true; let unexpected429 = true; let unexpected429cf = true; -const sublimitIntervals = { +const sublimitIntervals: { + reset: NodeJS.Timer | null; + retry: NodeJS.Timer | null; +} = { reset: null, retry: null, }; @@ -40,187 +71,16 @@ function startSublimitIntervals() { } } -nock(`${DefaultRestOptions.api}/v${DefaultRestOptions.version}`) - .persist() - .replyDate() - .get('/standard') - .times(3) - .reply((): nock.ReplyFnResult => { - const response = Date.now() >= resetAfter ? 204 : 429; - resetAfter = Date.now() + 250; - if (response === 204) { - return [ - 204, - undefined, - { - 'x-ratelimit-limit': '1', - 'x-ratelimit-remaining': '0', - 'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(), - 'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33', - via: '1.1 google', - }, - ]; - } - return [ - 429, - { - limit: '1', - remaining: '0', - resetAfter: (resetAfter / 1000).toString(), - bucket: '80c17d2f203122d936070c88c8d10f33', - retryAfter: (resetAfter - Date.now()).toString(), - }, - { - 'x-ratelimit-limit': '1', - 'x-ratelimit-remaining': '0', - 'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(), - 'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33', - 'retry-after': (resetAfter - Date.now()).toString(), - via: '1.1 google', - }, - ]; - }) - .get('/triggerGlobal') - .reply( - (): nock.ReplyFnResult => [ - 204, - { global: true }, - { - 'x-ratelimit-global': 'true', - 'retry-after': '1', - via: '1.1 google', - }, - ], - ) - .get('/regularRequest') - .reply(204, { test: true }) - .patch('/channels/:id', (body) => ['name', 'topic'].some((key) => Reflect.has(body as Record, key))) - .reply((): nock.ReplyFnResult => { - sublimitHits += 1; - sublimitRequests += 1; - const response = 2 - sublimitHits >= 0 && 10 - sublimitRequests >= 0 ? 204 : 429; - startSublimitIntervals(); - if (response === 204) { - return [ - 204, - undefined, - { - 'x-ratelimit-limit': '10', - 'x-ratelimit-remaining': `${10 - sublimitRequests}`, - 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), - via: '1.1 google', - }, - ]; - } - return [ - 429, - { - limit: '10', - remaining: `${10 - sublimitRequests}`, - resetAfter: (sublimitResetAfter / 1000).toString(), - retryAfter: ((retryAfter - Date.now()) / 1000).toString(), - }, - { - 'x-ratelimit-limit': '10', - 'x-ratelimit-remaining': `${10 - sublimitRequests}`, - 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), - 'retry-after': ((retryAfter - Date.now()) / 1000).toString(), - via: '1.1 google', - }, - ]; - }) - .patch('/channels/:id', (body) => - ['name', 'topic'].every((key) => !Reflect.has(body as Record, key)), - ) - .reply((): nock.ReplyFnResult => { - sublimitRequests += 1; - const response = 10 - sublimitRequests >= 0 ? 204 : 429; - startSublimitIntervals(); - if (response === 204) { - return [ - 204, - undefined, - { - 'x-ratelimit-limit': '10', - 'x-ratelimit-remaining': `${10 - sublimitRequests}`, - 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), - via: '1.1 google', - }, - ]; - } - return [ - 429, - { - limit: '10', - remaining: `${10 - sublimitRequests}`, - resetAfter: (sublimitResetAfter / 1000).toString(), - retryAfter: ((sublimitResetAfter - Date.now()) / 1000).toString(), - }, - { - 'x-ratelimit-limit': '10', - 'x-ratelimit-remaining': `${10 - sublimitRequests}`, - 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), - 'retry-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), - via: '1.1 google', - }, - ]; - }) - .get('/unexpected') - .times(3) - .reply((): nock.ReplyFnResult => { - if (unexpected429) { - unexpected429 = false; - return [ - 429, - undefined, - { - 'retry-after': '1', - via: '1.1 google', - }, - ]; - } - return [204, { test: true }]; - }) - .get('/unexpected-cf') - .times(2) - .reply((): nock.ReplyFnResult => { - if (unexpected429cf) { - unexpected429cf = false; - return [ - 429, - undefined, - { - 'retry-after': '1', - }, - ]; - } - return [204, { test: true }]; - }) - .get('/temp') - .times(2) - .reply((): nock.ReplyFnResult => { - if (serverOutage) { - serverOutage = false; - return [500]; - } - return [204, { test: true }]; - }) - .get('/outage') - .times(2) - .reply(500) - .get('/slow') - .times(2) - .delay(3000) - .reply(200) - .get('/badRequest') - .reply(403, { message: 'Missing Permissions', code: 50013 }) - .get('/unauthorized') - .reply(401, { message: '401: Unauthorized', code: 0 }) - .get('/malformedRequest') - .reply(601); - // This is tested first to ensure the count remains accurate test('Significant Invalid Requests', async () => { + mockPool + .intercept({ + path: genPath('/badRequest'), + method: 'GET', + }) + .reply(403, { message: 'Missing Permissions', code: 50013 }, responseOptions) + .times(10); + const invalidListener = jest.fn(); const invalidListener2 = jest.fn(); api.on(RESTEvents.InvalidRequestWarning, invalidListener); @@ -258,6 +118,54 @@ test('Significant Invalid Requests', async () => { }); test('Handle standard rate limits', async () => { + mockPool + .intercept({ + path: genPath('/standard'), + method: 'GET', + }) + .reply(() => { + const response = Date.now() >= resetAfter ? 204 : 429; + resetAfter = Date.now() + 250; + + if (response === 204) { + return { + statusCode: 204, + data: '', + responseOptions: { + headers: { + 'x-ratelimit-limit': '1', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(), + 'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33', + via: '1.1 google', + }, + }, + }; + } + + return { + statusCode: 429, + data: { + limit: '1', + remaining: '0', + resetAfter: (resetAfter / 1000).toString(), + bucket: '80c17d2f203122d936070c88c8d10f33', + retryAfter: (resetAfter - Date.now()).toString(), + }, + responseOptions: { + headers: { + 'x-ratelimit-limit': '1', + 'x-ratelimit-remaining': '0', + 'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(), + 'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33', + 'retry-after': (resetAfter - Date.now()).toString(), + via: '1.1 google', + }, + }, + }; + }) + .times(3); + const [a, b, c] = [api.get('/standard'), api.get('/standard'), api.get('/standard')]; const uint8 = new Uint8Array(); @@ -267,18 +175,107 @@ test('Handle standard rate limits', async () => { const previous2 = performance.now(); expect(new Uint8Array((await c) as ArrayBuffer)).toStrictEqual(uint8); const now = performance.now(); - expect(previous2).toBeGreaterThanOrEqual(previous1 + 250); - expect(now).toBeGreaterThanOrEqual(previous2 + 250); -}); - -test('Handle global rate limits', async () => { - const earlier = performance.now(); - expect(await api.get('/triggerGlobal')).toStrictEqual({ global: true }); - expect(await api.get('/regularRequest')).toStrictEqual({ test: true }); - expect(performance.now()).toBeGreaterThanOrEqual(earlier + 100); + expect(previous2).toBeGreaterThanOrEqual(previous1 + 200); + expect(now).toBeGreaterThanOrEqual(previous2 + 200); }); test('Handle sublimits', async () => { + mockPool + .intercept({ + path: genPath('/channels/:id'), + method: 'PATCH', + }) + .reply((t) => { + const body = JSON.parse(t.body as string) as Record; + + if ('name' in body || 'topic' in body) { + sublimitHits += 1; + sublimitRequests += 1; + const response = 2 - sublimitHits >= 0 && 10 - sublimitRequests >= 0 ? 200 : 429; + startSublimitIntervals(); + + if (response === 200) { + return { + statusCode: 200, + data: '', + responseOptions: { + headers: { + 'x-ratelimit-limit': '10', + 'x-ratelimit-remaining': `${10 - sublimitRequests}`, + 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), + via: '1.1 google', + }, + }, + }; + } + + return { + statusCode: 429, + data: { + limit: '10', + remaining: `${10 - sublimitRequests}`, + resetAfter: (sublimitResetAfter / 1000).toString(), + retryAfter: ((retryAfter - Date.now()) / 1000).toString(), + }, + responseOptions: { + headers: { + 'x-ratelimit-limit': '10', + 'x-ratelimit-remaining': `${10 - sublimitRequests}`, + 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), + 'retry-after': ((retryAfter - Date.now()) / 1000).toString(), + via: '1.1 google', + ...responseOptions.headers, + }, + }, + }; + } else if (!('name' in body) && !('topic' in body)) { + sublimitRequests += 1; + const response = 10 - sublimitRequests >= 0 ? 200 : 429; + startSublimitIntervals(); + + if (response === 200) { + return { + statusCode: 200, + data: '', + responseOptions: { + headers: { + 'x-ratelimit-limit': '10', + 'x-ratelimit-remaining': `${10 - sublimitRequests}`, + 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), + via: '1.1 google', + }, + }, + }; + } + + return { + statusCode: 429, + data: { + limit: '10', + remaining: `${10 - sublimitRequests}`, + resetAfter: (sublimitResetAfter / 1000).toString(), + retryAfter: ((sublimitResetAfter - Date.now()) / 1000).toString(), + }, + responseOptions: { + headers: { + 'x-ratelimit-limit': '10', + 'x-ratelimit-remaining': `${10 - sublimitRequests}`, + 'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), + 'retry-after': ((sublimitResetAfter - Date.now()) / 1000).toString(), + via: '1.1 google', + ...responseOptions.headers, + }, + }, + }; + } + + return { + statusCode: 420, + data: 'Oh no', + }; + }) + .persist(); + // Return the current time on these results as their response does not indicate anything // Queue all requests, don't wait, to allow retroactive check const [aP, bP, cP, dP, eP] = [ @@ -297,20 +294,60 @@ test('Handle sublimits', async () => { ]); // For additional sublimited checks const e = await eP; - expect(a).toBeLessThan(b); - expect(b).toBeLessThan(c); - expect(d).toBeLessThan(c); - expect(c).toBeLessThan(e); - expect(d).toBeLessThan(e); - expect(e).toBeLessThan(f); - expect(e).toBeLessThan(g); - expect(g).toBeLessThan(f); + expect(a).toBeLessThanOrEqual(b); + expect(b).toBeLessThanOrEqual(c); + expect(d).toBeLessThanOrEqual(c); + expect(c).toBeLessThanOrEqual(e); + expect(d).toBeLessThanOrEqual(e); + expect(e).toBeLessThanOrEqual(f); + expect(e).toBeLessThanOrEqual(g); + expect(g).toBeLessThanOrEqual(f); - clearInterval(sublimitIntervals.reset); - clearInterval(sublimitIntervals.retry); + clearInterval(sublimitIntervals.reset!); + clearInterval(sublimitIntervals.retry!); + + // Reject on RateLimit + const [aP2, bP2, cP2] = [ + rateLimitErrorApi.patch('/channels/:id', sublimit), + rateLimitErrorApi.patch('/channels/:id', sublimit), + rateLimitErrorApi.patch('/channels/:id', sublimit), + ]; + await expect(aP2).resolves; + await expect(bP2).rejects.toThrowError(); + await expect(bP2).rejects.toBeInstanceOf(RateLimitError); + await expect(cP2).rejects.toThrowError(); + await expect(cP2).rejects.toBeInstanceOf(RateLimitError); }); test('Handle unexpected 429', async () => { + mockPool + .intercept({ + path: genPath('/unexpected'), + method: 'GET', + }) + .reply(() => { + if (unexpected429) { + unexpected429 = false; + return { + statusCode: 429, + data: '', + responseOptions: { + headers: { + 'retry-after': '1', + via: '1.1 google', + }, + }, + }; + } + + return { + statusCode: 200, + data: { test: true }, + responseOptions, + }; + }) + .times(3); + const previous = performance.now(); let firstResolvedTime: number; let secondResolvedTime: number; @@ -330,33 +367,133 @@ test('Handle unexpected 429', async () => { }); test('Handle unexpected 429 cloudflare', async () => { + mockPool + .intercept({ + path: genPath('/unexpected-cf'), + method: 'GET', + }) + .reply(() => { + if (unexpected429cf) { + unexpected429cf = false; + + return { + statusCode: 429, + data: '', + responseOptions: { + headers: { + 'retry-after': '1', + }, + }, + }; + } + + return { + statusCode: 200, + data: { test: true }, + responseOptions, + }; + }) + .times(2); // twice because it re-runs the request after first 429 + const previous = Date.now(); expect(await api.get('/unexpected-cf')).toStrictEqual({ test: true }); expect(Date.now()).toBeGreaterThanOrEqual(previous + 1000); }); +test('Handle global rate limits', async () => { + mockPool + .intercept({ + path: genPath('/triggerGlobal'), + method: 'GET', + }) + .reply(() => ({ + data: { global: true }, + statusCode: 200, + responseOptions, + })); + + mockPool + .intercept({ + path: genPath('/regularRequest'), + method: 'GET', + }) + .reply(() => ({ + data: { test: true }, + statusCode: 200, + responseOptions, + })); + + expect(await api.get('/triggerGlobal')).toStrictEqual({ global: true }); + expect(await api.get('/regularRequest')).toStrictEqual({ test: true }); +}); + test('Handle temp server outage', async () => { + mockPool + .intercept({ + path: genPath('/temp'), + method: 'GET', + }) + .reply(() => { + if (serverOutage) { + serverOutage = false; + + return { + statusCode: 500, + data: '', + }; + } + + return { + statusCode: 200, + data: { test: true }, + responseOptions, + }; + }) + .times(2); + expect(await api.get('/temp')).toStrictEqual({ test: true }); }); test('perm server outage', async () => { + mockPool + .intercept({ + path: genPath('/outage'), + method: 'GET', + }) + .reply(500, '', responseOptions) + .times(4); + const promise = api.get('/outage'); await expect(promise).rejects.toThrowError(); await expect(promise).rejects.toBeInstanceOf(HTTPError); }); test('server responding too slow', async () => { - const promise = api.get('/slow'); - await expect(promise).rejects.toThrowError('The user aborted a request.'); -}, 10000); + const api2 = new REST({ timeout: 1 }).setToken('A-Very-Really-Real-Token'); -test('Bad Request', async () => { - const promise = api.get('/badRequest'); - await expect(promise).rejects.toThrowError('Missing Permissions'); - await expect(promise).rejects.toBeInstanceOf(DiscordAPIError); -}); + mockPool + .intercept({ + path: genPath('/slow'), + method: 'GET', + }) + .reply(200, '') + .delay(100) + .times(10); + + const promise = api2.get('/slow'); + + await expect(promise).rejects.toThrowError('Request aborted'); +}, 1000); test('Unauthorized', async () => { + mockPool + .intercept({ + path: genPath('/unauthorized'), + method: 'GET', + }) + .reply(401, { message: '401: Unauthorized', code: 0 }, responseOptions) + .times(2); + const setTokenSpy = jest.spyOn(invalidAuthApi.requestManager, 'setToken'); // Ensure authless requests don't reset the token @@ -372,19 +509,32 @@ test('Unauthorized', async () => { expect(setTokenSpy).toHaveBeenCalledTimes(1); }); -test('Reject on RateLimit', async () => { - const [aP, bP, cP] = [ - rateLimitErrorApi.patch('/channels/:id', sublimit), - rateLimitErrorApi.patch('/channels/:id', sublimit), - rateLimitErrorApi.patch('/channels/:id', sublimit), - ]; - await expect(aP).resolves; - await expect(bP).rejects.toThrowError(); - await expect(bP).rejects.toBeInstanceOf(RateLimitError); - await expect(cP).rejects.toThrowError(); - await expect(cP).rejects.toBeInstanceOf(RateLimitError); +test('Bad Request', async () => { + mockPool + .intercept({ + path: genPath('/badRequest'), + method: 'GET', + }) + .reply(403, { message: 'Missing Permissions', code: 50013 }, responseOptions); + + const promise = api.get('/badRequest'); + await expect(promise).rejects.toThrowError('Missing Permissions'); + await expect(promise).rejects.toBeInstanceOf(DiscordAPIError); }); test('malformedRequest', async () => { - expect(await api.get('/malformedRequest')).toBe(null); + // This test doesn't really make sense because + // there is no such thing as a 601 status code. + // So, what exactly is a malformed request? + mockPool + .intercept({ + path: genPath('/malformedRequest'), + method: 'GET', + }) + .reply(() => ({ + statusCode: 405, + data: '', + })); + + await expect(api.get('/malformedRequest')).rejects.toBeInstanceOf(DiscordAPIError); }); diff --git a/packages/rest/__tests__/RequestManager.test.ts b/packages/rest/__tests__/RequestManager.test.ts index d5efa5d8d..d4e6e8bb3 100644 --- a/packages/rest/__tests__/RequestManager.test.ts +++ b/packages/rest/__tests__/RequestManager.test.ts @@ -1,11 +1,33 @@ -import nock from 'nock'; -import { DefaultRestOptions, REST } from '../src'; +import { MockAgent, setGlobalDispatcher } from 'undici'; +import { Interceptable } from 'undici/types/mock-interceptor'; +import { genPath } from './util'; +import { REST } from '../src'; const api = new REST(); -nock(`${DefaultRestOptions.api}/v${DefaultRestOptions.version}`).get('/simpleGet').reply(200, { test: true }); +let mockAgent: MockAgent; +let mockPool: Interceptable; + +beforeEach(() => { + mockAgent = new MockAgent(); + mockAgent.disableNetConnect(); + setGlobalDispatcher(mockAgent); + + mockPool = mockAgent.get('https://discord.com'); +}); + +afterEach(async () => { + await mockAgent.close(); +}); test('no token', async () => { + mockPool + .intercept({ + path: genPath('/simpleGet'), + method: 'GET', + }) + .reply(200, 'Well this is awkward...'); + const promise = api.get('/simpleGet'); await expect(promise).rejects.toThrowError('Expected token to be set for this request, but none was present'); await expect(promise).rejects.toBeInstanceOf(Error); diff --git a/packages/rest/__tests__/Util.test.ts b/packages/rest/__tests__/Util.test.ts new file mode 100644 index 000000000..3f2345618 --- /dev/null +++ b/packages/rest/__tests__/Util.test.ts @@ -0,0 +1,66 @@ +import { Blob } from 'node:buffer'; +import { resolveBody, parseHeader } from '../src/lib/utils/utils'; + +test('GIVEN string parseHeader returns string', () => { + const header = 'application/json'; + + expect(parseHeader(header)).toBe(header); +}); + +test('GIVEN string[] parseHeader returns string', () => { + const header = ['application/json', 'wait sorry I meant text/html']; + + expect(parseHeader(header)).toBe(header.join(';')); +}); + +test('GIVEN undefined parseHeader return undefined', () => { + expect(parseHeader(undefined)).toBeUndefined(); +}); + +test('resolveBody', async () => { + await expect(resolveBody(null)).resolves.toBe(null); + await expect(resolveBody(undefined)).resolves.toBe(null); + await expect(resolveBody('Hello')).resolves.toBe('Hello'); + await expect(resolveBody(new Uint8Array([1, 2, 3]))).resolves.toStrictEqual(new Uint8Array([1, 2, 3])); + // ArrayBuffers gets resolved to Uint8Array + await expect(resolveBody(new ArrayBuffer(8))).resolves.toStrictEqual(new Uint8Array(new ArrayBuffer(8))); + + const urlSearchParams = new URLSearchParams([['a', 'b']]); + await expect(resolveBody(urlSearchParams)).resolves.toBe(urlSearchParams.toString()); + + const dataView = new DataView(new ArrayBuffer(8)); + await expect(resolveBody(dataView)).resolves.toStrictEqual(new Uint8Array(new ArrayBuffer(8))); + + const blob = new Blob(['hello']); + await expect(resolveBody(blob)).resolves.toStrictEqual(new Uint8Array(await blob.arrayBuffer())); + + const iterable: Iterable = { + *[Symbol.iterator]() { + for (let i = 0; i < 3; i++) { + yield new Uint8Array([1, 2, 3]); + } + }, + }; + await expect(resolveBody(iterable)).resolves.toStrictEqual(new Uint8Array([1, 2, 3, 1, 2, 3, 1, 2, 3])); + + const asyncIterable: AsyncIterable = { + [Symbol.asyncIterator]() { + let i = 0; + return { + next() { + if (i < 3) { + i++; + return Promise.resolve({ value: new Uint8Array([1, 2, 3]), done: false }); + } + + return Promise.resolve({ value: undefined, done: true }); + }, + }; + }, + }; + await expect(resolveBody(asyncIterable)).resolves.toStrictEqual(Buffer.from([1, 2, 3, 1, 2, 3, 1, 2, 3])); + + // unknown type + // @ts-expect-error This test is ensuring that this throws + await expect(resolveBody(true)).rejects.toThrow(TypeError); +}); diff --git a/packages/rest/__tests__/util.ts b/packages/rest/__tests__/util.ts index 895f021b6..6ef1c435b 100644 --- a/packages/rest/__tests__/util.ts +++ b/packages/rest/__tests__/util.ts @@ -1,7 +1,7 @@ import { DefaultRestOptions } from '../src'; -export function genPath(path: string) { - return `/api/v${DefaultRestOptions.version}${path}`; +export function genPath(path: `/${string}`) { + return `/api/v${DefaultRestOptions.version}${path}` as const; } export function jsonHeaders(headers: Record = {}) { diff --git a/packages/rest/jest.config.js b/packages/rest/jest.config.js index 320831bd7..3308a6056 100644 --- a/packages/rest/jest.config.js +++ b/packages/rest/jest.config.js @@ -15,5 +15,5 @@ module.exports = { statements: 70, }, }, - setupFilesAfterEnv: ['./jest.setup.js'], + coveragePathIgnorePatterns: ['/node_modules/', '/__tests__/'], }; diff --git a/packages/rest/jest.setup.js b/packages/rest/jest.setup.js deleted file mode 100644 index 3b166cd60..000000000 --- a/packages/rest/jest.setup.js +++ /dev/null @@ -1,10 +0,0 @@ -// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports -const nock = require('nock'); - -beforeAll(() => { - nock.disableNetConnect(); -}); - -afterAll(() => { - nock.restore(); -}); diff --git a/packages/rest/package.json b/packages/rest/package.json index 66aa3f400..f00c8c1f1 100644 --- a/packages/rest/package.json +++ b/packages/rest/package.json @@ -53,11 +53,9 @@ "@discordjs/collection": "workspace:^", "@sapphire/async-queue": "^1.3.1", "@sapphire/snowflake": "^3.2.1", - "@types/node-fetch": "^2.6.1", "discord-api-types": "^0.29.0", - "form-data": "^4.0.0", - "node-fetch": "^2.6.7", - "tslib": "^2.3.1" + "tslib": "^2.3.1", + "undici": "^5.2.0" }, "devDependencies": { "@babel/core": "^7.17.9", @@ -75,7 +73,6 @@ "eslint-config-prettier": "^8.5.0", "eslint-plugin-import": "^2.26.0", "jest": "^27.5.1", - "nock": "^13.2.4", "prettier": "^2.6.2", "tsup": "^5.12.5", "typedoc": "^0.22.15", diff --git a/packages/rest/src/lib/REST.ts b/packages/rest/src/lib/REST.ts index ddf955eaa..2d7511277 100644 --- a/packages/rest/src/lib/REST.ts +++ b/packages/rest/src/lib/REST.ts @@ -1,7 +1,6 @@ import { EventEmitter } from 'node:events'; -import type { AgentOptions } from 'node:https'; import type Collection from '@discordjs/collection'; -import type { RequestInit, Response } from 'node-fetch'; +import type { request, Dispatcher } from 'undici'; import { CDN } from './CDN'; import { HandlerRequestData, @@ -20,10 +19,9 @@ import { DefaultRestOptions, RESTEvents } from './utils/constants'; */ export interface RESTOptions { /** - * HTTPS Agent options - * @default {} + * The agent to set globally */ - agent: Omit; + agent: Dispatcher; /** * The base api path, without version * @default 'https://discord.com/api' @@ -169,7 +167,7 @@ export interface APIRequest { /** * Additional HTTP options for this request */ - options: RequestInit; + options: RequestOptions; /** * The data that was used to form the body of this request */ @@ -195,8 +193,7 @@ export interface RestEvents { invalidRequestWarning: [invalidRequestInfo: InvalidRequestWarningData]; restDebug: [info: string]; rateLimited: [rateLimitInfo: RateLimitData]; - request: [request: APIRequest]; - response: [request: APIRequest, response: Response]; + response: [request: APIRequest, response: Dispatcher.ResponseData]; newListener: [name: string, listener: (...args: any) => void]; removeListener: [name: string, listener: (...args: any) => void]; hashSweep: [sweptHashes: Collection]; @@ -220,6 +217,8 @@ export interface REST { ((event?: Exclude) => this); } +export type RequestOptions = Exclude[1], undefined>; + export class REST extends EventEmitter { public readonly cdn: CDN; public readonly requestManager: RequestManager; @@ -234,13 +233,29 @@ export class REST extends EventEmitter { .on(RESTEvents.HashSweep, this.emit.bind(this, RESTEvents.HashSweep)); this.on('newListener', (name, listener) => { - if (name === RESTEvents.Request || name === RESTEvents.Response) this.requestManager.on(name, listener); + if (name === RESTEvents.Response) this.requestManager.on(name, listener); }); this.on('removeListener', (name, listener) => { - if (name === RESTEvents.Request || name === RESTEvents.Response) this.requestManager.off(name, listener); + if (name === RESTEvents.Response) this.requestManager.off(name, listener); }); } + /** + * Gets the agent set for this instance + */ + public getAgent() { + return this.requestManager.agent; + } + + /** + * Sets the default agent to use for requests performed by this instance + * @param agent Sets the agent to use + */ + public setAgent(agent: Dispatcher) { + this.requestManager.setAgent(agent); + return this; + } + /** * Sets the authorization token that should be used for requests * @param token The authorization token to use diff --git a/packages/rest/src/lib/RequestManager.ts b/packages/rest/src/lib/RequestManager.ts index cc27acd24..ccfc9ce8d 100644 --- a/packages/rest/src/lib/RequestManager.ts +++ b/packages/rest/src/lib/RequestManager.ts @@ -1,14 +1,13 @@ +import { Blob } from 'node:buffer'; import { EventEmitter } from 'node:events'; -import { Agent as httpAgent } from 'node:http'; -import { Agent as httpsAgent } from 'node:https'; import Collection from '@discordjs/collection'; import { DiscordSnowflake } from '@sapphire/snowflake'; -import FormData from 'form-data'; -import type { RequestInit, BodyInit } from 'node-fetch'; -import type { RESTOptions, RestEvents } from './REST'; +import { FormData, type RequestInit, type BodyInit, type Dispatcher, Agent } from 'undici'; +import type { RESTOptions, RestEvents, RequestOptions } from './REST'; import type { IHandler } from './handlers/IHandler'; import { SequentialHandler } from './handlers/SequentialHandler'; import { DefaultRestOptions, DefaultUserAgent, RESTEvents } from './utils/constants'; +import { resolveBody } from './utils/utils'; /** * Represents a file to be added to the request @@ -53,6 +52,10 @@ export interface RequestData { * If providing as BodyInit, set `passThroughBody: true` */ body?: BodyInit | unknown; + /** + * The {@link https://undici.nodejs.org/#/docs/api/Agent Agent} to use for the request. + */ + dispatcher?: Agent; /** * Files to be attached to this request */ @@ -94,11 +97,11 @@ export interface RequestHeaders { * Possible API methods to be used when doing requests */ export const enum RequestMethod { - Delete = 'delete', - Get = 'get', - Patch = 'patch', - Post = 'post', - Put = 'put', + Delete = 'DELETE', + Get = 'GET', + Patch = 'PATCH', + Post = 'POST', + Put = 'PUT', } export type RouteLike = `/${string}`; @@ -157,6 +160,11 @@ export interface RequestManager { * Represents the class that manages handlers for endpoints */ export class RequestManager extends EventEmitter { + /** + * The {@link https://undici.nodejs.org/#/docs/api/Agent Agent} for all requests + * performed by this manager. + */ + public agent: Dispatcher | null = null; /** * The number of requests remaining in the global bucket */ @@ -187,7 +195,6 @@ export class RequestManager extends EventEmitter { private hashTimer!: NodeJS.Timer; private handlerTimer!: NodeJS.Timer; - private agent: httpsAgent | httpAgent | null = null; public readonly options: RESTOptions; @@ -196,6 +203,7 @@ export class RequestManager extends EventEmitter { this.options = { ...DefaultRestOptions, ...options }; this.options.offset = Math.max(0, this.options.offset); this.globalRemaining = this.options.globalRequestsPerSecond; + this.agent = options.agent ?? null; // Start sweepers this.setupSweepers(); @@ -263,6 +271,15 @@ export class RequestManager extends EventEmitter { } } + /** + * Sets the default agent to use for requests performed by this manager + * @param agent The agent to use + */ + public setAgent(agent: Dispatcher) { + this.agent = agent; + return this; + } + /** * Sets the authorization token that should be used for requests * @param token The authorization token to use @@ -291,8 +308,8 @@ export class RequestManager extends EventEmitter { this.handlers.get(`${hash.value}:${routeId.majorParameter}`) ?? this.createHandler(hash.value, routeId.majorParameter); - // Resolve the request into usable fetch/node-fetch options - const { url, fetchOptions } = this.resolveRequest(request); + // Resolve the request into usable fetch options + const { url, fetchOptions } = await this.resolveRequest(request); // Queue the request return handler.queueRequest(routeId, url, fetchOptions, { @@ -321,13 +338,9 @@ export class RequestManager extends EventEmitter { * Formats the request data to a usable format for fetch * @param request The request data */ - private resolveRequest(request: InternalRequest): { url: string; fetchOptions: RequestInit } { + private async resolveRequest(request: InternalRequest): Promise<{ url: string; fetchOptions: RequestOptions }> { const { options } = this; - this.agent ??= options.api.startsWith('https') - ? new httpsAgent({ ...options.agent, keepAlive: true }) - : new httpAgent({ ...options.agent, keepAlive: true }); - let query = ''; // If a query option is passed, use it @@ -372,7 +385,18 @@ export class RequestManager extends EventEmitter { // Attach all files to the request for (const [index, file] of request.files.entries()) { - formData.append(file.key ?? `files[${index}]`, file.data, file.name); + const fileKey = file.key ?? `files[${index}]`; + + // https://developer.mozilla.org/en-US/docs/Web/API/FormData/append#parameters + // FormData.append only accepts a string or Blob. + // https://developer.mozilla.org/en-US/docs/Web/API/Blob/Blob#parameters + // The Blob constructor accepts TypedArray/ArrayBuffer, strings, and Blobs. + if (Buffer.isBuffer(file.data) || typeof file.data === 'string') { + formData.append(fileKey, new Blob([file.data]), file.name); + } else { + // eslint-disable-next-line @typescript-eslint/restrict-template-expressions + formData.append(fileKey, new Blob([`${file.data}`]), file.name); + } } // If a JSON body was added as well, attach it to the form data, using payload_json unless otherwise specified @@ -389,8 +413,6 @@ export class RequestManager extends EventEmitter { // Set the final body to the form data finalBody = formData; - // Set the additional headers to the form data ones - additionalHeaders = formData.getHeaders(); // eslint-disable-next-line no-eq-null } else if (request.body != null) { @@ -404,14 +426,21 @@ export class RequestManager extends EventEmitter { } } - const fetchOptions = { - agent: this.agent, - body: finalBody, + finalBody = await resolveBody(finalBody); + + const fetchOptions: RequestOptions = { // eslint-disable-next-line @typescript-eslint/consistent-type-assertions headers: { ...(request.headers ?? {}), ...additionalHeaders, ...headers } as Record, - method: request.method, + method: request.method.toUpperCase() as Dispatcher.HttpMethod, }; + if (finalBody !== undefined) { + fetchOptions.body = finalBody as Exclude; + } + + // Prioritize setting an agent per request, use the agent for this instance otherwise. + fetchOptions.dispatcher = request.dispatcher ?? this.agent ?? undefined!; + return { url, fetchOptions }; } diff --git a/packages/rest/src/lib/errors/HTTPError.ts b/packages/rest/src/lib/errors/HTTPError.ts index 71d71ee49..a7653dfa5 100644 --- a/packages/rest/src/lib/errors/HTTPError.ts +++ b/packages/rest/src/lib/errors/HTTPError.ts @@ -8,7 +8,6 @@ export class HTTPError extends Error { public requestBody: RequestBody; /** - * @param message The error message * @param name The name of the error * @param status The status code of the response * @param method The method of the request that erred @@ -16,14 +15,13 @@ export class HTTPError extends Error { * @param bodyData The unparsed data for the request that errored */ public constructor( - message: string, public override name: string, public status: number, public method: string, public url: string, bodyData: Pick, ) { - super(message); + super(); this.requestBody = { files: bodyData.files, json: bodyData.body }; } diff --git a/packages/rest/src/lib/handlers/IHandler.ts b/packages/rest/src/lib/handlers/IHandler.ts index 1a93c782c..811bf4976 100644 --- a/packages/rest/src/lib/handlers/IHandler.ts +++ b/packages/rest/src/lib/handlers/IHandler.ts @@ -1,11 +1,11 @@ -import type { RequestInit } from 'node-fetch'; +import type { RequestOptions } from '../REST'; import type { HandlerRequestData, RouteData } from '../RequestManager'; export interface IHandler { queueRequest: ( routeId: RouteData, url: string, - options: RequestInit, + options: RequestOptions, requestData: HandlerRequestData, ) => Promise; // eslint-disable-next-line @typescript-eslint/method-signature-style -- This is meant to be a getter returning a bool diff --git a/packages/rest/src/lib/handlers/SequentialHandler.ts b/packages/rest/src/lib/handlers/SequentialHandler.ts index 841f79549..c8804fb2c 100644 --- a/packages/rest/src/lib/handlers/SequentialHandler.ts +++ b/packages/rest/src/lib/handlers/SequentialHandler.ts @@ -1,14 +1,14 @@ import { setTimeout as sleep } from 'node:timers/promises'; import { AsyncQueue } from '@sapphire/async-queue'; -import fetch, { RequestInit, Response } from 'node-fetch'; +import { request, type Dispatcher } from 'undici'; import type { IHandler } from './IHandler'; -import type { RateLimitData } from '../REST'; +import type { RateLimitData, RequestOptions } from '../REST'; import type { HandlerRequestData, RequestManager, RouteData } from '../RequestManager'; import { DiscordAPIError, DiscordErrorData, OAuthErrorData } from '../errors/DiscordAPIError'; import { HTTPError } from '../errors/HTTPError'; import { RateLimitError } from '../errors/RateLimitError'; import { RESTEvents } from '../utils/constants'; -import { hasSublimit, parseResponse } from '../utils/utils'; +import { hasSublimit, parseHeader, parseResponse } from '../utils/utils'; /* Invalid request limiting is done on a per-IP basis, not a per-token basis. * The best we can do is track invalid counts process-wide (on the theory that @@ -168,7 +168,7 @@ export class SequentialHandler implements IHandler { public async queueRequest( routeId: RouteData, url: string, - options: RequestInit, + options: RequestOptions, requestData: HandlerRequestData, ): Promise { let queue = this.#asyncQueue; @@ -218,14 +218,14 @@ export class SequentialHandler implements IHandler { * The method that actually makes the request to the api, and updates info about the bucket accordingly * @param routeId The generalized api route with literal ids for major parameters * @param url The fully resolved url to make the request to - * @param options The node-fetch options needed to make the request + * @param options The fetch options needed to make the request * @param requestData Extra data from the user's request needed for errors and additional processing * @param retries The number of retries this request has already attempted (recursion) */ private async runRequest( routeId: RouteData, url: string, - options: RequestInit, + options: RequestOptions, requestData: HandlerRequestData, retries = 0, ): Promise { @@ -287,26 +287,12 @@ export class SequentialHandler implements IHandler { const method = options.method ?? 'get'; - if (this.manager.listenerCount(RESTEvents.Request)) { - this.manager.emit(RESTEvents.Request, { - method, - path: routeId.original, - route: routeId.bucketRoute, - options, - data: requestData, - retries, - }); - } - const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), this.manager.options.timeout).unref(); - let res: Response; + let res: Dispatcher.ResponseData; try { - // node-fetch typings are a bit weird, so we have to cast to any to get the correct signature - // Type 'AbortSignal' is not assignable to type 'import("discord.js-modules/node_modules/@types/node-fetch/externals").AbortSignal' - // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment - res = await fetch(url, { ...options, signal: controller.signal as any }); + res = await request(url, { ...options, signal: controller.signal }); } catch (error: unknown) { // Retry the specified number of times for possible timed out requests if (error instanceof Error && error.name === 'AbortError' && retries !== this.manager.options.retries) { @@ -329,17 +315,18 @@ export class SequentialHandler implements IHandler { data: requestData, retries, }, - res.clone(), + { ...res }, ); } + const status = res.statusCode; let retryAfter = 0; - const limit = res.headers.get('X-RateLimit-Limit'); - const remaining = res.headers.get('X-RateLimit-Remaining'); - const reset = res.headers.get('X-RateLimit-Reset-After'); - const hash = res.headers.get('X-RateLimit-Bucket'); - const retry = res.headers.get('Retry-After'); + const limit = parseHeader(res.headers['x-ratelimit-limit']); + const remaining = parseHeader(res.headers['x-ratelimit-remaining']); + const reset = parseHeader(res.headers['x-ratelimit-reset-after']); + const hash = parseHeader(res.headers['x-ratelimit-bucket']); + const retry = parseHeader(res.headers['retry-after']); // Update the total number of requests that can be made before the rate limit resets this.limit = limit ? Number(limit) : Infinity; @@ -371,7 +358,7 @@ export class SequentialHandler implements IHandler { // Handle retryAfter, which means we have actually hit a rate limit let sublimitTimeout: number | null = null; if (retryAfter > 0) { - if (res.headers.get('X-RateLimit-Global')) { + if (res.headers['x-ratelimit-global'] !== undefined) { this.manager.globalRemaining = 0; this.manager.globalReset = Date.now() + retryAfter; } else if (!this.localLimited) { @@ -385,7 +372,7 @@ export class SequentialHandler implements IHandler { } // Count the invalid requests - if (res.status === 401 || res.status === 403 || res.status === 429) { + if (status === 401 || status === 403 || status === 429) { if (!invalidCountResetTime || invalidCountResetTime < Date.now()) { invalidCountResetTime = Date.now() + 1000 * 60 * 10; invalidCount = 0; @@ -404,9 +391,9 @@ export class SequentialHandler implements IHandler { } } - if (res.ok) { + if (status === 200) { return parseResponse(res); - } else if (res.status === 429) { + } else if (status === 429) { // A rate limit was hit - this may happen if the route isn't associated with an official bucket hash yet, or when first globally rate limited const isGlobal = this.globalLimited; let limit: number; @@ -468,24 +455,24 @@ export class SequentialHandler implements IHandler { } // Since this is not a server side issue, the next request should pass, so we don't bump the retries counter return this.runRequest(routeId, url, options, requestData, retries); - } else if (res.status >= 500 && res.status < 600) { + } else if (status >= 500 && status < 600) { // Retry the specified number of times for possible server side issues if (retries !== this.manager.options.retries) { return this.runRequest(routeId, url, options, requestData, ++retries); } // We are out of retries, throw an error - throw new HTTPError(res.statusText, res.constructor.name, res.status, method, url, requestData); + throw new HTTPError(res.constructor.name, status, method, url, requestData); } else { // Handle possible malformed requests - if (res.status >= 400 && res.status < 500) { + if (status >= 400 && status < 500) { // If we receive this status code, it means the token we had is no longer valid. - if (res.status === 401 && requestData.auth) { + if (status === 401 && requestData.auth) { this.manager.setToken(null!); } // The request will not succeed for some reason, parse the error returned from the api const data = (await parseResponse(res)) as DiscordErrorData | OAuthErrorData; // throw the API error - throw new DiscordAPIError(data, 'code' in data ? data.code : data.error, res.status, method, url, requestData); + throw new DiscordAPIError(data, 'code' in data ? data.code : data.error, status, method, url, requestData); } return null; } diff --git a/packages/rest/src/lib/utils/constants.ts b/packages/rest/src/lib/utils/constants.ts index 27e334a4f..ee6f4a74c 100644 --- a/packages/rest/src/lib/utils/constants.ts +++ b/packages/rest/src/lib/utils/constants.ts @@ -1,4 +1,5 @@ import { APIVersion } from 'discord-api-types/v10'; +import { getGlobalDispatcher } from 'undici'; import type { RESTOptions } from '../REST'; // eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports, @typescript-eslint/no-unsafe-assignment const Package = require('../../../package.json'); @@ -7,7 +8,9 @@ const Package = require('../../../package.json'); export const DefaultUserAgent = `DiscordBot (${Package.homepage}, ${Package.version})`; export const DefaultRestOptions: Required = { - agent: {}, + get agent() { + return getGlobalDispatcher(); + }, api: 'https://discord.com/api', authPrefix: 'Bot', cdn: 'https://cdn.discordapp.com', @@ -32,7 +35,6 @@ export const enum RESTEvents { Debug = 'restDebug', InvalidRequestWarning = 'invalidRequestWarning', RateLimited = 'rateLimited', - Request = 'request', Response = 'response', HashSweep = 'hashSweep', HandlerSweep = 'handlerSweep', diff --git a/packages/rest/src/lib/utils/utils.ts b/packages/rest/src/lib/utils/utils.ts index c7263035c..3b3778705 100644 --- a/packages/rest/src/lib/utils/utils.ts +++ b/packages/rest/src/lib/utils/utils.ts @@ -1,7 +1,21 @@ +import { Blob } from 'node:buffer'; +import { URLSearchParams } from 'node:url'; +import { types } from 'node:util'; import type { RESTPatchAPIChannelJSONBody } from 'discord-api-types/v10'; -import type { Response } from 'node-fetch'; +import { FormData, type Dispatcher, type RequestInit } from 'undici'; +import type { RequestOptions } from '../REST'; import { RequestMethod } from '../RequestManager'; +export function parseHeader(header: string | string[] | undefined): string | undefined { + if (header === undefined) { + return header; + } else if (typeof header === 'string') { + return header; + } + + return header.join(';'); +} + function serializeSearchParam(value: unknown): string | null { switch (typeof value) { case 'string': @@ -43,14 +57,15 @@ export function makeURLSearchParams(options?: Record) { /** * Converts the response to usable data - * @param res The node-fetch response + * @param res The fetch response */ -export function parseResponse(res: Response): Promise { - if (res.headers.get('Content-Type')?.startsWith('application/json')) { - return res.json(); +export function parseResponse(res: Dispatcher.ResponseData): Promise { + const header = parseHeader(res.headers['content-type']); + if (header?.startsWith('application/json')) { + return res.body.json(); } - return res.arrayBuffer(); + return res.body.arrayBuffer(); } /** @@ -75,3 +90,48 @@ export function hasSublimit(bucketRoute: string, body?: unknown, method?: string // If we are checking if a request has a sublimit on a route not checked above, sublimit all requests to avoid a flood of 429s return true; } + +export async function resolveBody(body: RequestInit['body']): Promise { + // eslint-disable-next-line no-eq-null + if (body == null) { + return null; + } else if (typeof body === 'string') { + return body; + } else if (types.isUint8Array(body)) { + return body; + } else if (types.isArrayBuffer(body)) { + return new Uint8Array(body); + } else if (body instanceof URLSearchParams) { + return body.toString(); + } else if (body instanceof DataView) { + return new Uint8Array(body.buffer); + } else if (body instanceof Blob) { + return new Uint8Array(await body.arrayBuffer()); + } else if (body instanceof FormData) { + return body; + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if ((body as Iterable)[Symbol.iterator]) { + const chunks = [...(body as Iterable)]; + const length = chunks.reduce((a, b) => a + b.length, 0); + + const uint8 = new Uint8Array(length); + let lengthUsed = 0; + + return chunks.reduce((a, b) => { + a.set(b, lengthUsed); + lengthUsed += b.length; + return a; + }, uint8); + // eslint-disable-next-line @typescript-eslint/no-unnecessary-condition + } else if ((body as AsyncIterable)[Symbol.asyncIterator]) { + const chunks: Uint8Array[] = []; + + for await (const chunk of body as AsyncIterable) { + chunks.push(chunk); + } + + return Buffer.concat(chunks); + } + + throw new TypeError(`Unable to resolve body.`); +} diff --git a/yarn.lock b/yarn.lock index 9bffa8380..f81d20532 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1912,7 +1912,6 @@ __metadata: "@sapphire/async-queue": ^1.3.1 "@sapphire/snowflake": ^3.2.1 "@types/jest": ^27.4.1 - "@types/node-fetch": ^2.6.1 "@typescript-eslint/eslint-plugin": ^5.19.0 "@typescript-eslint/parser": ^5.19.0 babel-plugin-const-enum: ^1.2.0 @@ -1922,15 +1921,13 @@ __metadata: eslint-config-marine: ^9.4.1 eslint-config-prettier: ^8.5.0 eslint-plugin-import: ^2.26.0 - form-data: ^4.0.0 jest: ^27.5.1 - nock: ^13.2.4 - node-fetch: ^2.6.7 prettier: ^2.6.2 tslib: ^2.3.1 tsup: ^5.12.5 typedoc: ^0.22.15 typescript: ^4.6.3 + undici: ^5.2.0 languageName: unknown linkType: soft @@ -2603,16 +2600,6 @@ __metadata: languageName: node linkType: hard -"@types/node-fetch@npm:^2.6.1": - version: 2.6.1 - resolution: "@types/node-fetch@npm:2.6.1" - dependencies: - "@types/node": "*" - form-data: ^3.0.0 - checksum: a3e5d7f413d1638d795dff03f7b142b1b0e0c109ed210479000ce7b3ea11f9a6d89d9a024c96578d9249570c5fe5287a5f0f4aaba98199222230196ff2d6b283 - languageName: node - linkType: hard - "@types/node@npm:*": version: 17.0.6 resolution: "@types/node@npm:17.0.6" @@ -4544,7 +4531,7 @@ __metadata: tslib: ^2.3.1 tslint: ^6.1.3 typescript: ^4.6.3 - undici: ^4.16.0 + undici: ^5.2.0 ws: ^8.5.0 languageName: unknown linkType: soft @@ -5648,17 +5635,6 @@ dts-critic@latest: languageName: node linkType: hard -"form-data@npm:^4.0.0": - version: 4.0.0 - resolution: "form-data@npm:4.0.0" - dependencies: - asynckit: ^0.4.0 - combined-stream: ^1.0.8 - mime-types: ^2.1.12 - checksum: 01135bf8675f9d5c61ff18e2e2932f719ca4de964e3be90ef4c36aacfc7b9cb2fceb5eca0b7e0190e3383fe51c5b37f4cb80b62ca06a99aaabfcfd6ac7c9328c - languageName: node - linkType: hard - "form-data@npm:~2.3.2": version: 2.3.3 resolution: "form-data@npm:2.3.3" @@ -7740,13 +7716,6 @@ dts-critic@latest: languageName: node linkType: hard -"lodash.set@npm:^4.3.2": - version: 4.3.2 - resolution: "lodash.set@npm:4.3.2" - checksum: a9122f49eef9f2d0fc9061a33d87f8e5b8c6b23d46e8b9e9ce1529d3588d79741bd1145a3abdfa3b13082703e65af27ff18d8a07bfc22b9be32f3fc36f763f70 - languageName: node - linkType: hard - "lodash.snakecase@npm:^4.1.1": version: 4.1.1 resolution: "lodash.snakecase@npm:4.1.1" @@ -8248,18 +8217,6 @@ dts-critic@latest: languageName: node linkType: hard -"nock@npm:^13.2.4": - version: 13.2.4 - resolution: "nock@npm:13.2.4" - dependencies: - debug: ^4.1.0 - json-stringify-safe: ^5.0.1 - lodash.set: ^4.3.2 - propagate: ^2.0.0 - checksum: 2750a82ea22eebd8203eb1d7669ae09c3daae1fd573026372bad2515adad48d723a804f647bd45d7a499eb3a9a632560da406bde05bca9df762d3027db9099b5 - languageName: node - linkType: hard - "node-fetch@npm:2.6.1": version: 2.6.1 resolution: "node-fetch@npm:2.6.1" @@ -8267,20 +8224,6 @@ dts-critic@latest: languageName: node linkType: hard -"node-fetch@npm:^2.6.7": - version: 2.6.7 - resolution: "node-fetch@npm:2.6.7" - dependencies: - whatwg-url: ^5.0.0 - peerDependencies: - encoding: ^0.1.0 - peerDependenciesMeta: - encoding: - optional: true - checksum: 8d816ffd1ee22cab8301c7756ef04f3437f18dace86a1dae22cf81db8ef29c0bf6655f3215cb0cdb22b420b6fe141e64b26905e7f33f9377a7fa59135ea3e10b - languageName: node - linkType: hard - "node-gyp@npm:latest": version: 8.4.1 resolution: "node-gyp@npm:8.4.1" @@ -8928,13 +8871,6 @@ dts-critic@latest: languageName: node linkType: hard -"propagate@npm:^2.0.0": - version: 2.0.1 - resolution: "propagate@npm:2.0.1" - checksum: c4febaee2be0979e82fb6b3727878fd122a98d64a7fa3c9d09b0576751b88514a9e9275b1b92e76b364d488f508e223bd7e1dcdc616be4cdda876072fbc2a96c - languageName: node - linkType: hard - "psl@npm:^1.1.28, psl@npm:^1.1.33": version: 1.8.0 resolution: "psl@npm:1.8.0" @@ -10289,13 +10225,6 @@ dts-critic@latest: languageName: node linkType: hard -"tr46@npm:~0.0.3": - version: 0.0.3 - resolution: "tr46@npm:0.0.3" - checksum: 726321c5eaf41b5002e17ffbd1fb7245999a073e8979085dacd47c4b4e8068ff5777142fc6726d6ca1fd2ff16921b48788b87225cbc57c72636f6efa8efbffe3 - languageName: node - linkType: hard - "tree-kill@npm:^1.2.2": version: 1.2.2 resolution: "tree-kill@npm:1.2.2" @@ -10849,10 +10778,10 @@ dts-critic@latest: languageName: node linkType: hard -"undici@npm:^4.16.0": - version: 4.16.0 - resolution: "undici@npm:4.16.0" - checksum: 5e88c2b3381085e25ed1d1a308610ac7ee985f478ac705af7a8e03213536e10f73ef8dd8d85e6ed38948d1883fa0ae935e04357c317b0f5d3d3c0211d0c8c393 +"undici@npm:^5.2.0": + version: 5.2.0 + resolution: "undici@npm:5.2.0" + checksum: b7d6fe077c3ab13b7f7a0e5f4b70354b8489c5d37ca06c471754f9543c34501e3b29394b130afc8f066f11e465d6474ec665b2eb5a61052dc84baa97eeb9c7c3 languageName: node linkType: hard @@ -11054,13 +10983,6 @@ dts-critic@latest: languageName: node linkType: hard -"webidl-conversions@npm:^3.0.0": - version: 3.0.1 - resolution: "webidl-conversions@npm:3.0.1" - checksum: c92a0a6ab95314bde9c32e1d0a6dfac83b578f8fa5f21e675bc2706ed6981bc26b7eb7e6a1fab158e5ce4adf9caa4a0aee49a52505d4d13c7be545f15021b17c - languageName: node - linkType: hard - "webidl-conversions@npm:^5.0.0": version: 5.0.0 resolution: "webidl-conversions@npm:5.0.0" @@ -11091,16 +11013,6 @@ dts-critic@latest: languageName: node linkType: hard -"whatwg-url@npm:^5.0.0": - version: 5.0.0 - resolution: "whatwg-url@npm:5.0.0" - dependencies: - tr46: ~0.0.3 - webidl-conversions: ^3.0.0 - checksum: b8daed4ad3356cc4899048a15b2c143a9aed0dfae1f611ebd55073310c7b910f522ad75d727346ad64203d7e6c79ef25eafd465f4d12775ca44b90fa82ed9e2c - languageName: node - linkType: hard - "whatwg-url@npm:^8.0.0, whatwg-url@npm:^8.5.0": version: 8.7.0 resolution: "whatwg-url@npm:8.7.0"