Nomchy/discord.js
1
0
Fork 0
mirror of https://github.com/discordjs/discord.js.git synced 2026-03-09 16:13:31 +01:00
Code Issues Packages Projects Releases Wiki Activity

feat(rest): use undici (#7747)

Browse Source 
Co-authored-by: Vlad Frangu <kingdgrizzle@gmail.com>
Co-authored-by: ckohen <chaikohen@gmail.com>
This commit is contained in:
Khafra
2022-05-12 16:49:15 -04:00
committed by GitHub
parent 4515a1ea80
commit d1ec8c37ff
19 changed files with 964 additions and 605 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
.vscode/settings.json vendored
View File

@@ -5,5 +5,8 @@
"editor.codeActionsOnSave": {
"source.fixAll": true,
"source.organizeImports": false
},
"files.exclude": {
"**/node_modules": true
}
}

2
packages/discord.js/package.json
View File

@@ -56,7 +56,7 @@
"fast-deep-equal": "^3.1.3",
"lodash.snakecase": "^4.1.1",
"tslib": "^2.3.1",
"undici": "^4.16.0",
"undici": "^5.2.0",
"ws": "^8.5.0"
},
"devDependencies": {

4
packages/rest/__tests__/CDN.test.ts
View File

@@ -66,6 +66,10 @@ test('guildMemberAvatar dynamic-not-animated', () => {
expect(cdn.guildMemberAvatar(id, id, hash)).toBe(`${base}/guilds/${id}/users/${id}/avatars/${hash}.webp`);
});
test('guildScheduledEventCover default', () => {
expect(cdn.guildScheduledEventCover(id, hash)).toBe(`${base}/guild-events/${id}/${hash}.webp`);
});
test('icon default', () => {
expect(cdn.icon(id, hash)).toBe(`${base}/icons/${id}/${hash}.webp`);
});

490
packages/rest/__tests__/REST.test.ts
View File

@@ -1,202 +1,224 @@
import { DiscordSnowflake } from '@sapphire/snowflake';
import { Routes, Snowflake } from 'discord-api-types/v10';
import nock from 'nock';
import { Response } from 'node-fetch';
import { REST, DefaultRestOptions, APIRequest } from '../src';
import { File, FormData, MockAgent, setGlobalDispatcher } from 'undici';
import type { Interceptable, MockInterceptor } from 'undici/types/mock-interceptor';
import { genPath } from './util';
import { REST } from '../src';
const newSnowflake: Snowflake = DiscordSnowflake.generate().toString();
const api = new REST().setToken('A-Very-Fake-Token');
nock(`${DefaultRestOptions.api}/v${DefaultRestOptions.version}`)
.get('/simpleGet')
.reply(200, { test: true })
.delete('/simpleDelete')
.reply(200, { test: true })
.patch('/simplePatch')
.reply(200, { test: true })
.put('/simplePut')
.reply(200, { test: true })
.post('/simplePost')
.reply(200, { test: true })
.get('/getQuery')
.query({ foo: 'bar', hello: 'world' })
.reply(200, { test: true })
.get('/getAuth')
.times(3)
.reply(200, function handler() {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
return { auth: this.req.headers.authorization?.[0] ?? null };
})
.get('/getReason')
.times(3)
.reply(200, function handler() {
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
return { reason: this.req.headers['x-audit-log-reason']?.[0] ?? null };
})
.post('/urlEncoded')
.reply(200, (_, body) => body)
.post('/postEcho')
.reply(200, (_, body) => body)
.post('/postFile')
.times(5)
.reply(200, (_, body) => ({
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call
body: body
.replace(/\r\n/g, '\n')
.replace(/-+\d+-*\n?/g, '')
.trim(),
}))
.delete('/channels/339942739275677727/messages/392063687801700356')
.reply(200, { test: true })
.delete(`/channels/339942739275677727/messages/${newSnowflake}`)
.reply(200, { test: true })
.get('/request')
.times(2)
.reply(200, { test: true });
// @discordjs/rest uses the `content-type` header to detect whether to parse
// the response as JSON or as an ArrayBuffer.
const responseOptions: MockInterceptor.MockResponseOptions = {
headers: {
'content-type': 'application/json',
},
};
let mockAgent: MockAgent;
let mockPool: Interceptable;
beforeEach(() => {
mockAgent = new MockAgent();
mockAgent.disableNetConnect(); // prevent actual requests to Discord
setGlobalDispatcher(mockAgent); // enabled the mock client to intercept requests
mockPool = mockAgent.get('https://discord.com');
});
afterEach(async () => {
await mockAgent.close();
});
test('simple GET', async () => {
mockPool
.intercept({
path: genPath('/simpleGet'),
method: 'GET',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.get('/simpleGet')).toStrictEqual({ test: true });
});
test('simple DELETE', async () => {
mockPool
.intercept({
path: genPath('/simpleDelete'),
method: 'DELETE',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.delete('/simpleDelete')).toStrictEqual({ test: true });
});
test('simple PATCH', async () => {
mockPool
.intercept({
path: genPath('/simplePatch'),
method: 'PATCH',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.patch('/simplePatch')).toStrictEqual({ test: true });
});
test('simple PUT', async () => {
mockPool
.intercept({
path: genPath('/simplePut'),
method: 'PUT',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.put('/simplePut')).toStrictEqual({ test: true });
});
test('simple POST', async () => {
mockPool
.intercept({
path: genPath('/simplePost'),
method: 'POST',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.post('/simplePost')).toStrictEqual({ test: true });
});
test('simple PUT', async () => {
mockPool
.intercept({
path: genPath('/simplePut'),
method: 'PUT',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.put('/simplePut')).toStrictEqual({ test: true });
});
test('getQuery', async () => {
const query = new URLSearchParams([
['foo', 'bar'],
['hello', 'world'],
]);
mockPool
.intercept({
path: `${genPath('/getQuery')}?${query.toString()}`,
method: 'GET',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(
await api.get('/getQuery', {
query: new URLSearchParams([
['foo', 'bar'],
['hello', 'world'],
]),
query: query,
}),
).toStrictEqual({ test: true });
});
test('getAuth default', async () => {
test('getAuth', async () => {
mockPool
.intercept({
path: genPath('/getAuth'),
method: 'GET',
})
.reply((t) => ({
data: { auth: (t.headers as unknown as Record<string, string | undefined>)['Authorization'] ?? null },
statusCode: 200,
responseOptions,
}))
.times(3);
// default
expect(await api.get('/getAuth')).toStrictEqual({ auth: 'Bot A-Very-Fake-Token' });
// unauthorized
expect(
await api.get('/getAuth', {
auth: false,
}),
).toStrictEqual({ auth: null });
// authorized
expect(
await api.get('/getAuth', {
auth: true,
}),
).toStrictEqual({ auth: 'Bot A-Very-Fake-Token' });
});
test('getAuth unauthorized', async () => {
expect(await api.get('/getAuth', { auth: false })).toStrictEqual({ auth: null });
});
test('getReason', async () => {
mockPool
.intercept({
path: genPath('/getReason'),
method: 'GET',
})
.reply((t) => ({
data: { reason: (t.headers as unknown as Record<string, string | undefined>)['X-Audit-Log-Reason'] ?? null },
statusCode: 200,
responseOptions,
}))
.times(3);
test('getAuth authorized', async () => {
expect(await api.get('/getAuth', { auth: true })).toStrictEqual({ auth: 'Bot A-Very-Fake-Token' });
});
test('getReason default', async () => {
// default
expect(await api.get('/getReason')).toStrictEqual({ reason: null });
});
test('getReason plain text', async () => {
expect(await api.get('/getReason', { reason: 'Hello' })).toStrictEqual({ reason: 'Hello' });
});
test('getReason encoded', async () => {
expect(await api.get('/getReason', { reason: '😄' })).toStrictEqual({ reason: '%F0%9F%98%84' });
});
test('postFile empty', async () => {
expect(await api.post('/postFile', { files: [] })).toStrictEqual({
body: '',
});
});
test('postFile file (string)', async () => {
// plain text
expect(
await api.post('/postFile', {
files: [{ name: 'out.txt', data: 'Hello' }],
await api.get('/getReason', {
reason: 'Hello',
}),
).toStrictEqual({
body: [
'Content-Disposition: form-data; name="files[0]"; filename="out.txt"',
'Content-Type: text/plain',
'',
'Hello',
].join('\n'),
});
});
).toStrictEqual({ reason: 'Hello' });
test('postFile file and JSON', async () => {
// encoded
expect(
await api.post('/postFile', {
files: [{ name: 'out.txt', data: Buffer.from('Hello') }],
body: { foo: 'bar' },
await api.get('/getReason', {
reason: '😄',
}),
).toStrictEqual({
body: [
'Content-Disposition: form-data; name="files[0]"; filename="out.txt"',
'Content-Type: text/plain',
'',
'Hello',
'Content-Disposition: form-data; name="payload_json"',
'',
'{"foo":"bar"}',
].join('\n'),
});
});
test('postFile files and JSON', async () => {
expect(
await api.post('/postFile', {
files: [
{ name: 'out.txt', data: Buffer.from('Hello') },
{ name: 'out.txt', data: Buffer.from('Hi') },
],
body: { files: [{ id: 0, description: 'test' }] },
}),
).toStrictEqual({
body: [
'Content-Disposition: form-data; name="files[0]"; filename="out.txt"',
'Content-Type: text/plain',
'',
'Hello',
'Content-Disposition: form-data; name="files[1]"; filename="out.txt"',
'Content-Type: text/plain',
'',
'Hi',
'Content-Disposition: form-data; name="payload_json"',
'',
'{"files":[{"id":0,"description":"test"}]}',
].join('\n'),
});
});
test('postFile sticker and JSON', async () => {
expect(
await api.post('/postFile', {
files: [{ key: 'file', name: 'sticker.png', data: Buffer.from('Sticker') }],
body: { foo: 'bar' },
appendToFormData: true,
}),
).toStrictEqual({
body: [
'Content-Disposition: form-data; name="file"; filename="sticker.png"',
'Content-Type: image/png',
'',
'Sticker',
'Content-Disposition: form-data; name="foo"',
'',
'bar',
].join('\n'),
});
).toStrictEqual({ reason: '%F0%9F%98%84' });
});
test('urlEncoded', async () => {
mockPool
.intercept({
path: genPath('/urlEncoded'),
method: 'POST',
})
.reply((t) => ({
data: t.body!,
statusCode: 200,
}));
const body = new URLSearchParams([
['client_id', '1234567890123545678'],
['client_secret', 'totally-valid-secret'],
@@ -204,6 +226,7 @@ test('urlEncoded', async () => {
['grant_type', 'authorization_code'],
['code', 'very-invalid-code'],
]);
expect(
new Uint8Array(
(await api.post('/urlEncoded', {
@@ -216,55 +239,156 @@ test('urlEncoded', async () => {
});
test('postEcho', async () => {
mockPool
.intercept({
path: genPath('/postEcho'),
method: 'POST',
})
.reply((t) => ({
data: t.body!,
statusCode: 200,
responseOptions,
}));
expect(await api.post('/postEcho', { body: { foo: 'bar' } })).toStrictEqual({ foo: 'bar' });
});
test('Old Message Delete Edge-Case: Old message', async () => {
mockPool
.intercept({
path: genPath('/channels/339942739275677727/messages/392063687801700356'),
method: 'DELETE',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.delete(Routes.channelMessage('339942739275677727', '392063687801700356'))).toStrictEqual({
test: true,
});
});
test('Old Message Delete Edge-Case: New message', async () => {
test('Old Message Delete Edge-Case: Old message', async () => {
mockPool
.intercept({
path: genPath(`/channels/339942739275677727/messages/${newSnowflake}`),
method: 'DELETE',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.delete(Routes.channelMessage('339942739275677727', newSnowflake))).toStrictEqual({ test: true });
});
test('Request and Response Events', async () => {
const requestListener = jest.fn();
const responseListener = jest.fn();
test('postFile', async () => {
const mockData = {
statusCode: 200,
data: 'Hello',
};
api.on('request', requestListener);
api.on('response', responseListener);
mockPool
.intercept({
path: genPath('/postFileEmptyArray'),
method: 'POST',
})
.reply(({ body }) => {
expect(body).toBeNull();
return mockData;
});
await api.get('/request');
// postFile empty
await api.post('/postFileEmptyArray', { files: [] });
expect(requestListener).toHaveBeenCalledTimes(1);
expect(responseListener).toHaveBeenCalledTimes(1);
expect(requestListener).toHaveBeenLastCalledWith<[APIRequest]>(
expect.objectContaining({
method: 'get',
path: '/request',
route: '/request',
data: { files: undefined, body: undefined, auth: true },
retries: 0,
}) as APIRequest,
);
expect(responseListener).toHaveBeenLastCalledWith<[APIRequest, Response]>(
expect.objectContaining({
method: 'get',
path: '/request',
route: '/request',
data: { files: undefined, body: undefined, auth: true },
retries: 0,
}) as APIRequest,
expect.objectContaining({ status: 200, statusText: 'OK' }) as Response,
);
mockPool
.intercept({
path: genPath('/postFileStringData'),
method: 'POST',
})
.reply(({ body }) => {
const fd = body as FormData;
api.off('request', requestListener);
api.off('response', responseListener);
expect(fd.get('files[0]')).toBeInstanceOf(File);
expect(fd.get('files[0]')).toHaveProperty('size', 5); // 'Hello'
await api.get('/request');
return mockData;
});
expect(requestListener).toHaveBeenCalledTimes(1);
expect(responseListener).toHaveBeenCalledTimes(1);
// postFile file (string)
await api.post('/postFileStringData', {
files: [{ name: 'out.txt', data: 'Hello' }],
});
mockPool
.intercept({
path: genPath('/postFileBufferWithJson'),
method: 'POST',
})
.reply(({ body }) => {
const fd = body as FormData;
expect(fd.get('files[0]')).toBeInstanceOf(File);
expect(fd.get('files[0]')).toHaveProperty('size', 5); // Buffer.from('Hello')
expect(fd.get('payload_json')).toStrictEqual(JSON.stringify({ foo: 'bar' }));
return mockData;
});
// postFile file and JSON
await api.post('/postFileBufferWithJson', {
files: [{ name: 'out.txt', data: Buffer.from('Hello') }],
body: { foo: 'bar' },
});
mockPool
.intercept({
path: genPath('/postFilesAndJson'),
method: 'POST',
})
.reply(({ body }) => {
const fd = body as FormData;
expect(fd.get('files[0]')).toBeInstanceOf(File);
expect(fd.get('files[1]')).toBeInstanceOf(File);
expect(fd.get('files[0]')).toHaveProperty('size', 5); // Buffer.from('Hello')
expect(fd.get('files[1]')).toHaveProperty('size', 2); // Buffer.from('Hi')
expect(fd.get('payload_json')).toStrictEqual(JSON.stringify({ files: [{ id: 0, description: 'test' }] }));
return mockData;
});
// postFile files and JSON
await api.post('/postFilesAndJson', {
files: [
{ name: 'out.txt', data: Buffer.from('Hello') },
{ name: 'out.txt', data: Buffer.from('Hi') },
],
body: { files: [{ id: 0, description: 'test' }] },
});
mockPool
.intercept({
path: genPath('/postFileStickerAndJson'),
method: 'POST',
})
.reply(({ body }) => {
const fd = body as FormData;
expect(fd.get('file')).toBeInstanceOf(File);
expect(fd.get('file')).toHaveProperty('size', 7); // Buffer.from('Sticker')
expect(fd.get('foo')).toStrictEqual('bar');
return mockData;
});
// postFile sticker and JSON
await api.post('/postFileStickerAndJson', {
files: [{ key: 'file', name: 'sticker.png', data: Buffer.from('Sticker') }],
body: { foo: 'bar' },
appendToFormData: true,
});
});

592
packages/rest/__tests__/RequestHandler.test.ts
View File

@@ -1,10 +1,38 @@
import nock from 'nock';
import { DefaultRestOptions, DiscordAPIError, HTTPError, RateLimitError, REST, RESTEvents } from '../src';
import { MockAgent, setGlobalDispatcher } from 'undici';
import type { Interceptable, MockInterceptor } from 'undici/types/mock-interceptor';
import { genPath } from './util';
import { DiscordAPIError, HTTPError, RateLimitError, REST, RESTEvents } from '../src';
let mockAgent: MockAgent;
let mockPool: Interceptable;
const api = new REST({ timeout: 2000, offset: 5 }).setToken('A-Very-Fake-Token');
const invalidAuthApi = new REST({ timeout: 2000 }).setToken('Definitely-Not-A-Fake-Token');
const rateLimitErrorApi = new REST({ rejectOnRateLimit: ['/channels'] }).setToken('Obviously-Not-A-Fake-Token');
beforeEach(() => {
mockAgent = new MockAgent();
mockAgent.disableNetConnect();
setGlobalDispatcher(mockAgent);
mockPool = mockAgent.get('https://discord.com');
api.setAgent(mockAgent);
invalidAuthApi.setAgent(mockAgent);
rateLimitErrorApi.setAgent(mockAgent);
});
afterEach(async () => {
await mockAgent.close();
});
// @discordjs/rest uses the `content-type` header to detect whether to parse
// the response as JSON or as an ArrayBuffer.
const responseOptions: MockInterceptor.MockResponseOptions = {
headers: {
'content-type': 'application/json',
},
};
let resetAfter = 0;
let sublimitResetAfter = 0;
let retryAfter = 0;
@@ -13,7 +41,10 @@ let sublimitHits = 0;
let serverOutage = true;
let unexpected429 = true;
let unexpected429cf = true;
const sublimitIntervals = {
const sublimitIntervals: {
reset: NodeJS.Timer | null;
retry: NodeJS.Timer | null;
} = {
reset: null,
retry: null,
};
@@ -40,187 +71,16 @@ function startSublimitIntervals() {
}
}
nock(`${DefaultRestOptions.api}/v${DefaultRestOptions.version}`)
.persist()
.replyDate()
.get('/standard')
.times(3)
.reply((): nock.ReplyFnResult => {
const response = Date.now() >= resetAfter ? 204 : 429;
resetAfter = Date.now() + 250;
if (response === 204) {
return [
204,
undefined,
{
'x-ratelimit-limit': '1',
'x-ratelimit-remaining': '0',
'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(),
'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33',
via: '1.1 google',
},
];
}
return [
429,
{
limit: '1',
remaining: '0',
resetAfter: (resetAfter / 1000).toString(),
bucket: '80c17d2f203122d936070c88c8d10f33',
retryAfter: (resetAfter - Date.now()).toString(),
},
{
'x-ratelimit-limit': '1',
'x-ratelimit-remaining': '0',
'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(),
'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33',
'retry-after': (resetAfter - Date.now()).toString(),
via: '1.1 google',
},
];
})
.get('/triggerGlobal')
.reply(
(): nock.ReplyFnResult => [
204,
{ global: true },
{
'x-ratelimit-global': 'true',
'retry-after': '1',
via: '1.1 google',
},
],
)
.get('/regularRequest')
.reply(204, { test: true })
.patch('/channels/:id', (body) => ['name', 'topic'].some((key) => Reflect.has(body as Record<string, unknown>, key)))
.reply((): nock.ReplyFnResult => {
sublimitHits += 1;
sublimitRequests += 1;
const response = 2 - sublimitHits >= 0 && 10 - sublimitRequests >= 0 ? 204 : 429;
startSublimitIntervals();
if (response === 204) {
return [
204,
undefined,
{
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
},
];
}
return [
429,
{
limit: '10',
remaining: `${10 - sublimitRequests}`,
resetAfter: (sublimitResetAfter / 1000).toString(),
retryAfter: ((retryAfter - Date.now()) / 1000).toString(),
},
{
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
'retry-after': ((retryAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
},
];
})
.patch('/channels/:id', (body) =>
['name', 'topic'].every((key) => !Reflect.has(body as Record<string, unknown>, key)),
)
.reply((): nock.ReplyFnResult => {
sublimitRequests += 1;
const response = 10 - sublimitRequests >= 0 ? 204 : 429;
startSublimitIntervals();
if (response === 204) {
return [
204,
undefined,
{
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
},
];
}
return [
429,
{
limit: '10',
remaining: `${10 - sublimitRequests}`,
resetAfter: (sublimitResetAfter / 1000).toString(),
retryAfter: ((sublimitResetAfter - Date.now()) / 1000).toString(),
},
{
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
'retry-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
},
];
})
.get('/unexpected')
.times(3)
.reply((): nock.ReplyFnResult => {
if (unexpected429) {
unexpected429 = false;
return [
429,
undefined,
{
'retry-after': '1',
via: '1.1 google',
},
];
}
return [204, { test: true }];
})
.get('/unexpected-cf')
.times(2)
.reply((): nock.ReplyFnResult => {
if (unexpected429cf) {
unexpected429cf = false;
return [
429,
undefined,
{
'retry-after': '1',
},
];
}
return [204, { test: true }];
})
.get('/temp')
.times(2)
.reply((): nock.ReplyFnResult => {
if (serverOutage) {
serverOutage = false;
return [500];
}
return [204, { test: true }];
})
.get('/outage')
.times(2)
.reply(500)
.get('/slow')
.times(2)
.delay(3000)
.reply(200)
.get('/badRequest')
.reply(403, { message: 'Missing Permissions', code: 50013 })
.get('/unauthorized')
.reply(401, { message: '401: Unauthorized', code: 0 })
.get('/malformedRequest')
.reply(601);
// This is tested first to ensure the count remains accurate
test('Significant Invalid Requests', async () => {
mockPool
.intercept({
path: genPath('/badRequest'),
method: 'GET',
})
.reply(403, { message: 'Missing Permissions', code: 50013 }, responseOptions)
.times(10);
const invalidListener = jest.fn();
const invalidListener2 = jest.fn();
api.on(RESTEvents.InvalidRequestWarning, invalidListener);
@@ -258,6 +118,54 @@ test('Significant Invalid Requests', async () => {
});
test('Handle standard rate limits', async () => {
mockPool
.intercept({
path: genPath('/standard'),
method: 'GET',
})
.reply(() => {
const response = Date.now() >= resetAfter ? 204 : 429;
resetAfter = Date.now() + 250;
if (response === 204) {
return {
statusCode: 204,
data: '',
responseOptions: {
headers: {
'x-ratelimit-limit': '1',
'x-ratelimit-remaining': '0',
'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(),
'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33',
via: '1.1 google',
},
},
};
}
return {
statusCode: 429,
data: {
limit: '1',
remaining: '0',
resetAfter: (resetAfter / 1000).toString(),
bucket: '80c17d2f203122d936070c88c8d10f33',
retryAfter: (resetAfter - Date.now()).toString(),
},
responseOptions: {
headers: {
'x-ratelimit-limit': '1',
'x-ratelimit-remaining': '0',
'x-ratelimit-reset-after': ((resetAfter - Date.now()) / 1000).toString(),
'x-ratelimit-bucket': '80c17d2f203122d936070c88c8d10f33',
'retry-after': (resetAfter - Date.now()).toString(),
via: '1.1 google',
},
},
};
})
.times(3);
const [a, b, c] = [api.get('/standard'), api.get('/standard'), api.get('/standard')];
const uint8 = new Uint8Array();
@@ -267,18 +175,107 @@ test('Handle standard rate limits', async () => {
const previous2 = performance.now();
expect(new Uint8Array((await c) as ArrayBuffer)).toStrictEqual(uint8);
const now = performance.now();
expect(previous2).toBeGreaterThanOrEqual(previous1 + 250);
expect(now).toBeGreaterThanOrEqual(previous2 + 250);
});
test('Handle global rate limits', async () => {
const earlier = performance.now();
expect(await api.get('/triggerGlobal')).toStrictEqual({ global: true });
expect(await api.get('/regularRequest')).toStrictEqual({ test: true });
expect(performance.now()).toBeGreaterThanOrEqual(earlier + 100);
expect(previous2).toBeGreaterThanOrEqual(previous1 + 200);
expect(now).toBeGreaterThanOrEqual(previous2 + 200);
});
test('Handle sublimits', async () => {
mockPool
.intercept({
path: genPath('/channels/:id'),
method: 'PATCH',
})
.reply((t) => {
const body = JSON.parse(t.body as string) as Record<string, unknown>;
if ('name' in body || 'topic' in body) {
sublimitHits += 1;
sublimitRequests += 1;
const response = 2 - sublimitHits >= 0 && 10 - sublimitRequests >= 0 ? 200 : 429;
startSublimitIntervals();
if (response === 200) {
return {
statusCode: 200,
data: '',
responseOptions: {
headers: {
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
},
},
};
}
return {
statusCode: 429,
data: {
limit: '10',
remaining: `${10 - sublimitRequests}`,
resetAfter: (sublimitResetAfter / 1000).toString(),
retryAfter: ((retryAfter - Date.now()) / 1000).toString(),
},
responseOptions: {
headers: {
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
'retry-after': ((retryAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
...responseOptions.headers,
},
},
};
} else if (!('name' in body) && !('topic' in body)) {
sublimitRequests += 1;
const response = 10 - sublimitRequests >= 0 ? 200 : 429;
startSublimitIntervals();
if (response === 200) {
return {
statusCode: 200,
data: '',
responseOptions: {
headers: {
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
},
},
};
}
return {
statusCode: 429,
data: {
limit: '10',
remaining: `${10 - sublimitRequests}`,
resetAfter: (sublimitResetAfter / 1000).toString(),
retryAfter: ((sublimitResetAfter - Date.now()) / 1000).toString(),
},
responseOptions: {
headers: {
'x-ratelimit-limit': '10',
'x-ratelimit-remaining': `${10 - sublimitRequests}`,
'x-ratelimit-reset-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
'retry-after': ((sublimitResetAfter - Date.now()) / 1000).toString(),
via: '1.1 google',
...responseOptions.headers,
},
},
};
}
return {
statusCode: 420,
data: 'Oh no',
};
})
.persist();
// Return the current time on these results as their response does not indicate anything
// Queue all requests, don't wait, to allow retroactive check
const [aP, bP, cP, dP, eP] = [
@@ -297,20 +294,60 @@ test('Handle sublimits', async () => {
]); // For additional sublimited checks
const e = await eP;
expect(a).toBeLessThan(b);
expect(b).toBeLessThan(c);
expect(d).toBeLessThan(c);
expect(c).toBeLessThan(e);
expect(d).toBeLessThan(e);
expect(e).toBeLessThan(f);
expect(e).toBeLessThan(g);
expect(g).toBeLessThan(f);
expect(a).toBeLessThanOrEqual(b);
expect(b).toBeLessThanOrEqual(c);
expect(d).toBeLessThanOrEqual(c);
expect(c).toBeLessThanOrEqual(e);
expect(d).toBeLessThanOrEqual(e);
expect(e).toBeLessThanOrEqual(f);
expect(e).toBeLessThanOrEqual(g);
expect(g).toBeLessThanOrEqual(f);
clearInterval(sublimitIntervals.reset);
clearInterval(sublimitIntervals.retry);
clearInterval(sublimitIntervals.reset!);
clearInterval(sublimitIntervals.retry!);
// Reject on RateLimit
const [aP2, bP2, cP2] = [
rateLimitErrorApi.patch('/channels/:id', sublimit),
rateLimitErrorApi.patch('/channels/:id', sublimit),
rateLimitErrorApi.patch('/channels/:id', sublimit),
];
await expect(aP2).resolves;
await expect(bP2).rejects.toThrowError();
await expect(bP2).rejects.toBeInstanceOf(RateLimitError);
await expect(cP2).rejects.toThrowError();
await expect(cP2).rejects.toBeInstanceOf(RateLimitError);
});
test('Handle unexpected 429', async () => {
mockPool
.intercept({
path: genPath('/unexpected'),
method: 'GET',
})
.reply(() => {
if (unexpected429) {
unexpected429 = false;
return {
statusCode: 429,
data: '',
responseOptions: {
headers: {
'retry-after': '1',
via: '1.1 google',
},
},
};
}
return {
statusCode: 200,
data: { test: true },
responseOptions,
};
})
.times(3);
const previous = performance.now();
let firstResolvedTime: number;
let secondResolvedTime: number;
@@ -330,33 +367,133 @@ test('Handle unexpected 429', async () => {
});
test('Handle unexpected 429 cloudflare', async () => {
mockPool
.intercept({
path: genPath('/unexpected-cf'),
method: 'GET',
})
.reply(() => {
if (unexpected429cf) {
unexpected429cf = false;
return {
statusCode: 429,
data: '',
responseOptions: {
headers: {
'retry-after': '1',
},
},
};
}
return {
statusCode: 200,
data: { test: true },
responseOptions,
};
})
.times(2); // twice because it re-runs the request after first 429
const previous = Date.now();
expect(await api.get('/unexpected-cf')).toStrictEqual({ test: true });
expect(Date.now()).toBeGreaterThanOrEqual(previous + 1000);
});
test('Handle global rate limits', async () => {
mockPool
.intercept({
path: genPath('/triggerGlobal'),
method: 'GET',
})
.reply(() => ({
data: { global: true },
statusCode: 200,
responseOptions,
}));
mockPool
.intercept({
path: genPath('/regularRequest'),
method: 'GET',
})
.reply(() => ({
data: { test: true },
statusCode: 200,
responseOptions,
}));
expect(await api.get('/triggerGlobal')).toStrictEqual({ global: true });
expect(await api.get('/regularRequest')).toStrictEqual({ test: true });
});
test('Handle temp server outage', async () => {
mockPool
.intercept({
path: genPath('/temp'),
method: 'GET',
})
.reply(() => {
if (serverOutage) {
serverOutage = false;
return {
statusCode: 500,
data: '',
};
}
return {
statusCode: 200,
data: { test: true },
responseOptions,
};
})
.times(2);
expect(await api.get('/temp')).toStrictEqual({ test: true });
});
test('perm server outage', async () => {
mockPool
.intercept({
path: genPath('/outage'),
method: 'GET',
})
.reply(500, '', responseOptions)
.times(4);
const promise = api.get('/outage');
await expect(promise).rejects.toThrowError();
await expect(promise).rejects.toBeInstanceOf(HTTPError);
});
test('server responding too slow', async () => {
const promise = api.get('/slow');
await expect(promise).rejects.toThrowError('The user aborted a request.');
}, 10000);
const api2 = new REST({ timeout: 1 }).setToken('A-Very-Really-Real-Token');
test('Bad Request', async () => {
const promise = api.get('/badRequest');
await expect(promise).rejects.toThrowError('Missing Permissions');
await expect(promise).rejects.toBeInstanceOf(DiscordAPIError);
});
mockPool
.intercept({
path: genPath('/slow'),
method: 'GET',
})
.reply(200, '')
.delay(100)
.times(10);
const promise = api2.get('/slow');
await expect(promise).rejects.toThrowError('Request aborted');
}, 1000);
test('Unauthorized', async () => {
mockPool
.intercept({
path: genPath('/unauthorized'),
method: 'GET',
})
.reply(401, { message: '401: Unauthorized', code: 0 }, responseOptions)
.times(2);
const setTokenSpy = jest.spyOn(invalidAuthApi.requestManager, 'setToken');
// Ensure authless requests don't reset the token
@@ -372,19 +509,32 @@ test('Unauthorized', async () => {
expect(setTokenSpy).toHaveBeenCalledTimes(1);
});
test('Reject on RateLimit', async () => {
const [aP, bP, cP] = [
rateLimitErrorApi.patch('/channels/:id', sublimit),
rateLimitErrorApi.patch('/channels/:id', sublimit),
rateLimitErrorApi.patch('/channels/:id', sublimit),
];
await expect(aP).resolves;
await expect(bP).rejects.toThrowError();
await expect(bP).rejects.toBeInstanceOf(RateLimitError);
await expect(cP).rejects.toThrowError();
await expect(cP).rejects.toBeInstanceOf(RateLimitError);
test('Bad Request', async () => {
mockPool
.intercept({
path: genPath('/badRequest'),
method: 'GET',
})
.reply(403, { message: 'Missing Permissions', code: 50013 }, responseOptions);
const promise = api.get('/badRequest');
await expect(promise).rejects.toThrowError('Missing Permissions');
await expect(promise).rejects.toBeInstanceOf(DiscordAPIError);
});
test('malformedRequest', async () => {
expect(await api.get('/malformedRequest')).toBe(null);
// This test doesn't really make sense because
// there is no such thing as a 601 status code.
// So, what exactly is a malformed request?
mockPool
.intercept({
path: genPath('/malformedRequest'),
method: 'GET',
})
.reply(() => ({
statusCode: 405,
data: '',
}));
await expect(api.get('/malformedRequest')).rejects.toBeInstanceOf(DiscordAPIError);
});

28
packages/rest/__tests__/RequestManager.test.ts
View File

@@ -1,11 +1,33 @@
import nock from 'nock';
import { DefaultRestOptions, REST } from '../src';
import { MockAgent, setGlobalDispatcher } from 'undici';
import { Interceptable } from 'undici/types/mock-interceptor';
import { genPath } from './util';
import { REST } from '../src';
const api = new REST();
nock(`${DefaultRestOptions.api}/v${DefaultRestOptions.version}`).get('/simpleGet').reply(200, { test: true });
let mockAgent: MockAgent;
let mockPool: Interceptable;
beforeEach(() => {
mockAgent = new MockAgent();
mockAgent.disableNetConnect();
setGlobalDispatcher(mockAgent);
mockPool = mockAgent.get('https://discord.com');
});
afterEach(async () => {
await mockAgent.close();
});
test('no token', async () => {
mockPool
.intercept({
path: genPath('/simpleGet'),
method: 'GET',
})
.reply(200, 'Well this is awkward...');
const promise = api.get('/simpleGet');
await expect(promise).rejects.toThrowError('Expected token to be set for this request, but none was present');
await expect(promise).rejects.toBeInstanceOf(Error);

66
packages/rest/__tests__/Util.test.ts Normal file
View File

@@ -0,0 +1,66 @@
import { Blob } from 'node:buffer';
import { resolveBody, parseHeader } from '../src/lib/utils/utils';
test('GIVEN string parseHeader returns string', () => {
const header = 'application/json';
expect(parseHeader(header)).toBe(header);
});
test('GIVEN string[] parseHeader returns string', () => {
const header = ['application/json', 'wait sorry I meant text/html'];
expect(parseHeader(header)).toBe(header.join(';'));
});
test('GIVEN undefined parseHeader return undefined', () => {
expect(parseHeader(undefined)).toBeUndefined();
});
test('resolveBody', async () => {
await expect(resolveBody(null)).resolves.toBe(null);
await expect(resolveBody(undefined)).resolves.toBe(null);
await expect(resolveBody('Hello')).resolves.toBe('Hello');
await expect(resolveBody(new Uint8Array([1, 2, 3]))).resolves.toStrictEqual(new Uint8Array([1, 2, 3]));
// ArrayBuffers gets resolved to Uint8Array
await expect(resolveBody(new ArrayBuffer(8))).resolves.toStrictEqual(new Uint8Array(new ArrayBuffer(8)));
const urlSearchParams = new URLSearchParams([['a', 'b']]);
await expect(resolveBody(urlSearchParams)).resolves.toBe(urlSearchParams.toString());
const dataView = new DataView(new ArrayBuffer(8));
await expect(resolveBody(dataView)).resolves.toStrictEqual(new Uint8Array(new ArrayBuffer(8)));
const blob = new Blob(['hello']);
await expect(resolveBody(blob)).resolves.toStrictEqual(new Uint8Array(await blob.arrayBuffer()));
const iterable: Iterable<Uint8Array> = {
*[Symbol.iterator]() {
for (let i = 0; i < 3; i++) {
yield new Uint8Array([1, 2, 3]);
}
},
};
await expect(resolveBody(iterable)).resolves.toStrictEqual(new Uint8Array([1, 2, 3, 1, 2, 3, 1, 2, 3]));
const asyncIterable: AsyncIterable<Uint8Array> = {
[Symbol.asyncIterator]() {
let i = 0;
return {
next() {
if (i < 3) {
i++;
return Promise.resolve({ value: new Uint8Array([1, 2, 3]), done: false });
}
return Promise.resolve({ value: undefined, done: true });
},
};
},
};
await expect(resolveBody(asyncIterable)).resolves.toStrictEqual(Buffer.from([1, 2, 3, 1, 2, 3, 1, 2, 3]));
// unknown type
// @ts-expect-error This test is ensuring that this throws
await expect(resolveBody(true)).rejects.toThrow(TypeError);
});

4
packages/rest/__tests__/util.ts
View File

@@ -1,7 +1,7 @@
import { DefaultRestOptions } from '../src';
export function genPath(path: string) {
return `/api/v${DefaultRestOptions.version}${path}`;
export function genPath(path: `/${string}`) {
return `/api/v${DefaultRestOptions.version}${path}` as const;
}
export function jsonHeaders(headers: Record<string, string> = {}) {

2
packages/rest/jest.config.js
View File

@@ -15,5 +15,5 @@ module.exports = {
statements: 70,
},
},
setupFilesAfterEnv: ['./jest.setup.js'],
coveragePathIgnorePatterns: ['/node_modules/', '/__tests__/'],
};

10
packages/rest/jest.setup.js
View File

@@ -1,10 +0,0 @@
// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports
const nock = require('nock');
beforeAll(() => {
nock.disableNetConnect();
});
afterAll(() => {
nock.restore();
});

7
packages/rest/package.json
View File

@@ -53,11 +53,9 @@
"@discordjs/collection": "workspace:^",
"@sapphire/async-queue": "^1.3.1",
"@sapphire/snowflake": "^3.2.1",
"@types/node-fetch": "^2.6.1",
"discord-api-types": "^0.29.0",
"form-data": "^4.0.0",
"node-fetch": "^2.6.7",
"tslib": "^2.3.1"
"tslib": "^2.3.1",
"undici": "^5.2.0"
},
"devDependencies": {
"@babel/core": "^7.17.9",
@@ -75,7 +73,6 @@
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-import": "^2.26.0",
"jest": "^27.5.1",
"nock": "^13.2.4",
"prettier": "^2.6.2",
"tsup": "^5.12.5",
"typedoc": "^0.22.15",

35
packages/rest/src/lib/REST.ts
View File

@@ -1,7 +1,6 @@
import { EventEmitter } from 'node:events';
import type { AgentOptions } from 'node:https';
import type Collection from '@discordjs/collection';
import type { RequestInit, Response } from 'node-fetch';
import type { request, Dispatcher } from 'undici';
import { CDN } from './CDN';
import {
HandlerRequestData,
@@ -20,10 +19,9 @@ import { DefaultRestOptions, RESTEvents } from './utils/constants';
*/
export interface RESTOptions {
/**
* HTTPS Agent options
* @default {}
* The agent to set globally
*/
agent: Omit<AgentOptions, 'keepAlive'>;
agent: Dispatcher;
/**
* The base api path, without version
* @default 'https://discord.com/api'
@@ -169,7 +167,7 @@ export interface APIRequest {
/**
* Additional HTTP options for this request
*/
options: RequestInit;
options: RequestOptions;
/**
* The data that was used to form the body of this request
*/
@@ -195,8 +193,7 @@ export interface RestEvents {
invalidRequestWarning: [invalidRequestInfo: InvalidRequestWarningData];
restDebug: [info: string];
rateLimited: [rateLimitInfo: RateLimitData];
request: [request: APIRequest];
response: [request: APIRequest, response: Response];
response: [request: APIRequest, response: Dispatcher.ResponseData];
newListener: [name: string, listener: (...args: any) => void];
removeListener: [name: string, listener: (...args: any) => void];
hashSweep: [sweptHashes: Collection<string, HashData>];
@@ -220,6 +217,8 @@ export interface REST {
(<S extends string | symbol>(event?: Exclude<S, keyof RestEvents>) => this);
}
export type RequestOptions = Exclude<Parameters<typeof request>[1], undefined>;
export class REST extends EventEmitter {
public readonly cdn: CDN;
public readonly requestManager: RequestManager;
@@ -234,13 +233,29 @@ export class REST extends EventEmitter {
.on(RESTEvents.HashSweep, this.emit.bind(this, RESTEvents.HashSweep));
this.on('newListener', (name, listener) => {
if (name === RESTEvents.Request || name === RESTEvents.Response) this.requestManager.on(name, listener);
if (name === RESTEvents.Response) this.requestManager.on(name, listener);
});
this.on('removeListener', (name, listener) => {
if (name === RESTEvents.Request || name === RESTEvents.Response) this.requestManager.off(name, listener);
if (name === RESTEvents.Response) this.requestManager.off(name, listener);
});
}
/**
* Gets the agent set for this instance
*/
public getAgent() {
return this.requestManager.agent;
}
/**
* Sets the default agent to use for requests performed by this instance
* @param agent Sets the agent to use
*/
public setAgent(agent: Dispatcher) {
this.requestManager.setAgent(agent);
return this;
}
/**
* Sets the authorization token that should be used for requests
* @param token The authorization token to use

79
packages/rest/src/lib/RequestManager.ts
View File

@@ -1,14 +1,13 @@
import { Blob } from 'node:buffer';
import { EventEmitter } from 'node:events';
import { Agent as httpAgent } from 'node:http';
import { Agent as httpsAgent } from 'node:https';
import Collection from '@discordjs/collection';
import { DiscordSnowflake } from '@sapphire/snowflake';
import FormData from 'form-data';
import type { RequestInit, BodyInit } from 'node-fetch';
import type { RESTOptions, RestEvents } from './REST';
import { FormData, type RequestInit, type BodyInit, type Dispatcher, Agent } from 'undici';
import type { RESTOptions, RestEvents, RequestOptions } from './REST';
import type { IHandler } from './handlers/IHandler';
import { SequentialHandler } from './handlers/SequentialHandler';
import { DefaultRestOptions, DefaultUserAgent, RESTEvents } from './utils/constants';
import { resolveBody } from './utils/utils';
/**
* Represents a file to be added to the request
@@ -53,6 +52,10 @@ export interface RequestData {
* If providing as BodyInit, set `passThroughBody: true`
*/
body?: BodyInit | unknown;
/**
* The {@link https://undici.nodejs.org/#/docs/api/Agent Agent} to use for the request.
*/
dispatcher?: Agent;
/**
* Files to be attached to this request
*/
@@ -94,11 +97,11 @@ export interface RequestHeaders {
* Possible API methods to be used when doing requests
*/
export const enum RequestMethod {
Delete = 'delete',
Get = 'get',
Patch = 'patch',
Post = 'post',
Put = 'put',
Delete = 'DELETE',
Get = 'GET',
Patch = 'PATCH',
Post = 'POST',
Put = 'PUT',
}
export type RouteLike = `/${string}`;
@@ -157,6 +160,11 @@ export interface RequestManager {
* Represents the class that manages handlers for endpoints
*/
export class RequestManager extends EventEmitter {
/**
* The {@link https://undici.nodejs.org/#/docs/api/Agent Agent} for all requests
* performed by this manager.
*/
public agent: Dispatcher | null = null;
/**
* The number of requests remaining in the global bucket
*/
@@ -187,7 +195,6 @@ export class RequestManager extends EventEmitter {
private hashTimer!: NodeJS.Timer;
private handlerTimer!: NodeJS.Timer;
private agent: httpsAgent | httpAgent | null = null;
public readonly options: RESTOptions;
@@ -196,6 +203,7 @@ export class RequestManager extends EventEmitter {
this.options = { ...DefaultRestOptions, ...options };
this.options.offset = Math.max(0, this.options.offset);
this.globalRemaining = this.options.globalRequestsPerSecond;
this.agent = options.agent ?? null;
// Start sweepers
this.setupSweepers();
@@ -263,6 +271,15 @@ export class RequestManager extends EventEmitter {
}
}
/**
* Sets the default agent to use for requests performed by this manager
* @param agent The agent to use
*/
public setAgent(agent: Dispatcher) {
this.agent = agent;
return this;
}
/**
* Sets the authorization token that should be used for requests
* @param token The authorization token to use
@@ -291,8 +308,8 @@ export class RequestManager extends EventEmitter {
this.handlers.get(`${hash.value}:${routeId.majorParameter}`) ??
this.createHandler(hash.value, routeId.majorParameter);
// Resolve the request into usable fetch/node-fetch options
const { url, fetchOptions } = this.resolveRequest(request);
// Resolve the request into usable fetch options
const { url, fetchOptions } = await this.resolveRequest(request);
// Queue the request
return handler.queueRequest(routeId, url, fetchOptions, {
@@ -321,13 +338,9 @@ export class RequestManager extends EventEmitter {
* Formats the request data to a usable format for fetch
* @param request The request data
*/
private resolveRequest(request: InternalRequest): { url: string; fetchOptions: RequestInit } {
private async resolveRequest(request: InternalRequest): Promise<{ url: string; fetchOptions: RequestOptions }> {
const { options } = this;
this.agent ??= options.api.startsWith('https')
? new httpsAgent({ ...options.agent, keepAlive: true })
: new httpAgent({ ...options.agent, keepAlive: true });
let query = '';
// If a query option is passed, use it
@@ -372,7 +385,18 @@ export class RequestManager extends EventEmitter {
// Attach all files to the request
for (const [index, file] of request.files.entries()) {
formData.append(file.key ?? `files[${index}]`, file.data, file.name);
const fileKey = file.key ?? `files[${index}]`;
// https://developer.mozilla.org/en-US/docs/Web/API/FormData/append#parameters
// FormData.append only accepts a string or Blob.
// https://developer.mozilla.org/en-US/docs/Web/API/Blob/Blob#parameters
// The Blob constructor accepts TypedArray/ArrayBuffer, strings, and Blobs.
if (Buffer.isBuffer(file.data) || typeof file.data === 'string') {
formData.append(fileKey, new Blob([file.data]), file.name);
} else {
// eslint-disable-next-line @typescript-eslint/restrict-template-expressions
formData.append(fileKey, new Blob([`${file.data}`]), file.name);
}
}
// If a JSON body was added as well, attach it to the form data, using payload_json unless otherwise specified
@@ -389,8 +413,6 @@ export class RequestManager extends EventEmitter {
// Set the final body to the form data
finalBody = formData;
// Set the additional headers to the form data ones
additionalHeaders = formData.getHeaders();
// eslint-disable-next-line no-eq-null
} else if (request.body != null) {
@@ -404,14 +426,21 @@ export class RequestManager extends EventEmitter {
}
}
const fetchOptions = {
agent: this.agent,
body: finalBody,
finalBody = await resolveBody(finalBody);
const fetchOptions: RequestOptions = {
// eslint-disable-next-line @typescript-eslint/consistent-type-assertions
headers: { ...(request.headers ?? {}), ...additionalHeaders, ...headers } as Record<string, string>,
method: request.method,
method: request.method.toUpperCase() as Dispatcher.HttpMethod,
};
if (finalBody !== undefined) {
fetchOptions.body = finalBody as Exclude<RequestOptions['body'], undefined>;
}
// Prioritize setting an agent per request, use the agent for this instance otherwise.
fetchOptions.dispatcher = request.dispatcher ?? this.agent ?? undefined!;
return { url, fetchOptions };
}

4
packages/rest/src/lib/errors/HTTPError.ts
View File

@@ -8,7 +8,6 @@ export class HTTPError extends Error {
public requestBody: RequestBody;
/**
* @param message The error message
* @param name The name of the error
* @param status The status code of the response
* @param method The method of the request that erred
@@ -16,14 +15,13 @@ export class HTTPError extends Error {
* @param bodyData The unparsed data for the request that errored
*/
public constructor(
message: string,
public override name: string,
public status: number,
public method: string,
public url: string,
bodyData: Pick<InternalRequest, 'files' | 'body'>,
) {
super(message);
super();
this.requestBody = { files: bodyData.files, json: bodyData.body };
}

4
packages/rest/src/lib/handlers/IHandler.ts
View File

@@ -1,11 +1,11 @@
import type { RequestInit } from 'node-fetch';
import type { RequestOptions } from '../REST';
import type { HandlerRequestData, RouteData } from '../RequestManager';
export interface IHandler {
queueRequest: (
routeId: RouteData,
url: string,
options: RequestInit,
options: RequestOptions,
requestData: HandlerRequestData,
) => Promise<unknown>;
// eslint-disable-next-line @typescript-eslint/method-signature-style -- This is meant to be a getter returning a bool

61
packages/rest/src/lib/handlers/SequentialHandler.ts
View File

@@ -1,14 +1,14 @@
import { setTimeout as sleep } from 'node:timers/promises';
import { AsyncQueue } from '@sapphire/async-queue';
import fetch, { RequestInit, Response } from 'node-fetch';
import { request, type Dispatcher } from 'undici';
import type { IHandler } from './IHandler';
import type { RateLimitData } from '../REST';
import type { RateLimitData, RequestOptions } from '../REST';
import type { HandlerRequestData, RequestManager, RouteData } from '../RequestManager';
import { DiscordAPIError, DiscordErrorData, OAuthErrorData } from '../errors/DiscordAPIError';
import { HTTPError } from '../errors/HTTPError';
import { RateLimitError } from '../errors/RateLimitError';
import { RESTEvents } from '../utils/constants';
import { hasSublimit, parseResponse } from '../utils/utils';
import { hasSublimit, parseHeader, parseResponse } from '../utils/utils';
/* Invalid request limiting is done on a per-IP basis, not a per-token basis.
* The best we can do is track invalid counts process-wide (on the theory that
@@ -168,7 +168,7 @@ export class SequentialHandler implements IHandler {
public async queueRequest(
routeId: RouteData,
url: string,
options: RequestInit,
options: RequestOptions,
requestData: HandlerRequestData,
): Promise<unknown> {
let queue = this.#asyncQueue;
@@ -218,14 +218,14 @@ export class SequentialHandler implements IHandler {
* The method that actually makes the request to the api, and updates info about the bucket accordingly
* @param routeId The generalized api route with literal ids for major parameters
* @param url The fully resolved url to make the request to
* @param options The node-fetch options needed to make the request
* @param options The fetch options needed to make the request
* @param requestData Extra data from the user's request needed for errors and additional processing
* @param retries The number of retries this request has already attempted (recursion)
*/
private async runRequest(
routeId: RouteData,
url: string,
options: RequestInit,
options: RequestOptions,
requestData: HandlerRequestData,
retries = 0,
): Promise<unknown> {
@@ -287,26 +287,12 @@ export class SequentialHandler implements IHandler {
const method = options.method ?? 'get';
if (this.manager.listenerCount(RESTEvents.Request)) {
this.manager.emit(RESTEvents.Request, {
method,
path: routeId.original,
route: routeId.bucketRoute,
options,
data: requestData,
retries,
});
}
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), this.manager.options.timeout).unref();
let res: Response;
let res: Dispatcher.ResponseData;
try {
// node-fetch typings are a bit weird, so we have to cast to any to get the correct signature
// Type 'AbortSignal' is not assignable to type 'import("discord.js-modules/node_modules/@types/node-fetch/externals").AbortSignal'
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
res = await fetch(url, { ...options, signal: controller.signal as any });
res = await request(url, { ...options, signal: controller.signal });
} catch (error: unknown) {
// Retry the specified number of times for possible timed out requests
if (error instanceof Error && error.name === 'AbortError' && retries !== this.manager.options.retries) {
@@ -329,17 +315,18 @@ export class SequentialHandler implements IHandler {
data: requestData,
retries,
},
res.clone(),
{ ...res },
);
}
const status = res.statusCode;
let retryAfter = 0;
const limit = res.headers.get('X-RateLimit-Limit');
const remaining = res.headers.get('X-RateLimit-Remaining');
const reset = res.headers.get('X-RateLimit-Reset-After');
const hash = res.headers.get('X-RateLimit-Bucket');
const retry = res.headers.get('Retry-After');
const limit = parseHeader(res.headers['x-ratelimit-limit']);
const remaining = parseHeader(res.headers['x-ratelimit-remaining']);
const reset = parseHeader(res.headers['x-ratelimit-reset-after']);
const hash = parseHeader(res.headers['x-ratelimit-bucket']);
const retry = parseHeader(res.headers['retry-after']);
// Update the total number of requests that can be made before the rate limit resets
this.limit = limit ? Number(limit) : Infinity;
@@ -371,7 +358,7 @@ export class SequentialHandler implements IHandler {
// Handle retryAfter, which means we have actually hit a rate limit
let sublimitTimeout: number | null = null;
if (retryAfter > 0) {
if (res.headers.get('X-RateLimit-Global')) {
if (res.headers['x-ratelimit-global'] !== undefined) {
this.manager.globalRemaining = 0;
this.manager.globalReset = Date.now() + retryAfter;
} else if (!this.localLimited) {
@@ -385,7 +372,7 @@ export class SequentialHandler implements IHandler {
}
// Count the invalid requests
if (res.status === 401 || res.status === 403 || res.status === 429) {
if (status === 401 || status === 403 || status === 429) {
if (!invalidCountResetTime || invalidCountResetTime < Date.now()) {
invalidCountResetTime = Date.now() + 1000 * 60 * 10;
invalidCount = 0;
@@ -404,9 +391,9 @@ export class SequentialHandler implements IHandler {
}
}
if (res.ok) {
if (status === 200) {
return parseResponse(res);
} else if (res.status === 429) {
} else if (status === 429) {
// A rate limit was hit - this may happen if the route isn't associated with an official bucket hash yet, or when first globally rate limited
const isGlobal = this.globalLimited;
let limit: number;
@@ -468,24 +455,24 @@ export class SequentialHandler implements IHandler {
}
// Since this is not a server side issue, the next request should pass, so we don't bump the retries counter
return this.runRequest(routeId, url, options, requestData, retries);
} else if (res.status >= 500 && res.status < 600) {
} else if (status >= 500 && status < 600) {
// Retry the specified number of times for possible server side issues
if (retries !== this.manager.options.retries) {
return this.runRequest(routeId, url, options, requestData, ++retries);
}
// We are out of retries, throw an error
throw new HTTPError(res.statusText, res.constructor.name, res.status, method, url, requestData);
throw new HTTPError(res.constructor.name, status, method, url, requestData);
} else {
// Handle possible malformed requests
if (res.status >= 400 && res.status < 500) {
if (status >= 400 && status < 500) {
// If we receive this status code, it means the token we had is no longer valid.
if (res.status === 401 && requestData.auth) {
if (status === 401 && requestData.auth) {
this.manager.setToken(null!);
}
// The request will not succeed for some reason, parse the error returned from the api
const data = (await parseResponse(res)) as DiscordErrorData | OAuthErrorData;
// throw the API error
throw new DiscordAPIError(data, 'code' in data ? data.code : data.error, res.status, method, url, requestData);
throw new DiscordAPIError(data, 'code' in data ? data.code : data.error, status, method, url, requestData);
}
return null;
}

6
packages/rest/src/lib/utils/constants.ts
View File

@@ -1,4 +1,5 @@
import { APIVersion } from 'discord-api-types/v10';
import { getGlobalDispatcher } from 'undici';
import type { RESTOptions } from '../REST';
// eslint-disable-next-line @typescript-eslint/no-var-requires, @typescript-eslint/no-require-imports, @typescript-eslint/no-unsafe-assignment
const Package = require('../../../package.json');
@@ -7,7 +8,9 @@ const Package = require('../../../package.json');
export const DefaultUserAgent = `DiscordBot (${Package.homepage}, ${Package.version})`;
export const DefaultRestOptions: Required<RESTOptions> = {
agent: {},
get agent() {
return getGlobalDispatcher();
},
api: 'https://discord.com/api',
authPrefix: 'Bot',
cdn: 'https://cdn.discordapp.com',
@@ -32,7 +35,6 @@ export const enum RESTEvents {
Debug = 'restDebug',
InvalidRequestWarning = 'invalidRequestWarning',
RateLimited = 'rateLimited',
Request = 'request',
Response = 'response',
HashSweep = 'hashSweep',
HandlerSweep = 'handlerSweep',

72
packages/rest/src/lib/utils/utils.ts
View File

@@ -1,7 +1,21 @@
import { Blob } from 'node:buffer';
import { URLSearchParams } from 'node:url';
import { types } from 'node:util';
import type { RESTPatchAPIChannelJSONBody } from 'discord-api-types/v10';
import type { Response } from 'node-fetch';
import { FormData, type Dispatcher, type RequestInit } from 'undici';
import type { RequestOptions } from '../REST';
import { RequestMethod } from '../RequestManager';
export function parseHeader(header: string | string[] | undefined): string | undefined {
if (header === undefined) {
return header;
} else if (typeof header === 'string') {
return header;
}
return header.join(';');
}
function serializeSearchParam(value: unknown): string | null {
switch (typeof value) {
case 'string':
@@ -43,14 +57,15 @@ export function makeURLSearchParams(options?: Record<string, unknown>) {
/**
* Converts the response to usable data
* @param res The node-fetch response
* @param res The fetch response
*/
export function parseResponse(res: Response): Promise<unknown> {
if (res.headers.get('Content-Type')?.startsWith('application/json')) {
return res.json();
export function parseResponse(res: Dispatcher.ResponseData): Promise<unknown> {
const header = parseHeader(res.headers['content-type']);
if (header?.startsWith('application/json')) {
return res.body.json();
}
return res.arrayBuffer();
return res.body.arrayBuffer();
}
/**
@@ -75,3 +90,48 @@ export function hasSublimit(bucketRoute: string, body?: unknown, method?: string
// If we are checking if a request has a sublimit on a route not checked above, sublimit all requests to avoid a flood of 429s
return true;
}
export async function resolveBody(body: RequestInit['body']): Promise<RequestOptions['body']> {
// eslint-disable-next-line no-eq-null
if (body == null) {
return null;
} else if (typeof body === 'string') {
return body;
} else if (types.isUint8Array(body)) {
return body;
} else if (types.isArrayBuffer(body)) {
return new Uint8Array(body);
} else if (body instanceof URLSearchParams) {
return body.toString();
} else if (body instanceof DataView) {
return new Uint8Array(body.buffer);
} else if (body instanceof Blob) {
return new Uint8Array(await body.arrayBuffer());
} else if (body instanceof FormData) {
return body;
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
} else if ((body as Iterable<Uint8Array>)[Symbol.iterator]) {
const chunks = [...(body as Iterable<Uint8Array>)];
const length = chunks.reduce((a, b) => a + b.length, 0);
const uint8 = new Uint8Array(length);
let lengthUsed = 0;
return chunks.reduce((a, b) => {
a.set(b, lengthUsed);
lengthUsed += b.length;
return a;
}, uint8);
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
} else if ((body as AsyncIterable<Uint8Array>)[Symbol.asyncIterator]) {
const chunks: Uint8Array[] = [];
for await (const chunk of body as AsyncIterable<Uint8Array>) {
chunks.push(chunk);
}
return Buffer.concat(chunks);
}
throw new TypeError(`Unable to resolve body.`);
}

100
yarn.lock
View File

@@ -1912,7 +1912,6 @@ __metadata:
"@sapphire/async-queue": ^1.3.1
"@sapphire/snowflake": ^3.2.1
"@types/jest": ^27.4.1
"@types/node-fetch": ^2.6.1
"@typescript-eslint/eslint-plugin": ^5.19.0
"@typescript-eslint/parser": ^5.19.0
babel-plugin-const-enum: ^1.2.0
@@ -1922,15 +1921,13 @@ __metadata:
eslint-config-marine: ^9.4.1
eslint-config-prettier: ^8.5.0
eslint-plugin-import: ^2.26.0
form-data: ^4.0.0
jest: ^27.5.1
nock: ^13.2.4
node-fetch: ^2.6.7
prettier: ^2.6.2
tslib: ^2.3.1
tsup: ^5.12.5
typedoc: ^0.22.15
typescript: ^4.6.3
undici: ^5.2.0
languageName: unknown
linkType: soft
@@ -2603,16 +2600,6 @@ __metadata:
languageName: node
linkType: hard
"@types/node-fetch@npm:^2.6.1":
version: 2.6.1
resolution: "@types/node-fetch@npm:2.6.1"
dependencies:
"@types/node": "*"
form-data: ^3.0.0
checksum: a3e5d7f413d1638d795dff03f7b142b1b0e0c109ed210479000ce7b3ea11f9a6d89d9a024c96578d9249570c5fe5287a5f0f4aaba98199222230196ff2d6b283
languageName: node
linkType: hard
"@types/node@npm:*":
version: 17.0.6
resolution: "@types/node@npm:17.0.6"
@@ -4544,7 +4531,7 @@ __metadata:
tslib: ^2.3.1
tslint: ^6.1.3
typescript: ^4.6.3
undici: ^4.16.0
undici: ^5.2.0
ws: ^8.5.0
languageName: unknown
linkType: soft
@@ -5648,17 +5635,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"form-data@npm:^4.0.0":
version: 4.0.0
resolution: "form-data@npm:4.0.0"
dependencies:
asynckit: ^0.4.0
combined-stream: ^1.0.8
mime-types: ^2.1.12
checksum: 01135bf8675f9d5c61ff18e2e2932f719ca4de964e3be90ef4c36aacfc7b9cb2fceb5eca0b7e0190e3383fe51c5b37f4cb80b62ca06a99aaabfcfd6ac7c9328c
languageName: node
linkType: hard
"form-data@npm:~2.3.2":
version: 2.3.3
resolution: "form-data@npm:2.3.3"
@@ -7740,13 +7716,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"lodash.set@npm:^4.3.2":
version: 4.3.2
resolution: "lodash.set@npm:4.3.2"
checksum: a9122f49eef9f2d0fc9061a33d87f8e5b8c6b23d46e8b9e9ce1529d3588d79741bd1145a3abdfa3b13082703e65af27ff18d8a07bfc22b9be32f3fc36f763f70
languageName: node
linkType: hard
"lodash.snakecase@npm:^4.1.1":
version: 4.1.1
resolution: "lodash.snakecase@npm:4.1.1"
@@ -8248,18 +8217,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"nock@npm:^13.2.4":
version: 13.2.4
resolution: "nock@npm:13.2.4"
dependencies:
debug: ^4.1.0
json-stringify-safe: ^5.0.1
lodash.set: ^4.3.2
propagate: ^2.0.0
checksum: 2750a82ea22eebd8203eb1d7669ae09c3daae1fd573026372bad2515adad48d723a804f647bd45d7a499eb3a9a632560da406bde05bca9df762d3027db9099b5
languageName: node
linkType: hard
"node-fetch@npm:2.6.1":
version: 2.6.1
resolution: "node-fetch@npm:2.6.1"
@@ -8267,20 +8224,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"node-fetch@npm:^2.6.7":
version: 2.6.7
resolution: "node-fetch@npm:2.6.7"
dependencies:
whatwg-url: ^5.0.0
peerDependencies:
encoding: ^0.1.0
peerDependenciesMeta:
encoding:
optional: true
checksum: 8d816ffd1ee22cab8301c7756ef04f3437f18dace86a1dae22cf81db8ef29c0bf6655f3215cb0cdb22b420b6fe141e64b26905e7f33f9377a7fa59135ea3e10b
languageName: node
linkType: hard
"node-gyp@npm:latest":
version: 8.4.1
resolution: "node-gyp@npm:8.4.1"
@@ -8928,13 +8871,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"propagate@npm:^2.0.0":
version: 2.0.1
resolution: "propagate@npm:2.0.1"
checksum: c4febaee2be0979e82fb6b3727878fd122a98d64a7fa3c9d09b0576751b88514a9e9275b1b92e76b364d488f508e223bd7e1dcdc616be4cdda876072fbc2a96c
languageName: node
linkType: hard
"psl@npm:^1.1.28, psl@npm:^1.1.33":
version: 1.8.0
resolution: "psl@npm:1.8.0"
@@ -10289,13 +10225,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"tr46@npm:~0.0.3":
version: 0.0.3
resolution: "tr46@npm:0.0.3"
checksum: 726321c5eaf41b5002e17ffbd1fb7245999a073e8979085dacd47c4b4e8068ff5777142fc6726d6ca1fd2ff16921b48788b87225cbc57c72636f6efa8efbffe3
languageName: node
linkType: hard
"tree-kill@npm:^1.2.2":
version: 1.2.2
resolution: "tree-kill@npm:1.2.2"
@@ -10849,10 +10778,10 @@ dts-critic@latest:
languageName: node
linkType: hard
"undici@npm:^4.16.0":
version: 4.16.0
resolution: "undici@npm:4.16.0"
checksum: 5e88c2b3381085e25ed1d1a308610ac7ee985f478ac705af7a8e03213536e10f73ef8dd8d85e6ed38948d1883fa0ae935e04357c317b0f5d3d3c0211d0c8c393
"undici@npm:^5.2.0":
version: 5.2.0
resolution: "undici@npm:5.2.0"
checksum: b7d6fe077c3ab13b7f7a0e5f4b70354b8489c5d37ca06c471754f9543c34501e3b29394b130afc8f066f11e465d6474ec665b2eb5a61052dc84baa97eeb9c7c3
languageName: node
linkType: hard
@@ -11054,13 +10983,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"webidl-conversions@npm:^3.0.0":
version: 3.0.1
resolution: "webidl-conversions@npm:3.0.1"
checksum: c92a0a6ab95314bde9c32e1d0a6dfac83b578f8fa5f21e675bc2706ed6981bc26b7eb7e6a1fab158e5ce4adf9caa4a0aee49a52505d4d13c7be545f15021b17c
languageName: node
linkType: hard
"webidl-conversions@npm:^5.0.0":
version: 5.0.0
resolution: "webidl-conversions@npm:5.0.0"
@@ -11091,16 +11013,6 @@ dts-critic@latest:
languageName: node
linkType: hard
"whatwg-url@npm:^5.0.0":
version: 5.0.0
resolution: "whatwg-url@npm:5.0.0"
dependencies:
tr46: ~0.0.3
webidl-conversions: ^3.0.0
checksum: b8daed4ad3356cc4899048a15b2c143a9aed0dfae1f611ebd55073310c7b910f522ad75d727346ad64203d7e6c79ef25eafd465f4d12775ca44b90fa82ed9e2c
languageName: node
linkType: hard
"whatwg-url@npm:^8.0.0, whatwg-url@npm:^8.5.0":
version: 8.7.0
resolution: "whatwg-url@npm:8.7.0"