From 0224138dc979544df8aae394a0d6fe8c21df932a Mon Sep 17 00:00:00 2001 From: Amish Shah Date: Sat, 13 Aug 2016 22:22:24 +0100 Subject: [PATCH] crappy ratelimiting it doesnt work but its here ok --- .eslintrc.js | 1 + src/client/rest/APIRequest.js | 43 ++++++++++++++ src/client/rest/Bucket.js | 62 ++++++++++++++++++++ src/client/rest/RESTManager.js | 101 ++++++--------------------------- src/util/Constants.js | 1 + test/random.js | 8 +++ 6 files changed, 131 insertions(+), 85 deletions(-) create mode 100644 src/client/rest/APIRequest.js create mode 100644 src/client/rest/Bucket.js diff --git a/.eslintrc.js b/.eslintrc.js index 2ea4a252b..a721fe464 100644 --- a/.eslintrc.js +++ b/.eslintrc.js @@ -12,5 +12,6 @@ module.exports = { "guard-for-in": 0, "no-restricted-syntax": 0, "no-param-reassign": 0, + "consistent-return": 0, } }; \ No newline at end of file diff --git a/src/client/rest/APIRequest.js b/src/client/rest/APIRequest.js new file mode 100644 index 000000000..462e78fdc --- /dev/null +++ b/src/client/rest/APIRequest.js @@ -0,0 +1,43 @@ +const request = require('superagent'); +const Constants = require('../../util/Constants'); + +class APIRequest { + constructor(rest, method, url, auth, data, file) { + this.rest = rest; + this.method = method; + this.url = url; + this.auth = auth; + this.data = data; + this.file = file; + } + + getBucketName() { + return `${this.method} ${this.url}`; + } + + getAuth() { + if (this.rest.client.store.token && this.rest.client.store.user && this.rest.client.store.user.bot) { + return `Bot ${this.rest.client.store.token}`; + } else if (this.rest.client.store.token) { + return this.rest.client.store.token; + } + throw Constants.Errors.NO_TOKEN; + } + + gen() { + const apiRequest = request[this.method](this.url); + if (this.auth) { + apiRequest.set('authorization', this.getAuth()); + } + if (this.data) { + apiRequest.send(this.data); + } + if (this.file) { + apiRequest.attach('file', this.file.file, this.file.name); + } + apiRequest.set('User-Agent', this.rest.userAgentManager.userAgent); + return apiRequest; + } +} + +module.exports = APIRequest; diff --git a/src/client/rest/Bucket.js b/src/client/rest/Bucket.js new file mode 100644 index 000000000..b93ca9607 --- /dev/null +++ b/src/client/rest/Bucket.js @@ -0,0 +1,62 @@ +class Bucket { + constructor(rest, limit, remainingRequests = 1, resetTime) { + this.rest = rest; + this.limit = limit; + this.remainingRequests = remainingRequests; + this.resetTime = resetTime; + this.locked = false; + this.queue = []; + this.nextCheck = null; + } + + setCheck(time) { + clearTimeout(this.nextCheck); + console.log('going to iterate in', time, 'remaining:', this.queue.length); + this.nextCheck = setTimeout(() => { + this.remainingRequests = this.limit - 1; + this.locked = false; + this.process(); + }, time); + } + + process() { + if (this.locked) { + return; + } + + this.locked = true; + + if (this.queue.length === 0) { + return; + } + + if (this.remainingRequests === 0) { + return; + } + console.log('bucket is going to iterate', Math.min(this.remainingRequests, this.queue.length), 'items with max', this.limit, 'and remaining', this.remainingRequests); + while (Math.min(this.remainingRequests, this.queue.length) > 0) { + const item = this.queue.shift(); + item.request.gen().end((err, res) => { + if (res && res.headers) { + this.limit = res.headers['x-ratelimit-limit']; + this.resetTime = Number(res.headers['x-ratelimit-reset']) * 1000; + this.setCheck((Math.max(500, this.resetTime - Date.now())) + 1000); + } + if (err) { + console.log(err.status, this.remainingRequests); + item.reject(err); + } else { + item.resolve(res && res.body ? res.body : {}); + } + }); + this.remainingRequests--; + } + } + + add(method) { + this.queue.push(method); + this.process(); + } +} + +module.exports = Bucket; diff --git a/src/client/rest/RESTManager.js b/src/client/rest/RESTManager.js index 2ebc184e8..d0275ff44 100644 --- a/src/client/rest/RESTManager.js +++ b/src/client/rest/RESTManager.js @@ -2,110 +2,41 @@ const request = require('superagent'); const Constants = require('../../util/Constants'); const UserAgentManager = require('./UserAgentManager'); const RESTMethods = require('./RESTMethods'); +const Bucket = require('./Bucket'); +const APIRequest = require('./APIRequest'); class RESTManager { constructor(client) { this.client = client; - this.queue = []; + this.buckets = {}; this.userAgentManager = new UserAgentManager(this); this.methods = new RESTMethods(this); this.rateLimitedEndpoints = {}; } - addRequestToQueue(method, url, auth, data, file, resolve, reject) { - const endpoint = url.replace(/\/[0-9]+/g, '/:id'); - - const rateLimitedEndpoint = this.rateLimitedEndpoints[endpoint]; - - rateLimitedEndpoint.queue = rateLimitedEndpoint.queue || []; - - rateLimitedEndpoint.queue.push({ - method, - url, - auth, - data, - file, - resolve, - reject, + addToBucket(bucket, apiRequest) { + return new Promise((resolve, reject) => { + bucket.add({ + request: apiRequest, + resolve, + reject, + }); }); } - processQueue(endpoint) { - const rateLimitedEndpoint = this.rateLimitedEndpoints[endpoint]; - - // prevent multiple queue processes - if (!rateLimitedEndpoint.timeout) { - return; - } - - // lock the queue - clearTimeout(rateLimitedEndpoint.timeout); - rateLimitedEndpoint.timeout = null; - - for (const item of rateLimitedEndpoint.queue) { - this.makeRequest(item.method, item.url, item.auth, item.data, item.file) - .then(item.resolve) - .catch(item.reject); - } - - rateLimitedEndpoint.queue = []; - } - makeRequest(method, url, auth, data, file) { /* file is {file, name} */ - const apiRequest = request[method](url); + const apiRequest = new APIRequest(this, method, url, auth, data, file); - const endpoint = url.replace(/\/[0-9]+/g, '/:id'); - - if (this.rateLimitedEndpoints[endpoint] && this.rateLimitedEndpoints[endpoint].timeout) { - return new Promise((resolve, reject) => { - this.addRequestToQueue(method, url, auth, data, file, resolve, reject); - }); + if (!this.buckets[apiRequest.getBucketName()]) { + console.log('new bucket', apiRequest.getBucketName()); + this.buckets[apiRequest.getBucketName()] = new Bucket(this, 1, 1); } - - if (auth) { - if (this.client.store.token && this.client.store.user && this.client.store.user.bot) { - apiRequest.set('authorization', `Bot ${this.client.store.token}`); - } else if (this.client.store.token) { - apiRequest.set('authorization', this.client.store.token); - } else { - throw Constants.Errors.NO_TOKEN; - } - } - - if (data) { - apiRequest.send(data); - } - - if (file) { - apiRequest.attach('file', file.file, file.name); - } - - apiRequest.set('User-Agent', this.userAgentManager.userAgent); - - return new Promise((resolve, reject) => { - apiRequest.end((err, res) => { - if (err) { - const retry = res.headers['retry-after'] || res.headers['Retry-After']; - if (retry) { - this.rateLimitedEndpoints[endpoint] = {}; - this.addRequestToQueue(method, url, auth, data, file, resolve, reject); - this.rateLimitedEndpoints[endpoint].timeout = setTimeout(() => { - this.processQueue(endpoint); - }, retry); - return; - } - - reject(err); - } else { - console.log(res.headers); - resolve(res ? res.body || {} : {}); - } - }); - }); + + return this.addToBucket(this.buckets[apiRequest.getBucketName()], apiRequest); } } diff --git a/src/util/Constants.js b/src/util/Constants.js index bfaf84e57..ae8ba951c 100644 --- a/src/util/Constants.js +++ b/src/util/Constants.js @@ -165,6 +165,7 @@ const PermissionFlags = exports.PermissionFlags = { ATTACH_FILES: 1 << 15, READ_MESSAGE_HISTORY: 1 << 16, MENTION_EVERYONE: 1 << 17, + EXTERNAL_EMOJIS: 1 << 18, CONNECT: 1 << 20, SPEAK: 1 << 21, diff --git a/test/random.js b/test/random.js index 54516b2be..ba89da291 100644 --- a/test/random.js +++ b/test/random.js @@ -141,6 +141,14 @@ client.on('message', message => { }).catch(console.log); } + if (message.content === 'ratelimittest') { + let i = 0; + while (i < 20) { + message.channel.sendMessage(`Testing my rates, item ${i} of 20`); + i++; + } + } + if (message.content === 'makerole') { message.guild.createRole().then(role => { message.channel.sendMessage(`Made role ${role.name}`);