feat(DataResolver): prefer streams over buffers (#4075)

* feat(DataResolver): prefer streams over buffers

* feat(DataResolver): add `resolveFileAsBuffer`

Add `resolveFileAsBuffer` to use it in `resolveImage` which still requires Buffers to work.

* fix(DataResolver): make sure `resolveFile` always returns a Promise

* refactor(DataResolver): use for-await-of

* fix(DataResolver): use forked form-data which supports custom streams

* fix(APIRequest): use forked form-data in code too

Co-authored-by: - <5144598+-@users.noreply.github.com>
Co-authored-by: SpaceEEC <spaceeec@yahoo.com>

package.json

@@ -48,8 +48,8 @@
   "unpkg": "./webpack/discord.min.js",
   "dependencies": {
     "@discordjs/collection": "^0.1.5",
+    "@discordjs/form-data": "^3.0.1",
     "abort-controller": "^3.0.0",
-    "form-data": "^3.0.0",
     "node-fetch": "^2.6.0",
     "prism-media": "^1.2.0",
     "setimmediate": "^1.0.5",

src/rest/APIRequest.js

@@ -1,8 +1,8 @@
 'use strict';
 
 const https = require('https');
+const FormData = require('@discordjs/form-data');
 const AbortController = require('abort-controller');
-const FormData = require('form-data');
 const fetch = require('node-fetch');
 const { browser, UserAgent } = require('../util/Constants');
 

src/util/DataResolver.js

@@ -2,6 +2,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const stream = require('stream');
 const fetch = require('node-fetch');
 const { Error: DiscordError, TypeError } = require('../errors');
 const { browser } = require('../util/Constants');
@@ -45,7 +46,7 @@ class DataResolver {
     if (typeof image === 'string' && image.startsWith('data:')) {
       return image;
     }
-    const file = await this.resolveFile(image);
+    const file = await this.resolveFileAsBuffer(image);
     return DataResolver.resolveBase64(file);
   }
 
@@ -79,42 +80,47 @@ class DataResolver {
    * @see {@link https://nodejs.org/api/stream.html}
    */
 
+  /**
+   * Resolves a BufferResolvable to a Buffer or a Stream.
+   * @param {BufferResolvable|Stream} resource The buffer or stream resolvable to resolve
+   * @returns {Promise<Buffer|Stream>}
+   */
+  static async resolveFile(resource) {
+    if (!browser && Buffer.isBuffer(resource)) return resource;
+    if (browser && resource instanceof ArrayBuffer) return Util.convertToBuffer(resource);
+    if (resource instanceof stream.Readable) return resource;
+
+    if (typeof resource === 'string') {
+      if (/^https?:\/\//.test(resource)) {
+        const res = await fetch(resource);
+        return browser ? res.blob() : res.body;
+      } else if (!browser) {
+        return new Promise((resolve, reject) => {
+          const file = path.resolve(resource);
+          fs.stat(file, (err, stats) => {
+            if (err) return reject(err);
+            if (!stats.isFile()) return reject(new DiscordError('FILE_NOT_FOUND', file));
+            return resolve(fs.createReadStream(file));
+          });
+        });
+      }
+    }
+
+    throw new TypeError('REQ_RESOURCE_TYPE');
+  }
+
   /**
    * Resolves a BufferResolvable to a Buffer.
    * @param {BufferResolvable|Stream} resource The buffer or stream resolvable to resolve
    * @returns {Promise<Buffer>}
    */
-  static resolveFile(resource) {
+  static async resolveFileAsBuffer(resource) {
-    if (!browser && Buffer.isBuffer(resource)) return Promise.resolve(resource);
+    const file = await this.resolveFile(resource);
-    if (browser && resource instanceof ArrayBuffer) return Promise.resolve(Util.convertToBuffer(resource));
+    if (Buffer.isBuffer(file)) return file;
 
-    if (typeof resource === 'string') {
+    const buffers = [];
-      if (/^https?:\/\//.test(resource)) {
+    for await (const data of file) buffers.push(data);
-        return fetch(resource).then(res => (browser ? res.blob() : res.buffer()));
+    return Buffer.concat(buffers);
-      } else if (!browser) {
-        return new Promise((resolve, reject) => {
-          const file = browser ? resource : path.resolve(resource);
-          fs.stat(file, (err, stats) => {
-            if (err) return reject(err);
-            if (!stats.isFile()) return reject(new DiscordError('FILE_NOT_FOUND', file));
-            fs.readFile(file, (err2, data) => {
-              if (err2) reject(err2);
-              else resolve(data);
-            });
-            return null;
-          });
-        });
-      }
-    } else if (typeof resource.pipe === 'function') {
-      return new Promise((resolve, reject) => {
-        const buffers = [];
-        resource.once('error', reject);
-        resource.on('data', data => buffers.push(data));
-        resource.once('end', () => resolve(Buffer.concat(buffers)));
-      });
-    }
-
-    return Promise.reject(new TypeError('REQ_RESOURCE_TYPE'));
   }
 }
 

typings/index.d.ts

@@ -11,10 +11,10 @@ declare enum ChannelType {
 
 declare module 'discord.js' {
   import BaseCollection from '@discordjs/collection';
-  import { EventEmitter } from 'events';
-  import { Stream, Readable, Writable } from 'stream';
   import { ChildProcess } from 'child_process';
+  import { EventEmitter } from 'events';
   import { PathLike } from 'fs';
+  import { Readable, Stream, Writable } from 'stream';
   import * as WebSocket from 'ws';
 
   export const version: string;
@@ -539,7 +539,8 @@ declare module 'discord.js' {
 
   export class DataResolver {
     public static resolveBase64(data: Base64Resolvable): string;
-    public static resolveFile(resource: BufferResolvable | Stream): Promise<Buffer>;
+    public static resolveFile(resource: BufferResolvable | Stream): Promise<Buffer | Stream>;
+    public static resolveFileAsBuffer(resource: BufferResolvable | Stream): Promise<Buffer>;
     public static resolveImage(resource: BufferResolvable | Base64Resolvable): Promise<string>;
     public static resolveInviteCode(data: InviteResolvable): string;
   }