Merge branch 'shard-overhaul'

.eslintrc.json

@@ -127,7 +127,11 @@
     "semi-spacing": "error",
     "semi": "error",
     "space-before-blocks": "error",
-    "space-before-function-paren": ["error", "never"],
+    "space-before-function-paren": ["error", {
+      "anonymous": "never",
+      "named": "never",
+      "asyncArrow": "always"
+    }],
     "space-in-parens": "error",
     "space-infix-ops": "error",
     "space-unary-ops": "error",

src/client/voice/util/Secretbox.js

@@ -15,7 +15,7 @@ const libs = {
 
 exports.methods = {};
 
-(async() => {
+(async () => {
   for (const libName of Object.keys(libs)) {
     try {
       const lib = require(libName);

src/errors/Messages.js

@@ -20,9 +20,13 @@ const Messages = {
   SHARDING_REQUIRED: 'This session would have handled too many guilds - Sharding is required.',
   SHARDING_CHILD_CONNECTION: 'Failed to send message to shard\'s process.',
   SHARDING_PARENT_CONNECTION: 'Failed to send message to master process.',
-  SHARDING_NO_SHARDS: 'No shards have been spawned',
-  SHARDING_IN_PROCESS: 'Shards are still being spawned',
-  SHARDING_ALREADY_SPAWNED: count => `Already spawned ${count} shards`,
+  SHARDING_NO_SHARDS: 'No shards have been spawned.',
+  SHARDING_IN_PROCESS: 'Shards are still being spawned.',
+  SHARDING_ALREADY_SPAWNED: count => `Already spawned ${count} shards.`,
+  SHARDING_PROCESS_EXISTS: id => `Shard ${id} already has an active process.`,
+  SHARDING_READY_TIMEOUT: id => `Shard ${id}'s Client took too long to become ready.`,
+  SHARDING_READY_DISCONNECTED: id => `Shard ${id}'s Client disconnected before becoming ready.`,
+  SHARDING_READY_DIED: id => `Shard ${id}'s process exited before its Client became ready.`,
 
   COLOR_RANGE: 'Color must be within the range 0 - 16777215 (0xFFFFFF).',
   COLOR_CONVERT: 'Unable to convert color to a number.',

src/sharding/Shard.js

@@ -5,7 +5,10 @@ const Util = require('../util/Util');
 const { Error } = require('../errors');
 
 /**
- * Represents a Shard spawned by the ShardingManager.
+ * A self-contained shard created by the {@link ShardingManager}. Each one has a {@link ChildProcess} that contains
+ * an instance of the bot and its {@link Client}. When its child process exits for any reason, the shard will spawn a
+ * new one to replace it as necessary.
+ * @extends EventEmitter
  */
 class Shard extends EventEmitter {
   /**
@@ -15,6 +18,7 @@ class Shard extends EventEmitter {
    */
   constructor(manager, id, args = []) {
     super();
+
     /**
      * Manager that created the shard
      * @type {ShardingManager}
@@ -27,6 +31,12 @@ class Shard extends EventEmitter {
      */
     this.id = id;
 
+    /**
+     * Arguments for the shard's process
+     * @type {string[]}
+     */
+    this.args = args;
+
     /**
      * Environment variables for the shard's process
      * @type {Object}
@@ -37,20 +47,18 @@ class Shard extends EventEmitter {
       CLIENT_TOKEN: this.manager.token,
     });
 
-    /**
-     * Process of the shard
-     * @type {ChildProcess}
-     */
-    this.process = childProcess.fork(path.resolve(this.manager.file), args, {
-      env: this.env,
-    }).on('message', this._handleMessage.bind(this));
-
     /**
      * Whether the shard's {@link Client} is ready
      * @type {boolean}
      */
     this.ready = false;
 
+    /**
+     * Process of the shard
+     * @type {?ChildProcess}
+     */
+    this.process = null;
+
     /**
      * Ongoing promises for calls to {@link Shard#eval}, mapped by the `script` they were called with
      * @type {Map<string, Promise>}
@@ -65,11 +73,56 @@ class Shard extends EventEmitter {
      */
     this._fetches = new Map();
 
-    // Handle the death of the process
-    this.process.once('exit', () => {
-      this.ready = false;
-      if (this.manager.respawn) this.manager.createShard(this.id).catch(err => { this.manager.emit('error', err); });
+    /**
+     * Listener function for the {@link ChildProcess}' `exit` event
+     * @type {Function}
+     * @private
+     */
+    this._exitListener = this._handleExit.bind(this, undefined);
+  }
+
+  /**
+   * Forks a child process for the shard.
+   * <warn>You should not need to call this manually.</warn>
+   * @param {boolean} [waitForReady=true] Whether to wait until the {@link Client} has become ready before resolving
+   * @returns {Promise<ChildProcess>}
+   */
+  async spawn(waitForReady = true) {
+    if (this.process) throw new Error('SHARDING_PROCESS_EXISTS', this.id);
+
+    this.process = childProcess.fork(path.resolve(this.manager.file), this.args, { env: this.env })
+      .on('message', this._handleMessage.bind(this))
+      .on('exit', this._exitListener);
+
+    /**
+     * Emitted upon the creation of the shard's child process.
+     * @event Shard#spawn
+     * @param {ChildProcess} process Child process that was created
+     */
+    this.emit('spawn', this.process);
+
+    if (!waitForReady) return this.process;
+    await new Promise((resolve, reject) => {
+      this.once('ready', resolve);
+      this.once('disconnect', () => reject(new Error('SHARDING_READY_DISCONNECTED', this.id)));
+      this.once('death', () => reject(new Error('SHARDING_READY_DIED', this.id)));
+      setTimeout(() => reject(new Error('SHARDING_READY_TIMEOUT', this.id)), 30000);
     });
+    return this.process;
+  }
+
+  /**
+   * Kills and restarts the shard's process.
+   * @param {number} [delay=500] How long to wait between killing the process and restarting it (in milliseconds)
+   * @param {boolean} [waitForReady=true] Whether to wait the {@link Client} has become ready before resolving
+   * @returns {Promise<ChildProcess>}
+   */
+  async respawn(delay = 500, waitForReady = true) {
+    this.process.removeListener('exit', this._exitListener);
+    this.process.kill();
+    this._handleExit(false);
+    if (delay > 0) await Util.delayFor(delay);
+    return this.spawn(waitForReady);
   }
 
   /**
@@ -121,7 +174,7 @@ class Shard extends EventEmitter {
   }
 
   /**
-   * Evaluates a script on the shard, in the context of the client.
+   * Evaluates a script on the shard, in the context of the {@link Client}.
    * @param {string} script JavaScript to run on the shard
    * @returns {Promise<*>} Result of the script execution
    */
@@ -205,15 +258,44 @@ class Shard extends EventEmitter {
         );
         return;
       }
+
+      // Shard is requesting a respawn of all shards
+      if (message._sRespawnAll) {
+        const { shardDelay, respawnDelay, waitForReady } = message._sRespawnAll;
+        this.manager.respawnAll(shardDelay, respawnDelay, waitForReady).catch(() => {
+          // Do nothing
+        });
+        return;
+      }
     }
 
     /**
-     * Emitted upon recieving a message from a shard.
-     * @event ShardingManager#message
-     * @param {Shard} shard Shard that sent the message
+     * Emitted upon recieving a message from the child process.
+     * @event Shard#message
      * @param {*} message Message that was received
      */
-    this.manager.emit('message', this, message);
+    this.emit('message', message);
+  }
+
+  /**
+   * Handles the shard's process exiting.
+   * @param {boolean} [respawn=this.manager.respawn] Whether to spawn the shard again
+   * @private
+   */
+  _handleExit(respawn = this.manager.respawn) {
+    /**
+     * Emitted upon the shard's child process exiting.
+     * @event Shard#death
+     * @param {ChildProcess} process Child process that exited
+     */
+    this.emit('death', this.process);
+
+    this.ready = false;
+    this.process = null;
+    this._evals.clear();
+    this._fetches.clear();
+
+    if (respawn) this.spawn().catch(err => this.emit('error', err));
   }
 }
 

src/sharding/ShardClientUtil.js

@@ -3,7 +3,8 @@ const { Events } = require('../util/Constants');
 const { Error } = require('../errors');
 
 /**
- * Helper class for sharded clients spawned as a child process, such as from a ShardingManager.
+ * Helper class for sharded clients spawned as a child process, such as from a {@link ShardingManager}.
+ * Utilises IPC to send and receive data to/from the master process and other shards.
  */
 class ShardClientUtil {
   /**
@@ -59,6 +60,7 @@ class ShardClientUtil {
    *     console.log(`${results.reduce((prev, val) => prev + val, 0)} total guilds`);
    *   })
    *   .catch(console.error);
+   * @see {@link ShardingManager#fetchClientValues}
    */
   fetchClientValues(prop) {
     return new Promise((resolve, reject) => {
@@ -77,9 +79,10 @@ class ShardClientUtil {
   }
 
   /**
-   * Evaluates a script on all shards, in the context of the Clients.
+   * Evaluates a script on all shards, in the context of the {@link Clients}.
    * @param {string} script JavaScript to run on each shard
    * @returns {Promise<Array<*>>} Results of the script execution
+   * @see {@link ShardingManager#broadcastEval}
    */
   broadcastEval(script) {
     return new Promise((resolve, reject) => {
@@ -97,6 +100,19 @@ class ShardClientUtil {
     });
   }
 
+  /**
+   * Requests a respawn of all shards.
+   * @param {number} [shardDelay=5000] How long to wait between shards (in milliseconds)
+   * @param {number} [respawnDelay=500] How long to wait between killing a shard's process and restarting it
+   * (in milliseconds)
+   * @param {boolean} [waitForReady=true] Whether to wait for a shard to become ready before continuing to another
+   * @returns {Promise<void>} Resolves upon the message being sent
+   * @see {@link ShardingManager#respawnAll}
+   */
+  respawnAll(shardDelay = 5000, respawnDelay = 500, waitForReady = true) {
+    return this.send({ _sRespawnAll: { shardDelay, respawnDelay, waitForReady } });
+  }
+
   /**
    * Handles an IPC message.
    * @param {*} message Message received

src/sharding/ShardingManager.js

@@ -7,9 +7,12 @@ const Util = require('../util/Util');
 const { Error, TypeError, RangeError } = require('../errors');
 
 /**
- * This is a utility class that can be used to help you spawn shards of your client. Each shard is completely separate
- * from the other. The Shard Manager takes a path to a file and spawns it under the specified amount of shards safely.
- * If you do not select an amount of shards, the manager will automatically decide the best amount.
+ * This is a utility class that makes multi-process sharding of a bot an easy and painless experience.
+ * It works by spawning a self-contained {@link ChildProcess} for each individual shard, each containing its own
+ * instance of your bot's {@link Client}. They all have a line of communication with the master process, and there are
+ * several useful methods that utilise it in order to simplify tasks that are normally difficult with sharding. It can
+ * spawn a specific number of shards or the amount that Discord suggests for the bot, and takes a path to your main bot
+ * script to launch for each one.
  * @extends {EventEmitter}
  */
 class ShardingManager extends EventEmitter {
@@ -82,33 +85,33 @@ class ShardingManager extends EventEmitter {
 
   /**
    * Spawns a single shard.
-   * @param {number} id The ID of the shard to spawn. **This is usually not necessary**
-   * @returns {Promise<Shard>}
+   * @param {number} [id=this.shards.size] ID of the shard to spawn -
+   * **This is usually not necessary to manually specify.**
+   * @returns {Shard}
    */
   createShard(id = this.shards.size) {
     const shard = new Shard(this, id, this.shardArgs);
     this.shards.set(id, shard);
     /**
-     * Emitted upon launching a shard.
-     * @event ShardingManager#launch
-     * @param {Shard} shard Shard that was launched
+     * Emitted upon creating a shard.
+     * @event ShardingManager#shardCreate
+     * @param {Shard} shard Shard that was created
      */
-    this.emit('launch', shard);
-    return Promise.resolve(shard);
+    this.emit('shardCreate', shard);
+    return shard;
   }
 
   /**
    * Spawns multiple shards.
    * @param {number} [amount=this.totalShards] Number of shards to spawn
-   * @param {number} [delay=7500] How long to wait in between spawning each shard (in milliseconds)
+   * @param {number} [delay=5500] How long to wait in between spawning each shard (in milliseconds)
+   * @param {boolean} [waitForReady=true] Whether to wait for a shard to become ready before continuing to another
    * @returns {Promise<Collection<number, Shard>>}
    */
-  spawn(amount = this.totalShards, delay = 7500) {
+  async spawn(amount = this.totalShards, delay = 5500, waitForReady = true) {
+    // Obtain/verify the number of shards to spawn
     if (amount === 'auto') {
-      return Util.fetchRecommendedShards(this.token).then(count => {
-        this.totalShards = count;
-        return this._spawn(count, delay);
-      });
+      amount = await Util.fetchRecommendedShards(this.token);
     } else {
       if (typeof amount !== 'number' || isNaN(amount)) {
         throw new TypeError('CLIENT_INVALID_OPTION', 'Amount of shards', 'a number.');
@@ -117,41 +120,22 @@ class ShardingManager extends EventEmitter {
       if (amount !== Math.floor(amount)) {
         throw new TypeError('CLIENT_INVALID_OPTION', 'Amount of shards', 'an integer.');
       }
-      return this._spawn(amount, delay);
     }
-  }
 
-  /**
-   * Actually spawns shards, unlike that poser above >:(
-   * @param {number} amount Number of shards to spawn
-   * @param {number} delay How long to wait in between spawning each shard (in milliseconds)
-   * @returns {Promise<Collection<number, Shard>>}
-   * @private
-   */
-  _spawn(amount, delay) {
-    return new Promise(resolve => {
-      if (this.shards.size >= amount) throw new Error('SHARDING_ALREADY_SPAWNED', this.shards.size);
-      this.totalShards = amount;
+    // Make sure this many shards haven't already been spawned
+    if (this.shards.size >= amount) throw new Error('SHARDING_ALREADY_SPAWNED', this.shards.size);
+    this.totalShards = amount;
 
-      this.createShard();
-      if (this.shards.size >= this.totalShards) {
-        resolve(this.shards);
-        return;
-      }
+    // Spawn the shards
+    for (let s = 1; s <= amount; s++) {
+      const promises = [];
+      const shard = this.createShard();
+      promises.push(shard.spawn(waitForReady));
+      if (delay > 0 && s !== amount) promises.push(Util.delayFor(delay));
+      await Promise.all(promises); // eslint-disable-line no-await-in-loop
+    }
 
-      if (delay <= 0) {
-        while (this.shards.size < this.totalShards) this.createShard();
-        resolve(this.shards);
-      } else {
-        const interval = setInterval(() => {
-          this.createShard();
-          if (this.shards.size >= this.totalShards) {
-            clearInterval(interval);
-            resolve(this.shards);
-          }
-        }, delay);
-      }
-    });
+    return this.shards;
   }
 
   /**
@@ -166,7 +150,7 @@ class ShardingManager extends EventEmitter {
   }
 
   /**
-   * Evaluates a script on all shards, in the context of the Clients.
+   * Evaluates a script on all shards, in the context of the {@link Client}s.
    * @param {string} script JavaScript to run on each shard
    * @returns {Promise<Array<*>>} Results of the script execution
    */
@@ -194,6 +178,24 @@ class ShardingManager extends EventEmitter {
     for (const shard of this.shards.values()) promises.push(shard.fetchClientValue(prop));
     return Promise.all(promises);
   }
+
+  /**
+   * Kills all running shards and respawns them.
+   * @param {number} [shardDelay=5000] How long to wait between shards (in milliseconds)
+   * @param {number} [respawnDelay=500] How long to wait between killing a shard's process and restarting it
+   * (in milliseconds)
+   * @param {boolean} [waitForReady=true] Whether to wait for a shard to become ready before continuing to another
+   * @returns {Promise<Collection<string, Shard>>}
+   */
+  async respawnAll(shardDelay = 5000, respawnDelay = 500, waitForReady = true) {
+    let s = 0;
+    for (const shard of this.shards) {
+      const promises = [shard.respawn(respawnDelay, waitForReady)];
+      if (++s < this.shards.size && shardDelay > 0) promises.push(Util.delayFor(shardDelay));
+      await Promise.all(promises); // eslint-disable-line no-await-in-loop
+    }
+    return this.shards;
+  }
 }
 
 module.exports = ShardingManager;

src/util/Util.js

@@ -366,6 +366,18 @@ class Util {
 
     return dec;
   }
+
+  /**
+   * Creates a Promise that resolves after a specified duration.
+   * @param {number} ms How long to wait before resolving (in milliseconds)
+   * @returns {Promise<void>}
+   * @private
+   */
+  static delayFor(ms) {
+    return new Promise(resolve => {
+      setTimeout(resolve, ms);
+    });
+  }
 }
 
 module.exports = Util;