diff --git a/src/client/voice/VoiceConnection.js b/src/client/voice/VoiceConnection.js
index 8adbbc740..33f72a125 100644
--- a/src/client/voice/VoiceConnection.js
+++ b/src/client/voice/VoiceConnection.js
@@ -1,7 +1,7 @@
 const VoiceWebSocket = require('./networking/VoiceWebSocket');
 const VoiceUDP = require('./networking/VoiceUDPClient');
 const Util = require('../../util/Util');
-const { OPCodes, VoiceOPCodes, VoiceStatus } = require('../../util/Constants');
+const { OPCodes, VoiceOPCodes, VoiceStatus, Events } = require('../../util/Constants');
 const AudioPlayer = require('./player/AudioPlayer');
 const VoiceReceiver = require('./receiver/Receiver');
 const EventEmitter = require('events');
@@ -94,12 +94,19 @@ class VoiceConnection extends EventEmitter {
     this.once('closing', () => this.player.destroy());
 
     /**
-     * Map SSRC to speaking values
-     * @type {Map<number, boolean>}
+     * Map SSRC values to user IDs
+     * @type {Map<number, Snowflake>}
      * @private
      */
     this.ssrcMap = new Map();
 
+    /**
+     * Tracks which users are talking
+     * @type {Map<Snowflake, boolean>}
+     * @private
+     */
+    this._speaking = new Map();
+
     /**
      * Object that wraps contains the `ws` and `udp` sockets of this voice connection
      * @type {Object}
@@ -431,6 +438,8 @@ class VoiceConnection extends EventEmitter {
     const guild = this.channel.guild;
     const user = this.client.users.get(user_id);
     this.ssrcMap.set(+ssrc, user_id);
+    const old = this._speaking.get(user_id);
+    this._speaking.set(user_id, speaking);
     /**
      * Emitted whenever a user starts/stops speaking.
      * @event VoiceConnection#speaking
@@ -445,7 +454,19 @@ class VoiceConnection extends EventEmitter {
         }
       }
     }
-    guild._memberSpeakUpdate(user_id, speaking);
+
+    if (guild && user && old !== speaking) {
+      const member = guild.member(user);
+      if (member) {
+        /**
+         * Emitted once a guild member starts/stops speaking.
+         * @event Client#guildMemberSpeaking
+         * @param {GuildMember} member The member that started/stopped speaking
+         * @param {boolean} speaking Whether or not the member is speaking
+         */
+        this.client.emit(Events.GUILD_MEMBER_SPEAKING, member, speaking);
+      }
+    }
   }
 
   /**
diff --git a/src/structures/Guild.js b/src/structures/Guild.js
index 184513159..eddfa52e2 100644
--- a/src/structures/Guild.js
+++ b/src/structures/Guild.js
@@ -879,26 +879,6 @@ class Guild extends Base {
       c.type === channel.type && (category || c.parent === channel.parent)
     ));
   }
-
-  /**
-   * Handles a user speaking update in a voice channel.
-   * @param {Snowflake} user ID of the user that the update is for
-   * @param {boolean} speaking Whether the user is speaking
-   * @private
-   */
-  _memberSpeakUpdate(user, speaking) {
-    const member = this.members.get(user);
-    if (member && member.speaking !== speaking) {
-      member.speaking = speaking;
-      /**
-       * Emitted once a guild member starts/stops speaking.
-       * @event Client#guildMemberSpeaking
-       * @param {GuildMember} member The member that started/stopped speaking
-       * @param {boolean} speaking Whether or not the member is speaking
-       */
-      this.client.emit(Events.GUILD_MEMBER_SPEAKING, member, speaking);
-    }
-  }
 }
 
 // TODO: Document this thing
@@ -914,7 +894,6 @@ class VoiceStateCollection extends Collection {
       if (member.voiceChannel && member.voiceChannel.id !== voiceState.channel_id) {
         member.voiceChannel.members.delete(member.id);
       }
-      if (!voiceState.channel_id) member.speaking = null;
       const newChannel = this.guild.channels.get(voiceState.channel_id);
       if (newChannel) newChannel.members.set(member.user.id, member);
     }
diff --git a/src/structures/GuildMember.js b/src/structures/GuildMember.js
index e9c5dc663..ea4ddab8e 100644
--- a/src/structures/GuildMember.js
+++ b/src/structures/GuildMember.js
@@ -56,14 +56,19 @@ class GuildMember extends Base {
     if (data) this._patch(data);
   }
 
-  _patch(data) {
-    /**
-     * Whether this member is speaking and the client is in the same channel
-     * @type {boolean}
-     * @name GuildMember#speaking
-     */
-    if (typeof this.speaking === 'undefined') this.speaking = false;
+  /**
+   * Whether this member is speaking. If the client isn't sure, then this will be undefined. Otherwise it will be
+   * true/false
+   * @type {?boolean}
+   * @name GuildMember#speaking
+   */
+  get speaking() {
+    return this.voiceChannel && this.voiceChannel.connection ?
+      Boolean(this.voiceChannel.connection._speaking.get(this.id)) :
+      undefined;
+  }
 
+  _patch(data) {
     /**
      * The nickname of this member, if they have one
      * @type {?string}