feat(Util): add SweptCollection for auto sweeping of caches (#6110)

Co-authored-by: DTrombett <73136330+DTrombett@users.noreply.github.com>
Co-authored-by: 1Computer1 <22125769+1Computer1@users.noreply.github.com>
Co-authored-by: Antonio Román <kyradiscord@gmail.com>
Co-authored-by: NotSugden <28943913+NotSugden@users.noreply.github.com>
Co-authored-by: Shino <shinotheshino@gmail.com>
Co-authored-by: SpaceEEC <24881032+SpaceEEC@users.noreply.github.com>
Co-authored-by: Noel <icrawltogo@gmail.com>
Co-authored-by: Vlad Frangu <kingdgrizzle@gmail.com>

src/util/Collection.js

@@ -1,6 +1,6 @@
 'use strict';
 
-const BaseCollection = require('@discordjs/collection');
+const { Collection: BaseCollection } = require('@discordjs/collection');
 const Util = require('./Util');
 
 class Collection extends BaseCollection {

src/util/Constants.js

@@ -971,6 +971,8 @@ exports.PrivacyLevels = createEnum([null, 'PUBLIC', 'GUILD_ONLY']);
  */
 exports.PremiumTiers = createEnum(['NONE', 'TIER_1', 'TIER_2', 'TIER_3']);
 
+exports._cleanupSymbol = Symbol('djsCleanup');
+
 function keyMirror(arr) {
   let tmp = Object.create(null);
   for (const value of arr) tmp[value] = value;

src/util/Options.js

@@ -35,10 +35,11 @@
  * (e.g. recommended shard count, shard count of the ShardingManager)
  * @property {CacheFactory} [makeCache] Function to create a cache.
  * You can use your own function, or the {@link Options} class to customize the Collection used for the cache.
- * @property {number} [messageCacheLifetime=0] How long a message should stay in the cache until it is considered
- * sweepable (in seconds, 0 for forever)
- * @property {number} [messageSweepInterval=0] How frequently to remove messages from the cache that are older than
- * the message cache lifetime (in seconds, 0 for never)
+ * @property {number} [messageCacheLifetime=0] DEPRECATED: Use `makeCache` with a `SweptCollection` instead.
+ * How long a message should stay in the cache until it is considered sweepable (in seconds, 0 for forever)
+ * @property {number} [messageSweepInterval=0] DEPRECATED: Use `makeCache` with a `SweptCollection` instead.
+ * How frequently to remove messages from the cache that are older than the message cache lifetime
+ * (in seconds, 0 for never)
  * @property {MessageMentionOptions} [allowedMentions] Default value for {@link MessageOptions#allowedMentions}
  * @property {number} [invalidRequestWarningInterval=0] The number of invalid REST requests (those that return
  * 401, 403, or 429) in a 10 minute window between emitted warnings (0 for no warnings). That is, if set to 500,
@@ -99,7 +100,15 @@ class Options extends null {
   static createDefault() {
     return {
       shardCount: 1,
-      makeCache: this.cacheWithLimits({ MessageManager: 200 }),
+      makeCache: this.cacheWithLimits({
+        MessageManager: 200,
+        ThreadManager: {
+          sweepFilter: require('./SweptCollection').filterByLifetime({
+            getComparisonTimestamp: e => e.archiveTimestamp,
+            excludeFromSweep: e => !e.archived,
+          }),
+        },
+      }),
       messageCacheLifetime: 0,
       messageSweepInterval: 0,
       invalidRequestWarningInterval: 0,
@@ -134,20 +143,52 @@ class Options extends null {
   }
 
   /**
-   * Create a cache factory using predefined limits.
-   * @param {Record<string, number>} [limits={}] Limits for structures.
+   * Create a cache factory using predefined settings to sweep or limit.
+   * @param {Object<string, SweptCollectionOptions|number>} [settings={}] Settings passed to the relevant constructor.
+   * If no setting is provided for a manager, it uses Collection.
+   * If SweptCollectionOptions are provided for a manager, it uses those settings to form a SweptCollection
+   * If a number is provided for a manager, it uses that number as the max size for a LimitedCollection
    * @returns {CacheFactory}
+   * @example
+   * // Store up to 200 messages per channel and discard archived threads if they were archived more than 4 hours ago.
+   * Options.cacheWithLimits({
+   *    MessageManager: 200,
+   *    ThreadManager: {
+   *      sweepFilter: SweptCollection.filterByLifetime({
+   *        getComparisonTimestamp: e => e.archiveTimestamp,
+   *        excludeFromSweep: e => !e.archived,
+   *      }),
+   *    },
+   *  });
+   * @example
+   * // Sweep messages every 5 minutes, removing messages that have not been edited or created in the last 30 minutes
+   * Options.cacheWithLimits({
+   *   MessageManager: {
+   *     sweepInterval: 300,
+   *     sweepFilter: SweptCollection.filterByLifetime({
+   *       lifetime: 1800,
+   *       getComparisonTimestamp: e => e.editedTimestamp ?? e.createdTimestamp,
+   *     })
+   *   }
+   * });
    */
-  static cacheWithLimits(limits = {}) {
+  static cacheWithLimits(settings = {}) {
     const Collection = require('./Collection');
     const LimitedCollection = require('./LimitedCollection');
+    const SweptCollection = require('./SweptCollection');
 
     return manager => {
-      const limit = limits[manager.name];
-      if (limit === null || limit === undefined || limit === Infinity) {
+      const setting = settings[manager.name];
+      if (typeof setting === 'number' && setting !== Infinity) return new LimitedCollection(setting);
+      if (
+        /* eslint-disable-next-line eqeqeq */
+        (setting?.sweepInterval == null && setting?.sweepFilter == null) ||
+        setting.sweepInterval <= 0 ||
+        setting.sweepInterval === Infinity
+      ) {
         return new Collection();
       }
-      return new LimitedCollection(limit);
+      return new SweptCollection(setting);
     };
   }
 

src/util/SweptCollection.js

@@ -0,0 +1,115 @@
+'use strict';
+
+const Collection = require('./Collection.js');
+const { _cleanupSymbol } = require('./Constants.js');
+const { TypeError } = require('../errors/DJSError.js');
+
+/**
+ * @typedef {Function} SweepFilter
+ * @param {SweptCollection} collection The collection being swept
+ * @returns {Function|null} Return `null` to skip sweeping, otherwise a function passed to `sweep()`,
+ * See {@link [Collection#sweep](https://discord.js.org/#/docs/collection/master/class/Collection?scrollTo=sweep)}
+ * for the definition of this function.
+ */
+
+/**
+ * Options for defining the behavior of a Swept Collection
+ * @typedef {Object} SweptCollectionOptions
+ * @property {?SweepFitler} [sweepFilter=null] A function run every `sweepInterval` to determine how to sweep
+ * @property {number} [sweepInterval=3600] How frequently, in seconds, to sweep the collection.
+ */
+
+/**
+ * A Collection which holds a max amount of entries and sweeps periodically.
+ * @extends {Collection}
+ * @param {SweptCollectionOptions} [options={}] Options for constructing the swept collection.
+ * @param {Iterable} [iterable=null] Optional entries passed to the Map constructor.
+ */
+class SweptCollection extends Collection {
+  constructor(options = {}, iterable) {
+    if (typeof options !== 'object' || options === null) {
+      throw new TypeError('INVALID_TYPE', 'options', 'object or iterable', true);
+    }
+    const { sweepFilter = null, sweepInterval = 3600 } = options;
+
+    if (sweepFilter !== null && typeof sweepFilter !== 'function') {
+      throw new TypeError('INVALID_TYPE', 'sweepFunction', 'function');
+    }
+    if (typeof sweepInterval !== 'number') throw new TypeError('INVALID_TYPE', 'sweepInterval', 'number');
+
+    super(iterable);
+
+    /**
+     * A function called every sweep interval that returns a function passed to `sweep`
+     * @type {?SweepFilter}
+     */
+    this.sweepFilter = sweepFilter;
+
+    /**
+     * The id of the interval being used to sweep.
+     * @type {?Timeout}
+     */
+    this.interval =
+      sweepInterval > 0 && sweepFilter
+        ? setInterval(() => {
+            const sweepFn = this.sweepFilter(this);
+            if (sweepFn === null) return;
+            if (typeof sweepFn !== 'function') throw new TypeError('SWEEP_FILTER_RETURN');
+            this.sweep(sweepFn);
+          }, sweepInterval * 1000).unref()
+        : null;
+  }
+
+  /**
+   * Options for generating a filter function based on lifetime
+   * @typedef {Object} LifetimeFilterOptions
+   * @property {number} [lifetime=14400] How long an entry should stay in the collection
+   * before it is considered sweepable
+   * @property {Function} [getComparisonTimestamp=`e => e.createdTimestamp`] A function that takes an entry, key,
+   * and the collection and returns a timestamp to compare against in order to determine the lifetime of the entry.
+   * @property {Function} [excludeFromSweep=`() => false`] A function that takes an entry, key, and the collection
+   * and returns a boolean, `true` when the entry should not be checked for sweepability.
+   */
+
+  /**
+   * Create a sweepFilter function that uses a lifetime to determine sweepability.
+   * @param {LifetimeFilterOptions} [options={}] The options used to generate the filter function
+   * @returns {SweepFilter}
+   */
+  static filterByLifetime({
+    lifetime = 14400,
+    getComparisonTimestamp = e => e?.createdTimestamp,
+    excludeFromSweep = () => false,
+  } = {}) {
+    if (typeof lifetime !== 'number') throw new TypeError('INVALID_TYPE', 'lifetime', 'number');
+    if (typeof getComparisonTimestamp !== 'function') {
+      throw new TypeError('INVALID_TYPE', 'getComparisonTimestamp', 'function');
+    }
+    if (typeof excludeFromSweep !== 'function') {
+      throw new TypeError('INVALID_TYPE', 'excludeFromSweep', 'function');
+    }
+    return () => {
+      if (lifetime <= 0) return null;
+      const lifetimeMs = lifetime * 1000;
+      const now = Date.now();
+      return (entry, key, coll) => {
+        if (excludeFromSweep(entry, key, coll)) {
+          return false;
+        }
+        const comparisonTimestamp = getComparisonTimestamp(entry, key, coll);
+        if (!comparisonTimestamp || typeof comparisonTimestamp !== 'number') return false;
+        return now - comparisonTimestamp > lifetimeMs;
+      };
+    };
+  }
+
+  [_cleanupSymbol]() {
+    clearInterval(this.interval);
+  }
+
+  static get [Symbol.species]() {
+    return Collection;
+  }
+}
+
+module.exports = SweptCollection;