diff --git a/client/tsconfig.json b/client/tsconfig.json index 4054c4f1..2c7cbdcd 100644 --- a/client/tsconfig.json +++ b/client/tsconfig.json @@ -11,6 +11,7 @@ "../server/log.ts", "../server/config.ts", "../server/client.ts", + "../server/storageCleaner.ts", "../server/clientManager.ts", "../server/identification.ts", "../server/plugins/changelog.ts", diff --git a/defaults/config.js b/defaults/config.js index 51bc090e..a943ca0f 100644 --- a/defaults/config.js +++ b/defaults/config.js @@ -304,6 +304,26 @@ module.exports = { // This value is set to `["sqlite", "text"]` by default. messageStorage: ["sqlite", "text"], + // ### `storagePolicy` + + // When the sqlite storage is in use, control the maximum storage duration. + // A background task will periodically clean up messages older than the limit. + + // The available keys for the `storagePolicy` object are: + // + // - `enabled`: If this is false, the cleaning task is not running. + // - `maxAgeDays`: Maximum age of an entry in days. + // - `deletionPolicy`: Controls what types of messages are being deleted. + // Valid options are: + // - `statusOnly`: Only delete message types which are status related (e.g. away, back, join, parts, mode, ctcp...) + // but keep actual messages from nicks. This keeps the DB size down while retaining "precious" messages. + // - `everything`: Delete everything, including messages from irc nicks + storagePolicy: { + enabled: false, + maxAgeDays: 7, + deletionPolicy: "statusOnly", + }, + // ### `useHexIp` // // When set to `true`, users' IP addresses will be encoded as hex. diff --git a/server/config.ts b/server/config.ts index 543a8135..bad5f522 100644 --- a/server/config.ts +++ b/server/config.ts @@ -76,6 +76,12 @@ type Debug = { raw: boolean; }; +type StoragePolicy = { + enabled: boolean; + maxAgeDays: number; + deletionPolicy: "statusOnly" | "everything"; +}; + export type ConfigType = { public: boolean; host: string | undefined; @@ -97,6 +103,7 @@ export type ConfigType = { defaults: Defaults; lockNetwork: boolean; messageStorage: string[]; + storagePolicy: StoragePolicy; useHexIp: boolean; webirc?: WebIRC; identd: Identd; diff --git a/server/storageCleaner.ts b/server/storageCleaner.ts new file mode 100644 index 00000000..cad486cc --- /dev/null +++ b/server/storageCleaner.ts @@ -0,0 +1,148 @@ +import SqliteMessageStorage from "./plugins/messageStorage/sqlite"; +import {MessageType} from "./models/msg"; +import Config from "./config"; +import {DeletionRequest} from "./plugins/messageStorage/types"; +import log from "./log"; + +const status_types = [ + MessageType.AWAY, + MessageType.BACK, + MessageType.INVITE, + MessageType.JOIN, + MessageType.KICK, + MessageType.MODE, + MessageType.MODE_CHANNEL, + MessageType.MODE_USER, + MessageType.NICK, + MessageType.PART, + MessageType.QUIT, + MessageType.CTCP, // not technically a status, but generally those are only of interest temporarily + MessageType.CTCP_REQUEST, + MessageType.CHGHOST, + MessageType.TOPIC, + MessageType.TOPIC_SET_BY, +]; + +export class StorageCleaner { + db: SqliteMessageStorage; + olderThanDays: number; + messageTypes: MessageType[] | null; + limit: number; + ticker?: ReturnType; + errCount: number; + isStopped: boolean; + + constructor(db: SqliteMessageStorage) { + this.errCount = 0; + this.isStopped = true; + this.db = db; + this.limit = 200; + const policy = Config.values.storagePolicy; + this.olderThanDays = policy.maxAgeDays; + + switch (policy.deletionPolicy) { + case "statusOnly": + this.messageTypes = status_types; + break; + case "everything": + this.messageTypes = null; + break; + default: + // exhaustive switch guard, blows up when user specifies a invalid policy enum + this.messageTypes = assertNoBadPolicy(policy.deletionPolicy); + } + } + + private genDeletionRequest(): DeletionRequest { + return { + limit: this.limit, + messageTypes: this.messageTypes, + olderThanDays: this.olderThanDays, + }; + } + + async runDeletesNoLimit(): Promise { + if (!Config.values.storagePolicy.enabled) { + // this is meant to be used by cli tools, so we guard against this + throw new Error("storage policy is disabled"); + } + + const req = this.genDeletionRequest(); + req.limit = -1; // unlimited + const num_deleted = await this.db.deleteMessages(req); + return num_deleted; + } + + private async runDeletes() { + if (this.isStopped) { + return; + } + + if (!this.db.isEnabled) { + // TODO: remove this once the server is intelligent enough to wait for init + this.schedule(30 * 1000); + return; + } + + const req = this.genDeletionRequest(); + + let num_deleted = 0; + + try { + num_deleted = await this.db.deleteMessages(req); + this.errCount = 0; // reset when it works + } catch (err: any) { + this.errCount++; + log.error("can't clean messages", err.message); + + if (this.errCount === 2) { + log.error("Cleaning failed too many times, will not retry"); + this.stop(); + return; + } + } + + // need to recheck here as the field may have changed since the await + if (this.isStopped) { + return; + } + + if (num_deleted < req.limit) { + this.schedule(5 * 60 * 1000); + } else { + this.schedule(5000); // give others a chance to execute queries + } + } + + private schedule(ms: number) { + const self = this; + + this.ticker = setTimeout(() => { + self.runDeletes().catch((err) => { + log.error("storageCleaner: unexpected failure"); + throw err; + }); + }, ms); + } + + start() { + this.isStopped = false; + this.schedule(0); + } + + stop() { + this.isStopped = true; + + if (!this.ticker) { + return; + } + + clearTimeout(this.ticker); + } +} + +function assertNoBadPolicy(_: never): never { + throw new Error( + `Invalid deletion policy "${Config.values.storagePolicy.deletionPolicy}" in the \`storagePolicy\` object, fix your config.` + ); +}