reducing server data: implemented new all-in-one method

server/src/ctrl/mariadb-database.class.ts

@@ -180,98 +180,43 @@ export class MariaDBDatabase implements DataProvider, HealthCheckDataProvider {
     return result.map(r => ({ time: new Date(r.Timegroup), avg: r.avg, peak: r.peak, max: r.max }));
   }
 
-  public async reduceServerData(serverID: number, type: ServerDataType, sectionMs: number, to: Date, from?: Date) {
-    const select_max = type !== 'cpu';
-    const select_types = select_max ? [type, type, type] : [type, type];
+  public async reduceServerData(intervalSecs: number, to: Date, from?: Date) {
     const select_daterange = from ? [from, to] : [to];
     const result: any[] = await this.db.query(
       `
       SELECT
-        CEIL(UNIX_TIMESTAMP(\`Timestamp\`) * 1000 / ?) * ? as 'Timegroup',
-        MIN(\`ServerDataEntry\`.\`ID\`) as 'entry_min_id',
-        MAX(\`ServerDataEntry\`.\`ID\`) as 'entry_max_id',
-
-        AVG(\`VALUE_AVG\`.\`Value\`) as 'avg',
-        MIN(\`VALUE_AVG\`.\`ID\`) as 'value_avg_min_id',
-        MAX(\`VALUE_AVG\`.\`ID\`) as 'value_avg_max_id',
-
-        MAX(\`VALUE_PEAK\`.\`Value\`) as 'peak',
-        MIN(\`VALUE_PEAK\`.\`ID\`) as 'value_peak_min_id',
-        MAX(\`VALUE_PEAK\`.\`ID\`) as 'value_peak_max_id'${
-          select_max
-            ? `,
-        MAX(\`VALUE_MAX\`.\`Value\`) as 'max',
-        MIN(\`VALUE_MAX\`.\`ID\`) as 'value_max_min_id',
-        MAX(\`VALUE_MAX\`.\`ID\`) as 'value_max_max_id'`
-            : ''
-        }
+        MIN(\`ServerDataEntry\`.\`ID\`) AS 'minEntryID',
+        MAX(\`ServerDataEntry\`.\`ID\`) AS 'maxEntryID',
+        \`ServerDataEntry\`.\`ServerID\` AS 'serverID',
+        FROM_UNIXTIME(CEIL(UNIX_TIMESTAMP(\`ServerDataEntry\`.\`Timestamp\`) / ?) * ?) AS 'timegroup',
+        \`VALUE_AVG\`.\`Type\` AS 'type',
+        MAX(\`VALUE_AVG\`.\`ID\`) AS 'maxAvgID',
+        AVG(\`VALUE_AVG\`.\`Value\`) AS 'avg',
+        MAX(\`VALUE_PEAK\`.\`ID\`) AS 'maxPeakID',
+        MAX(\`VALUE_PEAK\`.\`Value\`) AS 'peak',
+        MAX(\`VALUE_MAX\`.\`ID\`) AS 'maxMaxID',
+        MAX(\`VALUE_MAX\`.\`Value\`) AS 'max'
       FROM \`ServerDataEntry\`
-      JOIN \`ServerDataValue\` AS \`VALUE_AVG\` ON \`ServerDataEntry\`.\`ID\` = \`VALUE_AVG\`.\`EntryID\` AND \`VALUE_AVG\`.\`Type\` = ? AND \`VALUE_AVG\`.\`Key\` = 'avg'
-      JOIN \`ServerDataValue\` AS \`VALUE_PEAK\` ON \`ServerDataEntry\`.\`ID\` = \`VALUE_PEAK\`.\`EntryID\` AND \`VALUE_PEAK\`.\`Type\` = ? AND \`VALUE_PEAK\`.\`Key\` = 'peak'
-      ${
-        select_max
-          ? "JOIN `ServerDataValue` AS `VALUE_MAX` ON `ServerDataEntry`.`ID` = `VALUE_MAX`.`EntryID` AND `VALUE_MAX`.`Type` = ? AND `VALUE_MAX`.`Key` = 'max'"
-          : ''
-      }
-      WHERE \`ServerDataEntry\`.\`ServerID\` = ?
-      AND \`ServerDataEntry\`.\`Timestamp\` ${from ? `BETWEEN ? AND ?` : `< ?`}
-      GROUP BY \`Timegroup\`
-      ORDER BY \`Timegroup\`;`,
-      [sectionMs, sectionMs, ...select_types, serverID, ...select_daterange]
+      LEFT OUTER JOIN \`ServerDataValue\` AS \`VALUE_AVG\` ON \`ServerDataEntry\`.\`ID\` = \`VALUE_AVG\`.\`EntryID\` AND \`VALUE_AVG\`.\`Key\` = 'avg'
+      LEFT OUTER JOIN \`ServerDataValue\` AS \`VALUE_PEAK\` ON \`ServerDataEntry\`.\`ID\` = \`VALUE_PEAK\`.\`EntryID\` AND \`VALUE_AVG\`.\`Type\` = \`VALUE_PEAK\`.\`Type\` AND \`VALUE_PEAK\`.\`Key\` = 'peak'
+      LEFT OUTER JOIN \`ServerDataValue\` AS \`VALUE_MAX\` ON \`ServerDataEntry\`.\`ID\` = \`VALUE_MAX\`.\`EntryID\` AND \`VALUE_PEAK\`.\`Type\` = \`VALUE_MAX\`.\`Type\` AND \`VALUE_MAX\`.\`Key\` = 'max'
+      WHERE ${from ? '`ServerDataEntry`.`Timestamp` BETWEEN ? AND ?' : '`ServerDataEntry`.`Timestamp` < ?'}
+      GROUP BY \`serverID\`, \`timegroup\`, \`type\`
+      ORDER BY \`serverID\`, \`timegroup\`, \`maxAvgID\`;`,
+      [intervalSecs, intervalSecs, ...select_daterange]
     );
 
-    for (const row of result) {
-      // PARENT ENTRIES
-      const queries = [
-        // Avoid ER_DUP_ENTRY on UQ(ServerID, Timestamp):
-        'DELETE FROM `ServerDataEntry` WHERE `ID` != ? AND `ServerID` = ? AND `Timestamp` = ?;',
-        'UPDATE `ServerDataEntry` SET `Timestamp` = ? WHERE `ID` = ?;'
-      ];
-      const params = [...[row.entry_max_id, serverID, new Date(row.Timegroup)], ...[new Date(row.Timegroup), row.entry_max_id]];
-
-      // AVG VALUES
-      queries.push('UPDATE `ServerDataValue` SET `Value` = ? WHERE `ID` = ?;');
-      params.push(row.avg, row.value_avg_max_id);
-      if (row.value_avg_max_id - 1 - row.value_avg_min_id >= 0) {
-        queries.push('DELETE FROM `ServerDataValue` WHERE `ID` >= ? AND `ID` <= ? AND `Type` = ? AND `Key` = ?;');
-        params.push(row.value_avg_min_id, row.value_avg_max_id - 1, type, 'avg');
-      }
-
-      // PEAK VALUES
-      queries.push('UPDATE `ServerDataValue` SET `Value` = ? WHERE `ID` = ?;');
-      params.push(row.peak, row.value_peak_max_id);
-      if (row.value_peak_max_id - 1 - row.value_peak_min_id >= 0) {
-        queries.push('DELETE FROM `ServerDataValue` WHERE `ID` >= ? AND `ID` <= ? AND `Type` = ? AND `Key` = ?;');
-        params.push(row.value_peak_min_id, row.value_peak_max_id - 1, type, 'avg');
-      }
-
-      if (select_max) {
-        // MAX VALUES (?)
-        queries.push('UPDATE `ServerDataValue` SET `Value` = ? WHERE `ID` = ?;');
-        params.push(row.max, row.value_max_max_id);
-        if (row.value_max_max_id - 1 - row.value_max_min_id >= 0) {
-          queries.push('DELETE FROM `ServerDataValue` WHERE `ID` >= ? AND `ID` <= ? AND `Type` = ? AND `Key` = ?;');
-          params.push(row.value_max_min_id, row.value_max_max_id - 1, type, 'avg');
-        }
-      }
-
-      // DELETE PARENT ENTRIES (cascade-delete)
-      if (row.entry_max_id - 1 - row.entry_min_id >= 0) {
-        queries.push('DELETE FROM `ServerDataEntry` WHERE `ID` >= ? AND `ID` <= ?;');
-        params.push(row.entry_min_id, row.entry_max_id - 1);
-      }
-
+    let lastRow: any | undefined = undefined;
+    let queries: string[] = [];
+    let params: any[] = [];
+    const execTransaction = async (row: any) => {
       await this.db.beginTransaction();
       try {
         Logger.debug(
           '[MARIADB] Reducing Server Data:',
           JSON.stringify({
-            server: serverID,
-            type,
-            time: new Date(row.Timegroup),
-            avg: row.avg,
-            peak: row.peak,
-            max: row.max
+            server: row.serverID,
+            time: row.timegroup
           })
         );
         await this.db.query(queries.join('\n'), params);
@@ -280,6 +225,34 @@ export class MariaDBDatabase implements DataProvider, HealthCheckDataProvider {
         Logger.error('[MARIADB] Reducing Server Data failed:', new String(error).toString());
         await this.db.rollback();
       }
+    };
+    for (const row of result) {
+      const { minEntryID, maxEntryID, serverID, timegroup } = row;
+
+      if (lastRow?.maxEntryID !== maxEntryID) {
+        if (queries.length) await execTransaction(row);
+
+        queries = [];
+        params = [];
+      }
+
+      if (!queries.length) {
+        // PARENT ENTRIES
+        queries.push(
+          'DELETE FROM `ServerDataEntry` WHERE `ID` != ? AND `ServerID` = ? AND `Timestamp` = ?;', // Avoid ER_DUP_ENTRY on UQ(ServerID, Timestamp):
+          'UPDATE `ServerDataEntry` SET `Timestamp` = ? WHERE ID = ?;'
+        );
+        params.push(maxEntryID, serverID, timegroup, timegroup, maxEntryID);
+        if (maxEntryID - 1 - minEntryID >= 0) {
+          queries.push('DELETE FROM `ServerDataEntry` WHERE ServerID = ? AND ID >= ? AND ID <= ?;');
+          params.push(serverID, minEntryID, maxEntryID - 1);
+        }
+      }
+      lastRow = row;
+    }
+
+    if (queries.length) {
+      await execTransaction(lastRow);
     }
   }
 
@@ -744,27 +717,17 @@ export class MariaDBDatabase implements DataProvider, HealthCheckDataProvider {
     const nowMinus1M = moment().subtract(1, 'month').toDate();
     const nowMinus2W = moment().subtract(2, 'weeks').toDate();
 
-    const servers = await this.getAllServerConfigs();
-    for (const server of servers) {
-      const dataTypes = await this.getServerDataTypes(server.id);
-
-      for (const dataType of dataTypes) {
-        const loopTypes = dataType.type === 'hdd' ? (dataType.subtypes ?? []).map(s => `hdd:${s.type}`) : [dataType.type];
-        for (const type of loopTypes) {
-          // Reduce Data older than 3Mo to reduced data points like if requested in a 3Mo UI chart
-          let sectionMs = Math.floor(((365.2422 / 12) * 3 * 24 * 60 * 60) / 100) * 1000;
-          await this.reduceServerData(server.id, type, sectionMs, nowMinus3M);
+    // Reduce Data older than 3Mo to reduced data points like if requested in a 3Mo UI chart
+    let intervalSeconds = Math.floor(((365.2422 / 12) * 3 * 24 * 60 * 60) / 100);
+    await this.reduceServerData(intervalSeconds, nowMinus3M);
 
-          // Reduce Data older than 1Mo to reduced data points like if requested in a 1Mo UI chart
-          sectionMs = Math.floor(((365.2422 / 12) * 1 * 24 * 60 * 60) / 100) * 1000;
-          await this.reduceServerData(server.id, type, sectionMs, nowMinus1M, nowMinus3M);
+    // Reduce Data older than 1Mo to reduced data points like if requested in a 1Mo UI chart
+    intervalSeconds = Math.floor(((365.2422 / 12) * 1 * 24 * 60 * 60) / 100);
+    await this.reduceServerData(intervalSeconds, nowMinus1M, nowMinus3M);
 
-          // Reduce Data older than 2We to reduced data points like if requested in a 2We UI chart
-          sectionMs = Math.floor((14 * 24 * 60 * 60) / 100) * 1000;
-          await this.reduceServerData(server.id, type, sectionMs, nowMinus2W, nowMinus1M);
-        }
-      }
-    }
+    // Reduce Data older than 2We to reduced data points like if requested in a 2We UI chart
+    intervalSeconds = Math.floor((14 * 24 * 60 * 60) / 100);
+    await this.reduceServerData(intervalSeconds, nowMinus2W, nowMinus1M);
   }
 
   public async close(): Promise<void> {

server/src/index.ts

@@ -33,8 +33,8 @@ let connectionPool: Pool;
   };
 
   Timer.instance.start();
-  //Timer.instance.subscribe(7 * 24 * 60 * 60, db.cleanupData.bind(db));
-  //db.cleanupData();
+  Timer.instance.subscribe(7 * 24 * 60 * 60, db.cleanupData.bind(db));
+  db.cleanupData();
 
   new Webserver(Number(process.env.WEB_PORT ?? '80'), pool);
 })();