Daemon: CPU usage problems on Hetzner/arm64: changed to running "top" via durable Pseudo-TTY, reading data from TTY line-refreshes

daemon/install/install.sh

@@ -24,6 +24,27 @@ if [ -z "$EXC_NODE" ]; then
   exit 1
 fi
 
+EXC_PY3="$(/usr/bin/which python3)"
+if [ -z "$EXC_PY3" ]; then
+  echo "[ERROR] Missing required install dependency 'python3'." >&2
+  echo "Please install using \"apt install python3\"." >&2
+  exit 1
+fi
+
+EXC_MAKE="$(/usr/bin/which make)"
+if [ -z "$EXC_MAKE" ]; then
+  echo "[ERROR] Missing required install dependency 'make'." >&2
+  echo "Please install using \"apt install make\"." >&2
+  exit 1
+fi
+
+LIB_BUILD_ESSTL="$(apt list --installed 2>/dev/null | grep build-essential)"
+if [ -z "$LIB_BUILD_ESSTL" ]; then
+  echo "[ERROR] Missing required install dependency 'build-essential'." >&2
+  echo "Please install using \"apt install build-essential\"." >&2
+  exit 1
+fi
+
 # exit on error exit codes
 set -e
 

daemon/package.json

@@ -18,6 +18,7 @@
     "dotenv": "^16.0.2",
     "express": "^4.18.1",
     "moment": "^2.29.4",
+    "node-pty": "^1.0.0",
     "node-utils": "git+https://gogs.hostbbq.com/hostbbq/node-utils.git#1.0.6"
   }
 }

daemon/src/collector.class.ts

@@ -3,15 +3,20 @@ import fsp from 'fs/promises';
 import moment from 'moment';
 import { exec } from 'node-utils/shell';
 import path from 'path';
+import { format } from 'util';
 
 import { HttpStatusException } from '../../common/lib/http-status.exception';
 import { Logger } from '../../common/util/logger.class';
 
+import { MemoryUsage, TopTTY } from './top-tty';
+
 const DATA_DIR = process.env.DATA_DIR || 'data';
 const DATA_BUFFER_FILE = path.resolve(DATA_DIR, 'buffer.csv');
 const DATA_BUFFER_REMAINS = path.resolve(DATA_DIR, 'remains.csv');
 const DATA_REDUCED_FILE = path.resolve(DATA_DIR, 'reduced.csv');
 const TIMESTAMP_FORMAT = `YYYY-MM-DD[T]HH:mm:ss.SSSZZ`;
+const WRITE_BUFFER_INTERVAL = 500; // [ms]
+const READ_HDD_INTERVAL = 300; // [s]
 const REDUCE_INTERVAL_MINUTES = 5;
 const REDUCE_GROUP_MINUTES = 1;
 
@@ -42,6 +47,13 @@ const CSV_COLS = {
 export class Collector {
   private intervalHdl?: NodeJS.Timer;
 
+  private currentRamUsage: MemoryUsage = { unit: 'B', avail: 0, used: 0 };
+  private currentCpuUsage: number = 0;
+  private currentHddUsage: { time: number; mounts: Array<{ mount: string; stats: string }> } = {
+    time: 0,
+    mounts: []
+  };
+
   constructor() {
     (async () => {
       try {
@@ -60,26 +72,44 @@ export class Collector {
   }
 
   public startLoop() {
-    this.intervalHdl = setInterval(this.loop.bind(this), 500);
+    this.intervalHdl = setInterval(this.loop.bind(this), WRITE_BUFFER_INTERVAL);
+
+    const tty = new TopTTY();
+    tty.subscribe.cpu(usage => (this.currentCpuUsage = usage));
+    tty.subscribe.ram(usage => (this.currentRamUsage = usage));
+    tty.runloop();
   }
 
   private async loop() {
     try {
       const now = moment();
       const time = now.format(TIMESTAMP_FORMAT);
-      const cpu = (await exec(`./cpu.sh`)).trim();
-      const ram = (await exec(`./ram.sh`)).trim();
 
-      const hdd: string[] = [];
-      for (const mount of MONITOR_MOUNTS) {
-        try {
-          const stats = (await exec(`./hdd.sh "${mount}"`)).trim();
-          if (stats?.length) hdd.push(`${mount} ${stats}`);
-        } catch (err) {
-          Logger.warn('[WARN] Error while getting space usage of mount', mount, ':', err);
+      if (now.unix() - this.currentHddUsage.time > READ_HDD_INTERVAL) {
+        this.currentHddUsage.time = now.unix();
+        this.currentHddUsage.mounts = [];
+
+        for (const mount of MONITOR_MOUNTS) {
+          try {
+            const stats = (await exec(`./hdd.sh "${mount}"`)).trim();
+            if (stats?.length) this.currentHddUsage.mounts.push({ mount, stats });
+          } catch (err) {
+            Logger.warn('[WARN] Error while getting space usage of mount', mount, ':', err);
+          }
         }
       }
 
+      const hdd: string[] = this.currentHddUsage.mounts.map(({ mount, stats }) => `${mount} ${stats}`);
+
+      if (!this.currentRamUsage.avail) return;
+
+      const ram = format(
+        '%s/%s %s',
+        (Math.round(this.currentRamUsage.used * 100) / 100).toFixed(2),
+        (Math.round(this.currentRamUsage.avail * 100) / 100).toFixed(2),
+        this.currentRamUsage.unit
+      );
+      const cpu = (Math.round(this.currentCpuUsage * 100) / 100).toFixed(2);
       const data = `${time};${cpu};${ram}${hdd.length ? `;${hdd.join(';')}` : ''}\n`;
 
       // Time to reduce buffer?

daemon/src/index.ts

@@ -13,7 +13,7 @@ process.on('SIGABRT', exitGracefully);
 process.on('SIGQUIT', exitGracefully);
 process.on('SIGTERM', exitGracefully);
 
-Logger.info('[INFO] Starting Monitoring Deamon, pid:', process.pid);
+Logger.info('[INFO] Starting Monitoring Daemon, pid:', process.pid);
 
 const collector = new Collector();
 new Webserver(Number(process.env.WEB_PORT ?? '80'), collector);

daemon/src/top-tty.ts

@@ -0,0 +1,81 @@
+import { IPtyForkOptions, IWindowsPtyForkOptions, spawn } from 'node-pty';
+
+import { Logger, LogLevel } from '../../common/util/logger.class';
+
+const LOG_LEVEL: LogLevel = (process.env.LOG_LEVEL as LogLevel) || 'INFO';
+Logger.logLevel = LOG_LEVEL;
+
+export type MemoryUsage = { used: number; avail: number; unit: string };
+
+export class TopTTY {
+  private cpuSubscriber?: (usage: number) => void;
+  private ramSubscriber?: (usage: MemoryUsage) => void;
+
+  public subscribe = {
+    cpu: (subscriber: (usage: number) => void) => {
+      this.cpuSubscriber = subscriber;
+    },
+    ram: (subscriber: (usage: MemoryUsage) => void) => {
+      this.ramSubscriber = subscriber;
+    }
+  };
+
+  public async runloop() {
+    while (true) {
+      try {
+        await execTTY('top -d 0.5', { cols: 100, rows: 10 }, (line: string) => {
+          if (line.includes('Cpu')) {
+            line = line.replace(/\u001b(\[|\()([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGKB]/g, ''); // get rid of tty color format codes
+            line = line.replace(/:/g, ' ');
+            const tokens = line.split(/\s+|,/g).filter(v => !!v); // split by whitespace and colon, omit emties
+
+            const usage = Number(tokens[tokens.length - 14]) + Number(tokens[tokens.length - 16]);
+            if (this.cpuSubscriber) this.cpuSubscriber(usage);
+          } else if (/^([KMGPT]i)?B\s+Mem\s*:/.test(line)) {
+            line = line.replace(/\u001b(\[|\()([0-9]{1,3}(;[0-9]{1,2};?)?)?[mGKB]/g, ''); // get rid of tty color format codes
+            line = line.replace(/:/g, ' ');
+            const tokens = line.split(/\s+/g).filter(v => !!v); // split by whitespace, omit emties
+            const unit = tokens[0];
+            const avail = Number(tokens[2]);
+            const used = Number(tokens[6]);
+
+            if (this.ramSubscriber) this.ramSubscriber({ unit, avail, used });
+          }
+        });
+      } catch (err) {
+        Logger.error('[ERROR] top runloop crashed:', err);
+        Logger.info('[INFO] Restaring top runloop...');
+      }
+      await new Promise<void>(res => setTimeout(res, 1000));
+    }
+  }
+}
+
+function execTTY(command: string, options: IPtyForkOptions | IWindowsPtyForkOptions, stdout?: (...args: any[]) => void) {
+  return new Promise<void>((resolve, reject) => {
+    try {
+      let stdoutbuf = '';
+
+      // SPAWN PTY PROCESS
+      const p = spawn(command.split(' ')[0], command.split(' ').slice(1), options);
+
+      // PIPE STDOUT
+      if (typeof stdout === 'function') {
+        p.onData(chunk => {
+          stdoutbuf += chunk;
+          let i = -1;
+          while ((i = stdoutbuf.indexOf('\n')) >= 0) {
+            const line = stdoutbuf.substring(0, i);
+            stdoutbuf = stdoutbuf.substring(i + 1);
+            if (typeof stdout === 'function') {
+              stdout(line);
+            }
+          }
+        });
+        p.onExit(() => resolve());
+      }
+    } catch (err) {
+      reject(err);
+    }
+  });
+}