Server: Implemented HttpCheckController; Updating check scheduler on relevant changes in /services API

common/defaults.module.ts

@@ -0,0 +1,14 @@
+export const serverSync = {
+  interval: 300
+};
+
+export const serviceChecks = {
+  active: true,
+  httpTimeout: 10000,
+  interval: 300
+};
+
+export default {
+  serverSync,
+  serviceChecks
+};

common/interfaces/service-config.interface.ts

@@ -7,11 +7,11 @@ export interface ServiceConfig {
   params: Array<{
     type: ParamType;
     key: string;
-    value: string | string[] | number;
+    value: string | string[] | number | boolean;
   }>;
 }
 
-export const ParamTypes = ['text', 'number', 'regexp'] as const;
+export const ParamTypes = ['text', 'number', 'boolean', 'regexp'] as const;
 export type ParamType = typeof ParamTypes[number];
 
 export function validateParamType(input: string): ParamType {

common/lib/http-check-data.module.ts

@@ -0,0 +1,15 @@
+export enum HttpCheckStatus {
+  OK = 0,
+  Unknown = 1,
+  RequestFailed = 2,
+  Timeout = 3,
+  CheckFailed = 4
+}
+
+export type HttpCheckData = {
+  id: number;
+  configId: number;
+  time: Date;
+  status: HttpCheckStatus;
+  message: string;
+};

common/types/http-check-config.d.ts

@@ -4,6 +4,8 @@ type HttpCheckConfig = {
   title: string;
   type: 'http';
   url: string;
+  active: boolean;
   interval: number;
+  timeout?: number;
   checks: string[];
 };

server/docs/Monitoring.postman_collection.json

@@ -127,7 +127,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "{\r\n    \"title\": \"kaytobee.de\",\r\n    \"type\": \"http\",\r\n    \"url\": \"https://www.kaytobee.de\",\r\n    \"interval\": 50,\r\n    \"checks\": [\r\n        \"kaytobee\",\r\n        \"kaytobee\\\\.(com|de|fr)\",\r\n        \"reg|exp\"\r\n    ]\r\n}",
+							"raw": "{\r\n    \"title\": \"kaytobee.de\",\r\n    \"type\": \"http\",\r\n    \"url\": \"https://www.kaytobee.de\",\r\n    \"interval\": 50,\r\n    \"active\": true,\r\n    \"timeout\": 10000,\r\n    \"checks\": [\r\n        \"kaytobee\",\r\n        \"kaytobee\\\\.(com|de|fr)\",\r\n        \"reg|exp\"\r\n    ]\r\n}",
 							"options": {
 								"raw": {
 									"language": "json"
@@ -153,12 +153,12 @@
 					"response": []
 				},
 				{
-					"name": "/services/{:serverId} - Delete a Service Check",
+					"name": "/services/{:serverId}/{:serviceId} - Delete a Service Check",
 					"request": {
 						"method": "DELETE",
 						"header": [],
 						"url": {
-							"raw": "http://10.8.0.1:8880/services/3/9",
+							"raw": "http://10.8.0.1:8880/services/2/1",
 							"protocol": "http",
 							"host": [
 								"10",
@@ -169,8 +169,8 @@
 							"port": "8880",
 							"path": [
 								"services",
-								"3",
-								"9"
+								"2",
+								"1"
 							]
 						}
 					},

server/src/ctrl/controller-pool.interface.ts

@@ -1,7 +1,9 @@
 import { Database } from './database.class';
+import { HttpCheckController } from './http-check-controller.class';
 import { ServerConnector } from './server-connector.class';
 
 export interface ControllerPool {
   db: Database;
   serverConnector: ServerConnector;
+  httpChecks: HttpCheckController;
 }

server/src/ctrl/database.class.ts

@@ -4,13 +4,23 @@ import moment from 'moment';
 import path from 'path';
 import { Database as SQLiteDB, OPEN_CREATE, OPEN_READWRITE } from 'sqlite3';
 
+import defaults from '../../../common/defaults.module';
 import { ServiceConfig, validateParamType } from '../../../common/interfaces/service-config.interface';
+import { HttpCheckData, HttpCheckStatus } from '../../../common/lib/http-check-data.module';
 import { Logger } from '../../../common/util/logger.class';
 
 import { ValidationException } from '../lib/validation-exception.class';
 import { DBMigration } from './db-migration.class';
 import { SQLiteController } from './sqlite-controller.base';
 
+export enum ServiceChangedStatus {
+  None,
+  Created,
+  Activated,
+  Deactivated,
+  Rescheduled
+}
+
 export class Database extends SQLiteController {
   public set onError(listener: (error: any) => void) {
     this._onError = listener;
@@ -235,7 +245,7 @@ export class Database extends SQLiteController {
     return result.rows.map(r => ({ time: new Date(r.Timegroup), avg: r.avg, peak: r.peak, max: r.max }));
   }
 
-  private async getHealthCheckConfigs(serverID: number) {
+  private async getHealthCheckConfigs(serverID?: number, type = 'http') {
     const res = await this.stmt(
       `SELECT 
         HealthCheckConfig.*,
@@ -244,15 +254,16 @@ export class Database extends SQLiteController {
         HealthCheckParams.Value as '_ParamValue'
         FROM HealthCheckConfig
         LEFT OUTER JOIN HealthCheckParams ON HealthCheckConfig.ID = HealthCheckParams.ConfigID
-        WHERE HealthCheckConfig.ServerID = ?
+        WHERE HealthCheckConfig.Type = ?
+        ${!!serverID ? 'AND HealthCheckConfig.ServerID = ?' : ''}
         ORDER BY HealthCheckConfig.Title, _ParamType, _ParamKey`,
-      [serverID]
+      [type, serverID]
     );
 
     return this.configFromResultRows(res.rows);
   }
 
-  public async getHttpCheckConfigs(serverID: number) {
+  public async getHttpCheckConfigs(serverID?: number) {
     return (await this.getHealthCheckConfigs(serverID)).map(this.httpCheckConfigFrom);
   }
 
@@ -289,6 +300,7 @@ export class Database extends SQLiteController {
     if (validationErrors) throw new ValidationException('Validation of HttpCheckConfig object failed', validationErrors);
 
     conf.serverId = serverID;
+    let status = ServiceChangedStatus.None;
 
     const oldConf = await this.getHttpCheckConfigByID(serverID, conf.id);
     await this.beginTransaction();
@@ -300,8 +312,16 @@ export class Database extends SQLiteController {
         }
 
         let updValues: any[][] = [];
-        if (oldConf.interval !== conf.interval) updValues.push([conf.interval, conf.id, 'interval']);
         if (oldConf.url !== conf.url) updValues.push([conf.url, conf.id, 'url']);
+        if (oldConf.interval !== conf.interval) {
+          updValues.push([conf.interval, conf.id, 'interval']);
+          status = ServiceChangedStatus.Rescheduled;
+        }
+        if (oldConf.timeout !== conf.timeout) updValues.push([conf.timeout ?? defaults.serviceChecks.httpTimeout, conf.id, 'timeout']);
+        if (oldConf.active !== conf.active) {
+          updValues.push([conf.active ?? defaults.serviceChecks.active ? 1 : 0, conf.id, 'active']);
+          status = conf.active ?? defaults.serviceChecks.active ? ServiceChangedStatus.Activated : ServiceChangedStatus.Deactivated;
+        }
         if (updValues.length) {
           for (const data of updValues) {
             await this.run(`UPDATE HealthCheckParams SET Value = ? WHERE ConfigID = ? AND Key = ?;`, data);
@@ -339,22 +359,29 @@ export class Database extends SQLiteController {
         // INSERT
         const res = await this.run('INSERT INTO HealthCheckConfig(ServerID, Type, Title) VALUES(?, ?, ?);', [serverID, 'http', conf.title]);
         conf.id = res.lastID;
+        if (conf.active ?? defaults.serviceChecks.active) {
+          status = ServiceChangedStatus.Created;
+        }
 
         const insCheckValues = conf.checks.map(c => [res.lastID, 'regexp', 'check', c]);
         await this.run(
           `INSERT INTO HealthCheckParams(ConfigID, Type, Key, Value) VALUES
           (?, ?, ?, ?),
+          (?, ?, ?, ?),
+          (?, ?, ?, ?),
           (?, ?, ?, ?)${conf.checks.length ? `,${insCheckValues.map(() => '(?, ?, ?, ?)').join(',')}` : ''}`,
           [
             ...[res.lastID, 'text', 'url', conf.url],
+            ...[res.lastID, 'boolean', 'active', conf.active ?? defaults.serviceChecks.active ? 1 : 0],
             ...[res.lastID, 'number', 'interval', conf.interval],
+            ...[res.lastID, 'number', 'timeout', conf.timeout ?? defaults.serviceChecks.httpTimeout],
             ...conf.checks.reduce((ret, check) => [...ret, res.lastID, 'regexp', 'check', check], [] as any[])
           ]
         );
       }
 
       await this.commit();
-      return conf;
+      return { status, result: conf };
     } catch (err) {
       await this.rollback();
       throw err;
@@ -369,6 +396,23 @@ export class Database extends SQLiteController {
     return true;
   }
 
+  async insertHealthCheckData(confID: number, time: Date, status: HttpCheckStatus, message: string) {
+    const res = await this.run('INSERT INTO HealthCheckDataEntry(ConfigID, Timestamp, Status, Message) VALUES(?, ?, ?, ?);', [
+      confID,
+      time.getTime(),
+      status,
+      message
+    ]);
+
+    return {
+      id: res.lastID,
+      configId: confID,
+      time,
+      status,
+      message
+    } as HttpCheckData;
+  }
+
   private configFromResultRows(rows: any[]) {
     return rows.reduce((res: ServiceConfig[], line, i) => {
       const configID = line['ID'];
@@ -378,6 +422,7 @@ export class Database extends SQLiteController {
           id: configID,
           title: line['Title'],
           type: line['Type'],
+          serverId: line['ServerID'],
           params: []
         };
         res.push(config);
@@ -404,7 +449,7 @@ export class Database extends SQLiteController {
           config.params.push({
             type,
             key,
-            value: type === 'number' ? Number(line['_ParamValue']) : line['_ParamValue']
+            value: type === 'number' ? Number(line['_ParamValue']) : type === 'boolean' ? Boolean(Number(line['_ParamValue'])) : line['_ParamValue']
           });
         }
       }
@@ -417,13 +462,16 @@ export class Database extends SQLiteController {
     if (!hcConf) return null;
     const params = {
       url: hcConf.params?.find(p => p.key === 'url')?.value as string,
+      active: (hcConf.params?.find(p => p.key === 'active')?.value as boolean) ?? defaults.serviceChecks.active,
       interval: hcConf.params?.find(p => p.key === 'interval')?.value as number,
+      timeout: (hcConf.params?.find(p => p.key === 'timeout')?.value as number) ?? defaults.serviceChecks.httpTimeout,
       checks: hcConf.params?.reduce((res, p) => (p.key === 'check' && Array.isArray(p.value) ? [...res, ...p.value] : res), [] as string[])
     };
     return {
       id: hcConf.id,
       title: hcConf.title,
       type: hcConf.type,
+      serverId: hcConf.serverId,
       ...params
     };
   }
@@ -440,4 +488,8 @@ export class Database extends SQLiteController {
 
     return Object.keys(errors).length ? errors : null;
   }
+
+  async close() {
+    return new Promise<void>((res, rej) => this.db.close(err => (err ? rej(err) : res())));
+  }
 }

server/src/ctrl/http-check-controller.class.ts

@@ -0,0 +1,143 @@
+import axios, { AxiosError, AxiosRequestConfig } from 'axios';
+
+import defaults from '../../../common/defaults.module';
+import { HttpCheckStatus } from '../../../common/lib/http-check-data.module';
+import { Logger } from '../../../common/util/logger.class';
+
+import { Database, ServiceChangedStatus } from './database.class';
+import { Timer } from '../timer.class';
+
+type Subscriber = { id: number; interval: number; conf: HttpCheckConfig; db: Database };
+
+export class HttpCheckController {
+  private subscriptions: Array<Subscriber> = [];
+
+  constructor(db: Database) {
+    (async () => {
+      try {
+        const configs = await db.getHttpCheckConfigs();
+
+        configs.forEach(async conf => {
+          if (!conf) return;
+          if (!conf.active) return;
+
+          const sub = await this.scheduleCheck(conf);
+
+          process.nextTick(async () => {
+            Logger.info('[INFO] Initial HTTP Service Check for', conf.title, '...');
+            await this.timerTick(conf, sub.db);
+          });
+        });
+      } catch (err) {
+        Logger.error('[FATAL] Initializing ServerConnector failed:', err);
+        Logger.error('[EXITING]');
+        process.exit(1);
+      }
+    })();
+  }
+
+  async updateCheck(status: ServiceChangedStatus, conf: HttpCheckConfig) {
+    const subscriber = this.subscriptions.find(sub => sub.conf.id === conf.id);
+
+    switch (status) {
+      case ServiceChangedStatus.Created:
+      case ServiceChangedStatus.Activated:
+        await this.scheduleCheck(conf);
+        break;
+      case ServiceChangedStatus.Deactivated:
+        await this.unscheduleCheck(subscriber);
+        break;
+      case ServiceChangedStatus.Rescheduled:
+        await this.unscheduleCheck(subscriber);
+        await this.scheduleCheck(conf);
+        break;
+      default:
+        break;
+    }
+  }
+
+  private async scheduleCheck(conf: HttpCheckConfig) {
+    const serverDB = new Database();
+    await serverDB.open();
+
+    let interval = Number(conf.interval);
+    if (Number.isNaN(interval)) interval = defaults.serviceChecks.interval;
+
+    Logger.info(`[INFO] Starting HTTP Service Check Controller for "${conf.title}" with interval ${interval} seconds ...`);
+    const id = Timer.instance.subscribe(interval, async () => await this.timerTick(conf, serverDB));
+    const sub = { id, interval, conf, db: serverDB };
+    this.subscriptions.push(sub);
+    return sub;
+  }
+
+  private async unscheduleCheck(sub?: Subscriber) {
+    if (!sub) return;
+
+    Logger.info('[INFO] Removing HTTP Service Check for', sub.conf.title);
+    Timer.instance.unsubscribe(sub.id);
+    await sub.db.close();
+    this.subscriptions = this.subscriptions.filter(s => s.id !== sub.id);
+  }
+
+  private async timerTick(conf: HttpCheckConfig, db: Database) {
+    Logger.debug('[DEBUG] TICK', new Date(), JSON.stringify(conf));
+
+    const now = new Date();
+    const options: AxiosRequestConfig<any> = {
+      timeout: conf.timeout,
+      responseType: 'text'
+    };
+    try {
+      const current = await db.getHttpCheckConfigByID(conf.serverId ?? 0, conf.id);
+
+      if (!current) {
+        Logger.warn(`[WARN] HealthCheckConfig(${conf.id}) not found in Database but still scheduled in Timer!`);
+        return;
+      }
+
+      options.timeout = current.timeout;
+      let response = await axios.get(current.url, options);
+      const responseText = new String(response.data).toString();
+
+      let success = true;
+      for (const check of current.checks) {
+        const reg = new RegExp(check, 'i');
+        if (!reg.test(responseText)) {
+          Logger.debug(`[DEBUG] Regular expression /${check}/i not found in response`);
+          await db.insertHealthCheckData(current.id, now, HttpCheckStatus.CheckFailed, `Regular expression /${check}/i not found in response`);
+          success = false;
+        }
+      }
+
+      if (success) {
+        Logger.debug(`[DEBUG] HTTP Service Check "${current.title}": OK.`);
+        await db.insertHealthCheckData(current.id, now, HttpCheckStatus.OK, 'OK');
+      }
+    } catch (err) {
+      let log = false;
+      if (err instanceof AxiosError) {
+        // err.code = 'ECONNREFUSED' | 'ECONNABORTED' | 'ERR_BAD_REQUEST' | 'ERR_BAD_RESPONSE' | ...?
+        try {
+          if (err.code === 'ECONNABORTED') {
+            await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Timeout, err.message);
+          } else if (err.code && ['ERR_BAD_REQUEST', 'ERR_BAD_RESPONSE'].includes(err.code)) {
+            await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, `${err.response?.status} ${err.response?.statusText}`);
+          } else {
+            await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, err.message);
+          }
+        } catch (insertErr) {
+          Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
+          log = true;
+        }
+      } else {
+        try {
+          await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Unknown, new String(err).toString());
+        } catch (insertErr) {
+          Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
+        }
+        log = true;
+      }
+      if (log) Logger.error('[ERROR] HTTP Service Check failed:', err);
+    }
+  }
+}

server/src/ctrl/server-connector.class.ts

@@ -1,6 +1,8 @@
 import axios from 'axios';
 
+import defaults from '../../../common/defaults.module';
 import { Logger } from '../../../common/util/logger.class';
+
 import { Database } from './database.class';
 import { Timer } from '../timer.class';
 
@@ -16,7 +18,9 @@ export class ServerConnector {
           const serverDB = new Database();
           await serverDB.open();
 
-          const interval = Number(server.config['syncInterval'] ?? '300');
+          let interval = Number(server.config['syncInterval']);
+          if (Number.isNaN(interval)) interval = defaults.serverSync.interval;
+
           Logger.info('[INFO] Starting Server Sync Connector for', server.title, 'with interval', interval, 'seconds ...');
           const id = Timer.instance.subscribe(interval, async () => await this.timerTick(server, serverDB));
           this.subscriptions.push({ id, interval, server });

server/src/index.ts

@@ -4,6 +4,7 @@ import { Logger, LogLevel } from '../../common/util/logger.class';
 
 import { ControllerPool } from './ctrl/controller-pool.interface';
 import { Database } from './ctrl/database.class';
+import { HttpCheckController } from './ctrl/http-check-controller.class';
 import { ServerConnector } from './ctrl/server-connector.class';
 import { Webserver } from './webserver.class';
 import { Timer } from './timer.class';
@@ -19,7 +20,8 @@ Logger.logLevel = LOG_LEVEL;
 
   const pool: ControllerPool = {
     db,
-    serverConnector: new ServerConnector(db)
+    serverConnector: new ServerConnector(db),
+    httpChecks: new HttpCheckController(db)
   };
 
   Timer.instance.start();

server/src/timer.class.ts

@@ -62,7 +62,7 @@ export class Timer {
   }
 
   public unsubscribe(id: number) {
-    if (typeof this.subscribers[id] === 'function') {
+    if (typeof this.subscribers[id] !== 'undefined') {
       delete this.subscribers[id];
     }
   }

server/src/webhdl/services-api-handler.class.ts

@@ -3,6 +3,7 @@ import { RouterOptions, json } from 'express';
 import { HttpStatusException } from '../../../common/lib/http-status.exception';
 
 import { ControllerPool } from '../ctrl/controller-pool.interface';
+import { ServiceChangedStatus } from '../ctrl/database.class';
 import { WebHandler } from './web-handler.base';
 
 export class ServicesAPIHandler extends WebHandler {
@@ -27,9 +28,13 @@ export class ServicesAPIHandler extends WebHandler {
       try {
         const serverID = this.validateNumber(req.params.serverID, 'server id');
 
-        const service = await this.ctrlPool.db.saveHttpCheckConfig(serverID, req.body);
+        const result = await this.ctrlPool.db.saveHttpCheckConfig(serverID, req.body);
 
-        res.send(service);
+        if (result.status !== ServiceChangedStatus.None) {
+          await this.ctrlPool.httpChecks.updateCheck(result.status, result.result);
+        }
+
+        res.send(result.result);
       } catch (err) {
         next(err);
       }
@@ -42,6 +47,10 @@ export class ServicesAPIHandler extends WebHandler {
 
         const deleted = await this.ctrlPool.db.deleteHealthCheckConfig(serverID, serviceID);
 
+        if (deleted) {
+          await this.ctrlPool.httpChecks.updateCheck(ServiceChangedStatus.Deactivated, { id: serviceID } as HttpCheckConfig);
+        }
+
         res.send({ deleted });
       } catch (err) {
         next(err);