hostbbq-monitoring/server/src/ctrl/http-check-controller.class.ts

import axios, { AxiosError, AxiosRequestConfig } from 'axios';
import moment from 'moment';

import defaults from '../../../common/defaults.module';
import { HttpCheckStatus } from '../../../common/lib/http-check-data.module';
import { Logger } from '../../../common/util/logger.class';

import { Timer } from '../timer.class';
import { Database, ServiceChangedStatus } from './database.class';
import { FCMController } from './fcm-controller.class';

const FCM_TOPIC_SERVICES = 'monitoring-services';

type Subscriber = { id: number; interval: number; conf: HttpCheckConfig };

export class HttpCheckController {
  private subscriptions: Array<Subscriber> = [];
  private db!: Database;

  constructor() {
    this.db = new Database();
    (async () => {
      try {
        await this.db.open();
        const configs = await this.db.getHttpCheckConfigs();

        for (const conf of configs) {
          if (!conf) return;
          if (!conf.active) return;

          await this.scheduleCheck(conf);

          Logger.info('[INFO] Initial HTTP Service Check for', conf.title, '...');
          await this.timerTick(conf);
        }
      } catch (err) {
        Logger.error('[FATAL] Initializing ServerConnector failed:', err);
        Logger.error('[EXITING]');
        process.exit(1);
      }
    })();
  }

  async updateCheck(status: ServiceChangedStatus, conf: HttpCheckConfig) {
    const subscriber = this.subscriptions.find(sub => sub.conf.id === conf.id);

    switch (status) {
      case ServiceChangedStatus.Created:
      case ServiceChangedStatus.Activated:
        await this.scheduleCheck(conf);
        break;
      case ServiceChangedStatus.Deactivated:
        await this.unscheduleCheck(subscriber);
        break;
      case ServiceChangedStatus.Rescheduled:
        await this.rescheduleCheck(conf, subscriber);
        break;
      default:
        break;
    }
  }

  private async scheduleCheck(conf: HttpCheckConfig, log = true) {
    let interval = Number(conf.interval);
    if (Number.isNaN(interval)) interval = defaults.serviceChecks.interval;

    if (log) Logger.info(`[INFO] Starting HTTP Service Check Controller for "${conf.title}" with interval ${interval} seconds ...`);
    const id = Timer.instance.subscribe(interval, async () => await this.timerTick(conf));
    const sub = { id, interval, conf };
    this.subscriptions.push(sub);
    return sub;
  }

  private async rescheduleCheck(conf: HttpCheckConfig, sub?: Subscriber) {
    Logger.info('[INFO] Rescheduling HTTP Service Check for', conf.title);
    await this.unscheduleCheck(sub, false);
    await this.scheduleCheck(conf, false);
  }

  private async unscheduleCheck(sub?: Subscriber, log = true) {
    if (!sub) return;

    if (log) Logger.info('[INFO] Removing HTTP Service Check for', sub.conf.title);
    Timer.instance.unsubscribe(sub.id);
    this.subscriptions = this.subscriptions.filter(s => s.id !== sub.id);
  }

  private async timerTick(conf: HttpCheckConfig) {
    Logger.debug('[DEBUG] TICK', new Date(), JSON.stringify(conf));

    const now = new Date();
    const options: AxiosRequestConfig<any> = {
      timeout: conf.timeout,
      responseType: 'text'
    };
    let success = true;
    try {
      const id = conf.id;
      conf = (await this.db.getHttpCheckConfigByID(conf.serverId ?? 0, id)) as HttpCheckConfig;

      if (!conf) {
        Logger.warn(`[WARN] HealthCheckConfig(${id}) not found in Database but still scheduled in Timer!`);
        return;
      }

      options.timeout = conf.timeout;
      let response = await axios.get(conf.url, options);
      const responseText = new String(response.data).toString();

      for (const check of conf.checks) {
        const reg = new RegExp(check, 'i');
        if (!reg.test(responseText)) {
          Logger.debug(`[DEBUG] Regular expression /${check}/i not found in response`);
          await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.CheckFailed, `Regular expression /${check}/i not found in response`);
          success = false;
        }
      }

      if (success) {
        Logger.debug(`[DEBUG] HTTP Service Check "${conf.title}": OK.`);
        await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.OK, 'OK');
      }
    } catch (err) {
      let log = false;
      success = false;
      if (err instanceof AxiosError) {
        // err.code = 'ECONNREFUSED' | 'ECONNABORTED' | 'ERR_BAD_REQUEST' | 'ERR_BAD_RESPONSE' | ...?
        try {
          if (err.code === 'ECONNABORTED') {
            await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Timeout, err.message);
          } else if (err.code && ['ERR_BAD_REQUEST', 'ERR_BAD_RESPONSE'].includes(err.code)) {
            await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, `${err.response?.status} ${err.response?.statusText}`);
          } else {
            await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, err.message);
          }
        } catch (insertErr) {
          Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
          log = true;
        }
      } else {
        try {
          await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Unknown, new String(err).toString());
        } catch (insertErr) {
          Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
        }
        log = true;
      }
      if (log) Logger.error('[ERROR] HTTP Service Check failed:', err);
    }
    if (!success && conf.notify) {
      try {
        const lastErrors = await this.db.getLastErrors(conf.id, conf.notifyThreshold + 1);
        if (lastErrors.length > conf.notifyThreshold) {
          Logger.debug(`[DEBUG] Sending FCM Notification for`, conf.title);
          const lastCheck = lastErrors[0];
          const lastError = lastCheck.data[0];
          await FCMController.instance.sendNotificationToTopic(FCM_TOPIC_SERVICES, {
            title: `[CRIT] ${conf.title} since ${moment(lastCheck.time).format('HH:mm')}`,
            body:
              `HTTP Check '${conf.title}' has failed over ${conf.notifyThreshold} times in a row\n` +
              `Last error status was: (${lastError.status}) ${lastError.message}`
          });
        }
      } catch (err) {
        Logger.error('[ERROR] Notification failure:', err);
      }
    }
  }

  async close() {
    if (!this.db) return;
    await this.db.close();
  }
}