import axios, { AxiosError, AxiosRequestConfig } from 'axios'; import moment from 'moment'; import defaults from '../../../common/defaults.module'; import { HttpCheckStatus } from '../../../common/lib/http-check-data.module'; import { Logger } from '../../../common/util/logger.class'; import { Timer } from '../timer.class'; import { Database, ServiceChangedStatus } from './database.class'; import { FCMController } from './fcm-controller.class'; const FCM_TOPIC_SERVICES = 'monitoring-services'; type Subscriber = { id: number; interval: number; conf: HttpCheckConfig }; export class HttpCheckController { private subscriptions: Array = []; private db!: Database; constructor() { this.db = new Database(); (async () => { try { await this.db.open(); const configs = await this.db.getHttpCheckConfigs(); for (const conf of configs) { if (!conf) return; if (!conf.active) return; await this.scheduleCheck(conf); Logger.info('[INFO] Initial HTTP Service Check for', conf.title, '...'); await this.timerTick(conf); } } catch (err) { Logger.error('[FATAL] Initializing ServerConnector failed:', err); Logger.error('[EXITING]'); process.exit(1); } })(); } async updateCheck(status: ServiceChangedStatus, conf: HttpCheckConfig) { const subscriber = this.subscriptions.find(sub => sub.conf.id === conf.id); switch (status) { case ServiceChangedStatus.Created: case ServiceChangedStatus.Activated: await this.scheduleCheck(conf); break; case ServiceChangedStatus.Deactivated: await this.unscheduleCheck(subscriber); break; case ServiceChangedStatus.Rescheduled: await this.rescheduleCheck(conf, subscriber); break; default: break; } } private async scheduleCheck(conf: HttpCheckConfig, log = true) { let interval = Number(conf.interval); if (Number.isNaN(interval)) interval = defaults.serviceChecks.interval; if (log) Logger.info(`[INFO] Starting HTTP Service Check Controller for "${conf.title}" with interval ${interval} seconds ...`); const id = Timer.instance.subscribe(interval, async () => await this.timerTick(conf)); const sub = { id, interval, conf }; this.subscriptions.push(sub); return sub; } private async rescheduleCheck(conf: HttpCheckConfig, sub?: Subscriber) { Logger.info('[INFO] Rescheduling HTTP Service Check for', conf.title); await this.unscheduleCheck(sub, false); await this.scheduleCheck(conf, false); } private async unscheduleCheck(sub?: Subscriber, log = true) { if (!sub) return; if (log) Logger.info('[INFO] Removing HTTP Service Check for', sub.conf.title); Timer.instance.unsubscribe(sub.id); this.subscriptions = this.subscriptions.filter(s => s.id !== sub.id); } private async timerTick(conf: HttpCheckConfig) { Logger.debug('[DEBUG] TICK', new Date(), JSON.stringify(conf)); const now = new Date(); const options: AxiosRequestConfig = { timeout: conf.timeout, responseType: 'text' }; let success = true; try { const id = conf.id; conf = (await this.db.getHttpCheckConfigByID(conf.serverId ?? 0, id)) as HttpCheckConfig; if (!conf) { Logger.warn(`[WARN] HealthCheckConfig(${id}) not found in Database but still scheduled in Timer!`); return; } options.timeout = conf.timeout; let response = await axios.get(conf.url, options); const responseText = new String(response.data).toString(); for (const check of conf.checks) { const reg = new RegExp(check, 'i'); if (!reg.test(responseText)) { Logger.debug(`[DEBUG] Regular expression /${check}/i not found in response`); await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.CheckFailed, `Regular expression /${check}/i not found in response`); success = false; } } if (success) { Logger.debug(`[DEBUG] HTTP Service Check "${conf.title}": OK.`); await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.OK, 'OK'); } } catch (err) { let log = false; success = false; if (err instanceof AxiosError) { // err.code = 'ECONNREFUSED' | 'ECONNABORTED' | 'ERR_BAD_REQUEST' | 'ERR_BAD_RESPONSE' | ...? try { if (err.code === 'ECONNABORTED') { await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Timeout, err.message); } else if (err.code && ['ERR_BAD_REQUEST', 'ERR_BAD_RESPONSE'].includes(err.code)) { await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, `${err.response?.status} ${err.response?.statusText}`); } else { await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, err.message); } } catch (insertErr) { Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr); log = true; } } else { try { await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Unknown, new String(err).toString()); } catch (insertErr) { Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr); } log = true; } if (log) Logger.error('[ERROR] HTTP Service Check failed:', err); } if (!success && conf.notify) { try { const lastErrors = await this.db.getLastErrors(conf.id, conf.notifyThreshold + 1); if (lastErrors.length > conf.notifyThreshold) { Logger.debug(`[DEBUG] Sending FCM Notification for`, conf.title); const lastCheck = lastErrors[0]; const lastError = lastCheck.data[0]; await FCMController.instance.sendNotificationToTopic(FCM_TOPIC_SERVICES, { title: `[CRIT] ${conf.title} since ${moment(lastCheck.time).format('HH:mm')}`, body: `HTTP Check '${conf.title}' has failed over ${conf.notifyThreshold} times in a row\n` + `Last error status was: (${lastError.status}) ${lastError.message}` }); } } catch (err) { Logger.error('[ERROR] Notification failure:', err); } } } async close() { if (!this.db) return; await this.db.close(); } }