|
@@ -8,8 +8,10 @@ import { Logger } from '../../../common/util/logger.class';
|
|
|
import { Timer } from '../timer.class';
|
|
import { Timer } from '../timer.class';
|
|
|
import { Database, ServiceChangedStatus } from './database.class';
|
|
import { Database, ServiceChangedStatus } from './database.class';
|
|
|
import { FCMController } from './fcm-controller.class';
|
|
import { FCMController } from './fcm-controller.class';
|
|
|
|
|
+import { HealthCheckDataProvider } from './health-check-data-provider.interface';
|
|
|
|
|
|
|
|
type Subscriber = { id: number; interval: number; conf: HttpCheckConfig };
|
|
type Subscriber = { id: number; interval: number; conf: HttpCheckConfig };
|
|
|
|
|
+type ContentCheckError = { type: 'contentCheck'; status: HttpCheckStatus; message: string };
|
|
|
|
|
|
|
|
export class HttpCheckController {
|
|
export class HttpCheckController {
|
|
|
private subscriptions: Array<Subscriber> = [];
|
|
private subscriptions: Array<Subscriber> = [];
|
|
@@ -29,7 +31,7 @@ export class HttpCheckController {
|
|
|
this.scheduleCheck(conf);
|
|
this.scheduleCheck(conf);
|
|
|
|
|
|
|
|
Logger.info('[INFO] Initial HTTP Service Check for', conf.title, '...');
|
|
Logger.info('[INFO] Initial HTTP Service Check for', conf.title, '...');
|
|
|
- await this.timerTick(conf);
|
|
|
|
|
|
|
+ await this.runCheck(conf, this.db);
|
|
|
}
|
|
}
|
|
|
} catch (err) {
|
|
} catch (err) {
|
|
|
Logger.error('[FATAL] Initializing ServerConnector failed:', err);
|
|
Logger.error('[FATAL] Initializing ServerConnector failed:', err);
|
|
@@ -63,7 +65,7 @@ export class HttpCheckController {
|
|
|
if (Number.isNaN(interval)) interval = defaults.serviceChecks.interval;
|
|
if (Number.isNaN(interval)) interval = defaults.serviceChecks.interval;
|
|
|
|
|
|
|
|
if (log) Logger.info(`[INFO] Starting HTTP Service Check Controller for "${conf.title}" with interval ${interval} seconds ...`);
|
|
if (log) Logger.info(`[INFO] Starting HTTP Service Check Controller for "${conf.title}" with interval ${interval} seconds ...`);
|
|
|
- const id = Timer.instance.subscribe(interval, async () => await this.timerTick(conf));
|
|
|
|
|
|
|
+ const id = Timer.instance.subscribe(interval, async () => await this.runCheck(conf, this.db));
|
|
|
const sub = { id, interval, conf };
|
|
const sub = { id, interval, conf };
|
|
|
this.subscriptions.push(sub);
|
|
this.subscriptions.push(sub);
|
|
|
return sub;
|
|
return sub;
|
|
@@ -83,7 +85,7 @@ export class HttpCheckController {
|
|
|
this.subscriptions = this.subscriptions.filter(s => s.id !== sub.id);
|
|
this.subscriptions = this.subscriptions.filter(s => s.id !== sub.id);
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
- private async timerTick(conf: HttpCheckConfig) {
|
|
|
|
|
|
|
+ public async runCheck(conf: HttpCheckConfig, db: HealthCheckDataProvider) {
|
|
|
Logger.debug('[DEBUG] TICK', new Date(), JSON.stringify(conf));
|
|
Logger.debug('[DEBUG] TICK', new Date(), JSON.stringify(conf));
|
|
|
|
|
|
|
|
const now = new Date();
|
|
const now = new Date();
|
|
@@ -94,7 +96,7 @@ export class HttpCheckController {
|
|
|
let success = true;
|
|
let success = true;
|
|
|
try {
|
|
try {
|
|
|
const id = conf.id;
|
|
const id = conf.id;
|
|
|
- conf = (await this.db.getHttpCheckConfigByID(conf.serverId ?? 0, id)) as HttpCheckConfig;
|
|
|
|
|
|
|
+ conf = (await db.getHttpCheckConfigByID(conf.serverId ?? 0, id)) as HttpCheckConfig;
|
|
|
|
|
|
|
|
if (!conf) {
|
|
if (!conf) {
|
|
|
Logger.warn(`[WARN] HealthCheckConfig(${id}) not found in Database but still scheduled in Timer!`);
|
|
Logger.warn(`[WARN] HealthCheckConfig(${id}) not found in Database but still scheduled in Timer!`);
|
|
@@ -104,20 +106,17 @@ export class HttpCheckController {
|
|
|
options.timeout = conf.timeout;
|
|
options.timeout = conf.timeout;
|
|
|
let response = await axios.get(conf.url, options);
|
|
let response = await axios.get(conf.url, options);
|
|
|
const responseText = new String(response.data).toString();
|
|
const responseText = new String(response.data).toString();
|
|
|
|
|
+ const errors = this.recurseDisjunctChecks(conf.checks, responseText);
|
|
|
|
|
|
|
|
- for (const check of conf.checks) {
|
|
|
|
|
- const reg = new RegExp(check, 'i');
|
|
|
|
|
- if (!reg.test(responseText)) {
|
|
|
|
|
- Logger.debug(`[DEBUG] Regular expression /${check}/i not found in response`);
|
|
|
|
|
- await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.CheckFailed, `Regular expression /${check}/i not found in response`);
|
|
|
|
|
- success = false;
|
|
|
|
|
- }
|
|
|
|
|
|
|
+ for (const error of errors) {
|
|
|
|
|
+ await db.insertHealthCheckData(conf.id, now, error.status, error.message);
|
|
|
|
|
+ success = false;
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
if (success) {
|
|
if (success) {
|
|
|
if (conf.notify) {
|
|
if (conf.notify) {
|
|
|
try {
|
|
try {
|
|
|
- const lastErrors = await this.db.getLastErrors(conf.id, conf.notifyThreshold + 1);
|
|
|
|
|
|
|
+ const lastErrors = await db.getLastErrors(conf.id, conf.notifyThreshold + 1);
|
|
|
if (lastErrors.length > conf.notifyThreshold) {
|
|
if (lastErrors.length > conf.notifyThreshold) {
|
|
|
Logger.debug(`[DEBUG] Sending [RECOVERY] FCM Notification for`, conf.title);
|
|
Logger.debug(`[DEBUG] Sending [RECOVERY] FCM Notification for`, conf.title);
|
|
|
await FCMController.instance.sendNotificationToTopic(defaults.fcmTopics.services, {
|
|
await FCMController.instance.sendNotificationToTopic(defaults.fcmTopics.services, {
|
|
@@ -131,7 +130,7 @@ export class HttpCheckController {
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
Logger.debug(`[DEBUG] HTTP Service Check "${conf.title}": OK.`);
|
|
Logger.debug(`[DEBUG] HTTP Service Check "${conf.title}": OK.`);
|
|
|
- await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.OK, 'OK');
|
|
|
|
|
|
|
+ await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.OK, 'OK');
|
|
|
}
|
|
}
|
|
|
} catch (err) {
|
|
} catch (err) {
|
|
|
let log = false;
|
|
let log = false;
|
|
@@ -140,11 +139,11 @@ export class HttpCheckController {
|
|
|
// err.code = 'ECONNREFUSED' | 'ECONNABORTED' | 'ERR_BAD_REQUEST' | 'ERR_BAD_RESPONSE' | ...?
|
|
// err.code = 'ECONNREFUSED' | 'ECONNABORTED' | 'ERR_BAD_REQUEST' | 'ERR_BAD_RESPONSE' | ...?
|
|
|
try {
|
|
try {
|
|
|
if (err.code === 'ECONNABORTED') {
|
|
if (err.code === 'ECONNABORTED') {
|
|
|
- await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Timeout, err.message);
|
|
|
|
|
|
|
+ await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Timeout, err.message);
|
|
|
} else if (err.code && ['ERR_BAD_REQUEST', 'ERR_BAD_RESPONSE'].includes(err.code)) {
|
|
} else if (err.code && ['ERR_BAD_REQUEST', 'ERR_BAD_RESPONSE'].includes(err.code)) {
|
|
|
- await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, `${err.response?.status} ${err.response?.statusText}`);
|
|
|
|
|
|
|
+ await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, `${err.response?.status} ${err.response?.statusText}`);
|
|
|
} else {
|
|
} else {
|
|
|
- await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, err.message);
|
|
|
|
|
|
|
+ await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.RequestFailed, err.message);
|
|
|
}
|
|
}
|
|
|
} catch (insertErr) {
|
|
} catch (insertErr) {
|
|
|
Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
|
|
Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
|
|
@@ -152,7 +151,7 @@ export class HttpCheckController {
|
|
|
}
|
|
}
|
|
|
} else {
|
|
} else {
|
|
|
try {
|
|
try {
|
|
|
- await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Unknown, new String(err).toString());
|
|
|
|
|
|
|
+ await db.insertHealthCheckData(conf.id, now, HttpCheckStatus.Unknown, new String(err).toString());
|
|
|
} catch (insertErr) {
|
|
} catch (insertErr) {
|
|
|
Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
|
|
Logger.error(`[ERROR] Inserting HealthCheckData on Error failed:`, insertErr);
|
|
|
}
|
|
}
|
|
@@ -162,7 +161,7 @@ export class HttpCheckController {
|
|
|
}
|
|
}
|
|
|
if (!success && conf.notify && !process.env.DEV_MODE) {
|
|
if (!success && conf.notify && !process.env.DEV_MODE) {
|
|
|
try {
|
|
try {
|
|
|
- const lastErrors = await this.db.getLastErrors(conf.id, conf.notifyThreshold + 1);
|
|
|
|
|
|
|
+ const lastErrors = await db.getLastErrors(conf.id, conf.notifyThreshold + 1);
|
|
|
if (lastErrors.length > conf.notifyThreshold) {
|
|
if (lastErrors.length > conf.notifyThreshold) {
|
|
|
Logger.debug(`[DEBUG] Sending [CRIT] FCM Notification for`, conf.title);
|
|
Logger.debug(`[DEBUG] Sending [CRIT] FCM Notification for`, conf.title);
|
|
|
const lastCheck = lastErrors[0];
|
|
const lastCheck = lastErrors[0];
|
|
@@ -180,6 +179,66 @@ export class HttpCheckController {
|
|
|
}
|
|
}
|
|
|
}
|
|
}
|
|
|
|
|
|
|
|
|
|
+ private recurseDisjunctChecks(checks: CheckDisjunction, responseText: string): ContentCheckError[] {
|
|
|
|
|
+ const errorBuffer: ContentCheckError[] = [];
|
|
|
|
|
+ Logger.debug(`[DEBUG] Processing ${checks.length} disjunctive checks ...`);
|
|
|
|
|
+ for (const check of checks) {
|
|
|
|
|
+ const errors: ContentCheckError[] = [];
|
|
|
|
|
+ if (typeof check === 'string') {
|
|
|
|
|
+ try {
|
|
|
|
|
+ this.doCheck(check, responseText);
|
|
|
|
|
+ } catch (error: any) {
|
|
|
|
|
+ if (error.type === 'contentCheck') {
|
|
|
|
|
+ errors.push(error as ContentCheckError);
|
|
|
|
|
+ } else throw error;
|
|
|
|
|
+ }
|
|
|
|
|
+ } else if (Array.isArray(check)) {
|
|
|
|
|
+ errors.push(...this.recurseDisjunctChecks(check, responseText));
|
|
|
|
|
+ } else {
|
|
|
|
|
+ errors.push(...this.recurseConjunctChecks(check, responseText));
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ if (errors.length) {
|
|
|
|
|
+ errorBuffer.push(...errors);
|
|
|
|
|
+ } else {
|
|
|
|
|
+ return [];
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ Logger.debug(`[DEBUG] All disjunctive checks failed, collected ${errorBuffer.length} errors`);
|
|
|
|
|
+ return errorBuffer;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private recurseConjunctChecks(check: CheckConjunction, responseText: string): ContentCheckError[] {
|
|
|
|
|
+ const errorBuffer: ContentCheckError[] = [];
|
|
|
|
|
+ Logger.debug(`[DEBUG] Processing ${check.and.length} conjunctive checks ...`);
|
|
|
|
|
+ for (const con of check.and) {
|
|
|
|
|
+ try {
|
|
|
|
|
+ if (typeof con === 'string') {
|
|
|
|
|
+ this.doCheck(con, responseText);
|
|
|
|
|
+ } else if (Array.isArray(con)) {
|
|
|
|
|
+ errorBuffer.push(...this.recurseDisjunctChecks(con, responseText));
|
|
|
|
|
+ } else {
|
|
|
|
|
+ errorBuffer.push(...this.recurseConjunctChecks(con, responseText));
|
|
|
|
|
+ }
|
|
|
|
|
+ } catch (error: any) {
|
|
|
|
|
+ if (error.type === 'contentCheck') {
|
|
|
|
|
+ errorBuffer.push(error as ContentCheckError);
|
|
|
|
|
+ } else throw error;
|
|
|
|
|
+ }
|
|
|
|
|
+ }
|
|
|
|
|
+ Logger.debug(`[DEBUG] Ran through conjunctive checks, collected ${errorBuffer.length} errors`);
|
|
|
|
|
+ return errorBuffer;
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
|
|
+ private doCheck(check: string, responseText: string) {
|
|
|
|
|
+ const reg = new RegExp(check, 'i');
|
|
|
|
|
+ if (!reg.test(responseText)) {
|
|
|
|
|
+ Logger.debug(`[DEBUG] Regular expression /${check}/i not found in response`);
|
|
|
|
|
+ throw { type: 'contentCheck', status: HttpCheckStatus.CheckFailed, message: `Regular expression /${check}/i not found in response` };
|
|
|
|
|
+ }
|
|
|
|
|
+ Logger.debug(`[DEBUG] RegExp check /${check}/i successful ✔︎`);
|
|
|
|
|
+ }
|
|
|
|
|
+
|
|
|
async close() {
|
|
async close() {
|
|
|
if (!this.db) return;
|
|
if (!this.db) return;
|
|
|
await this.db.close();
|
|
await this.db.close();
|