Server: [RECOVERY] Notifications for servers and HTTP service checks

server/src/ctrl/http-check-controller.class.ts

@@ -115,6 +115,21 @@ export class HttpCheckController {
       }
 
       if (success) {
+        if (conf.notify) {
+          try {
+            const lastErrors = await this.db.getLastErrors(conf.id, conf.notifyThreshold + 1);
+            if (lastErrors.length > conf.notifyThreshold) {
+              Logger.debug(`[DEBUG] Sending [RECOVERY] FCM Notification for`, conf.title);
+              await FCMController.instance.sendNotificationToTopic(defaults.fcmTopics.services, {
+                title: `[RECOVERY] ${conf.title} since ${moment(now).format('HH:mm')}`,
+                body: `HTTP Check '${conf.title}' has recovered to [OK].`
+              });
+            }
+          } catch (err) {
+            Logger.error('[ERROR] Notification failure:', err);
+          }
+        }
+
         Logger.debug(`[DEBUG] HTTP Service Check "${conf.title}": OK.`);
         await this.db.insertHealthCheckData(conf.id, now, HttpCheckStatus.OK, 'OK');
       }
@@ -149,13 +164,13 @@ export class HttpCheckController {
       try {
         const lastErrors = await this.db.getLastErrors(conf.id, conf.notifyThreshold + 1);
         if (lastErrors.length > conf.notifyThreshold) {
-          Logger.debug(`[DEBUG] Sending FCM Notification for`, conf.title);
+          Logger.debug(`[DEBUG] Sending [CRIT] FCM Notification for`, conf.title);
           const lastCheck = lastErrors[0];
           const lastError = lastCheck.data[0];
           await FCMController.instance.sendNotificationToTopic(defaults.fcmTopics.services, {
-            title: `[CRIT] ${conf.title} since ${moment(lastCheck.time).format('HH:mm')}`,
+            title: `[CRIT] ${conf.title} failed at ${moment(lastCheck.time).format('HH:mm')}`,
             body:
-              `HTTP Check '${conf.title}' has failed over ${conf.notifyThreshold} times in a row\n` +
+              `HTTP Check '${conf.title}' has failed over ${conf.notifyThreshold} times in a row.\n` +
               `Last error status was: (${lastError.status}) ${lastError.message}`
           });
         }

server/src/ctrl/server-connector.class.ts

@@ -37,7 +37,7 @@ export class ServerConnector {
     })();
   }
 
-  private async timerTick(server: Server) {
+  private async timerTick(server: Server & { errors?: number }) {
     Logger.debug('[DEBUG] TICK', new Date(), JSON.stringify(server));
     if (process.env.DEV_MODE) return Logger.warn('[WARN] DEV_MODE active - sync inactive.');
 
@@ -56,12 +56,26 @@ export class ServerConnector {
 
       // Commit Transaction
       await axios.patch(`http://${server.fqdn}:8890/${trxHdl}`, null, { responseType: 'json' });
+
+      if (server.errors) {
+        // notify [RECOVERY]
+        try {
+          await FCMController.instance.sendNotificationToTopic(defaults.fcmTopics.serverData, {
+            title: `[RECOVERY] Server '${server.title}': daemon OK`,
+            body: `[RECOVERY] Server '${server.title}': daemon OK`
+          });
+        } catch (err) {
+          Logger.error('[ERROR] Notification failure:', err);
+        }
+      }
+      server.errors = 0;
     } catch (err) {
       Logger.error('[ERROR] Server data sync failed:', err);
 
       if (err instanceof AxiosError) {
         if (['ECONNREFUSED', 'ECONNABORTED'].includes(err.code ?? '')) {
           try {
+            server.errors = (server.errors ?? 0) + 1;
             await FCMController.instance.sendNotificationToTopic(defaults.fcmTopics.serverData, {
               title: `[WARN] Server '${server.title}': daemon unreachable (${err.code})`,
               body: new String(err).toString()