From 39596493afa217727b680308fd4fe2f0111eeedd Mon Sep 17 00:00:00 2001
From: David Trattnig <david.trattnig@o94.at>
Date: Fri, 8 May 2020 09:04:38 +0200
Subject: [PATCH] Mail notifications, healthy state ressurection.
---
modules/core/engine.py | 4 +--
modules/core/monitor.py | 58 +++++++++++++++++++++++++++++++++++++----
2 files changed, 55 insertions(+), 7 deletions(-)
diff --git a/modules/core/engine.py b/modules/core/engine.py
index 58a88e22..3279bd22 100644
--- a/modules/core/engine.py
+++ b/modules/core/engine.py
@@ -78,9 +78,9 @@ class SoundSystem():
self.client = LiquidSoapPlayerClient(config, "engine.sock")
# self.lqcr = LiquidSoapRecorderClient(config, "record.sock")
- self.monitoring = Monitoring(config, self)
-
self.auramailer = AuraMailer(self.config)
+ self.monitoring = Monitoring(config, self, self.auramailer)
+
self.is_active()
# Initialize Default Channels
diff --git a/modules/core/monitor.py b/modules/core/monitor.py
index 5a270cc5..f4bbce0d 100644
--- a/modules/core/monitor.py
+++ b/modules/core/monitor.py
@@ -18,21 +18,24 @@
+import meta
+
import sys
import urllib
import logging
import json
import os.path
import threading
+import platform
from os import path
from enum import Enum
-from socket import socket, AF_INET, SOCK_DGRAM
from time import time, ctime, sleep
+from socket import socket, AF_INET, SOCK_DGRAM, SOL_SOCKET, SO_BROADCAST
-import meta
-from modules.communication.redis.adapter import ClientRedisAdapter
+from modules.communication.redis.adapter import ClientRedisAdapter
+from modules.base.utils import SimpleUtil
class MonitorResponseCode(Enum):
@@ -52,7 +55,10 @@ class Monitoring:
"""
logger = None
soundsystem = None
+ mailer = None
status = None
+ already_invalid = None
+ engine_id = None
heartbeat_server = None
heartbeat_port = None
@@ -61,13 +67,14 @@ class Monitoring:
heartbeat_running = None
- def __init__(self, config, soundsystem):
+ def __init__(self, config, soundsystem, mailer):
"""
Initialize Monitoring
"""
self.logger = logging.getLogger("AuraEngine")
self.config = config
self.soundsystem = soundsystem
+ self.mailer = mailer
self.status = dict()
self.status["engine"] = dict()
self.status["soundsystem"] = dict()
@@ -76,6 +83,7 @@ class Monitoring:
self.status["api"]["steering"] = dict()
self.status["api"]["tank"] = dict()
self.status["api"]["engine"] = dict()
+ self.already_invalid = False
# Heartbeat settings
self.heartbeat_running = False
@@ -84,7 +92,8 @@ class Monitoring:
self.heartbeat_frequency = config.get("heartbeat_frequency")
self.heartbeat_socket = socket(AF_INET, SOCK_DGRAM)
-
+ self.engine_id = self.get_engine_id()
+
#
# PUBLIC METHODS
@@ -194,6 +203,24 @@ class Monitoring:
"""
if self.has_valid_status(True):
self.heartbeat_socket.sendto(str.encode("OK"), (self.heartbeat_server, self.heartbeat_port))
+
+ # Engine resurrected into normal state
+ if self.already_invalid:
+ self.already_invalid = False
+ status = json.dumps(self.get_status())
+ self.logger.info(SimpleUtil.green("OK - Engine turned back into some healthy state!")+"\n"+str(status))
+ self.mailer.send_admin_mail( \
+ "OK - Engine turned back into some HEALTHY STATE!", \
+ "Things seem fine again at '%s':\n\n%s" % (self.engine_id, status))
+ else:
+ # Engine turned into invalid state
+ if not self.already_invalid:
+ self.already_invalid = True
+ status = json.dumps(self.get_status())
+ self.logger.critical(SimpleUtil.red("Engine turned into some INVALID STATE!")+"\n"+str(status))
+ self.mailer.send_admin_mail( \
+ "ERROR - Engine turned into some INVALID STATE!", \
+ "There's an issue with Aura Engine '%s':\n\n%s" % (self.engine_id, status))
threading.Timer(self.config.get("heartbeat_frequency"), self.heartbeat).start()
@@ -280,3 +307,24 @@ class Monitoring:
self.logger.error("Error while connecting to URL '%s' - %s" % (url, e))
return MonitorResponseCode.INVALID_STATE.value
+
+
+
+ def get_engine_id(self):
+ """
+ Retrieves a String identifier consisting of IP and Hostname to differentiate
+ the engine in mails and status broadcasts.
+ """
+ host = platform.node()
+ return "%s (%s)" % (self.get_ip(), host)
+
+
+
+ def get_ip(self):
+ """
+ Returns the IP of the Engine instance.
+ """
+ s = socket(AF_INET, SOCK_DGRAM)
+ s.setsockopt(SOL_SOCKET, SO_BROADCAST, 1)
+ s.connect(('<broadcast>', 0))
+ return s.getsockname()[0]
--
GitLab