From 3a76b6100e845067b3d9890068a7acad8cdfcc10 Mon Sep 17 00:00:00 2001
From: David Trattnig <david.trattnig@o94.at>
Date: Fri, 28 Aug 2020 14:37:13 +0200
Subject: [PATCH] Store health info. #29
---
modules/core/events.py | 19 +++++++++-------
modules/plugins/monitor.py | 44 ++++++++++++++++++++++++++++++++++++++
2 files changed, 55 insertions(+), 8 deletions(-)
diff --git a/modules/core/events.py b/modules/core/events.py
index 2f4d7bfb..074ae383 100644
--- a/modules/core/events.py
+++ b/modules/core/events.py
@@ -27,7 +27,7 @@ from modules.plugins.monitor import AuraMonitor
from modules.core.state import PlayerStateService
-from modules.plugins.trackservice import TrackserviceHandler
+from modules.plugins.trackservice import TrackServiceHandler
class EventBinding():
@@ -57,7 +57,7 @@ class EventBinding():
return self
- def get_instances(self):
+ def get_instance(self):
"""
Returns the object within that binding.
"""
@@ -77,7 +77,7 @@ class EngineEventDispatcher():
soundsystem = None
player_state = None
scheduler = None
- api_handler = None
+ monitor = None
def __init__(self, soundsystem, scheduler):
@@ -94,7 +94,10 @@ class EngineEventDispatcher():
binding = self.attach(AuraMonitor)
binding.subscribe("on_boot")
- binding = self.attach(TrackserviceHandler)
+ binding.subscribe("on_sick")
+ binding.subscribe("on_resurrect")
+
+ binding = self.attach(TrackServiceHandler)
binding.subscribe("on_play")
@@ -223,20 +226,20 @@ class EngineEventDispatcher():
self.call_event("on_queue", entries)
- def on_sick(self):
+ def on_sick(self, data):
"""
Called when the engine is in some unhealthy state.
"""
self.logger.debug("on_sick(..)")
- self.call_event("on_sick", None)
+ self.call_event("on_sick", data)
- def on_resurrect(self):
+ def on_resurrect(self, data):
"""
Called when the engine turned healthy again after being sick.
"""
self.logger.debug("on_resurrect(..)")
- self.call_event("on_resurrect", None)
+ self.call_event("on_resurrect", data)
def on_critical(self, subject, message, data=None):
diff --git a/modules/plugins/monitor.py b/modules/plugins/monitor.py
index fed52675..776f3bc6 100644
--- a/modules/plugins/monitor.py
+++ b/modules/plugins/monitor.py
@@ -18,9 +18,11 @@
import os
+import datetime
import urllib
import logging
import json
+import requests
import threading
import platform
@@ -115,9 +117,26 @@ class AuraMonitor:
self.logger.info("Status Monitor:\n%s" % json.dumps(status, indent=4))
if not is_valid:
self.logger.info("Engine Status: " + SU.red(status["engine"]["status"]))
+ self.post_health(status, False)
raise EngineMalfunctionException
else:
self.logger.info("Engine Status: " + SU.green("[OK]"))
+ self.post_health(status, True)
+
+
+
+ def on_sick(self, data):
+ """
+ Called when the engine is in some unhealthy state.
+ """
+ self.post_health(data, False)
+
+
+ def on_resurrect(self, data):
+ """
+ Called when the engine turned healthy again after being sick.
+ """
+ self.post_health(data, True)
#
@@ -172,6 +191,27 @@ class AuraMonitor:
#
+ def post_health(self, data, is_healthy):
+ """
+ Post unhealthy state info to Engine API.
+ """
+ body = dict()
+ body["log_time"] = datetime.datetime.now()
+ body["is_healthy"] = is_healthy
+ body["details"] = json.dumps(data, default=str)
+ json_data = json.dumps(body, default=str)
+
+ url = self.config.get("api_engine_store_health")
+ url = url.replace("${ENGINE_NUMBER}", str(self.config.get("api_engine_number")))
+ headers = {'content-type': 'application/json'}
+ r = requests.post(url, data=json_data, headers=headers)
+
+ if r.status_code == 204:
+ self.logger.info("Successfully posted healthy=%s state to Engine API!" % is_healthy)
+ else:
+ self.logger.error("HTTP %s | Error while pushing health state to Engine API: %s" % (r.status_code, str(r.json())))
+
+
def update_status(self):
"""
Requests the current status of all components
@@ -233,6 +273,8 @@ class AuraMonitor:
self.mailer.send_admin_mail( \
"OK - Engine turned back into some HEALTHY STATE!", \
"Things seem fine again at '%s':\n\n%s" % (self.engine_id, status))
+ # Route call of event via event dispatcher to provide ability for additional hooks
+ self.soundsystem.event_dispatcher.on_resurrect(status)
else:
# Engine turned into invalid state
if not self.already_invalid:
@@ -242,6 +284,8 @@ class AuraMonitor:
self.mailer.send_admin_mail( \
"ERROR - Engine turned into some INVALID STATE!", \
"There's an issue with Aura Engine '%s':\n\n%s" % (self.engine_id, status))
+ # Route call of event via event dispatcher to provide ability for additional hooks
+ self.soundsystem.event_dispatcher.on_sick(status)
threading.Timer(self.config.get("heartbeat_frequency"), self.heartbeat).start()
--
GitLab