Commit ca62e57a authored by Gottfried Gaisbauer's avatar Gottfried Gaisbauer
Browse files

checking free diskspace. send warnings (email & logging) and stop recorder if...

checking free diskspace. send warnings (email & logging) and stop recorder if a critical value is reached.
parent 88626302
......@@ -9,6 +9,7 @@ from modules.scheduling.scheduler import AuraScheduler
from modules.communication.liquidsoap.communicator import LiquidSoapCommunicator
from modules.communication.redis.adapter import ServerRedisAdapter
from modules.web.routes import Routes
from modules.monitoring.diskspace_watcher import DiskSpaceWatcher
from libraries.base.logger import AuraLogger
from testing.test import TestConfig, TestLogger
......@@ -49,6 +50,9 @@ class Aura(AuraLogger):
self.messenger.scheduler = self.scheduler
self.messenger.liquidsoapcommunicator = self.liquidsoapcommunicator
self.diskspace_watcher = DiskSpaceWatcher(self.config, self.logger, self.liquidsoapcommunicator)
self.diskspace_watcher.run()
def receive_signal(signum, stack):
print("received signal")
server.reload()
......
......@@ -7,27 +7,30 @@ db_name="engine"
db_pass="engine"
db_host="localhost"
[socket]
# if you change this settings, you have to restart liquidsoap
socketdir="/home/gg/PycharmProjects/engine/modules/liquidsoap"
[logging]
# if you change this settings, you have to restart liquidsoap
logdir="/var/log/aura"
# possible values: debug, info, warning, error, critical
loglevel="info"
[monitoring]
# how often should i check the diskspace
diskspace_check_interval=10
# under which value should i start sending admin mails. possible values k, M, G, T or no metric prefix. defaults to 2G
diskspace_warning_value=1G
# under which value should i stop recording. defaults to 200M
diskspace_critical_value=10g0M
[mail]
mail_server="mail.servus.at"
mail_server=""
mail_server_port=""
mail_user=""
mail_pass=""
# multiple adminmails => space separated
admin_mail="gogo@servus.at gottfried@servus.at"
from_mail=""
from_mail="monitor@aura.py"
# The beginning of the subject
mail_prefix="[AURA]"
[dataurls]
calendarurl="http://localhost:8000/api/v1/playout"
importerurl="http://localhost:8008/api/v1/groups/_public/playlists/"
# how often should the calendar be fetched in seconds (This determines the time of the last change before a specific show)
fetching_frequency=3600
#######################
# LiquidSoap Settings #
......@@ -40,6 +43,14 @@ importerurl="http://localhost:8008/api/v1/groups/_public/playlists/"
daemongroup="gg"
daemonuser="gg"
[socket]
socketdir="/home/gg/PycharmProjects/engine/modules/liquidsoap"
[logging]
logdir="/var/log/aura"
# possible values: debug, info, warning, error, critical
loglevel="info"
# track_sensitive => fallback_folder track sensitivity
# max_blank => maximum time of blank from source (float)
# min_noise => minimum duration of noise on source to switch back over (float)
......
......@@ -13,3 +13,9 @@ class RedisConnectionException(Exception):
class PlaylistException(Exception):
pass
class MailingException(Exception):
pass
class DiskSpaceException(Exception):
pass
......@@ -47,7 +47,7 @@ class LiquidSoapCommunicator(ExceptionLogger):
self.error_data = simplejson.load(f)
f.close()
self.auramailer = AuraMailer(config.get("admin_mail"), config.get("from_mail"))
self.auramailer = AuraMailer(self.config)
# ------------------------------------------------------------------------------------------ #
def set_volume(self, mixernumber, volume):
......@@ -130,6 +130,39 @@ class LiquidSoapCommunicator(ExceptionLogger):
# we already caught and handled this error in __send_lqc_command__, but we do not want to execute this function further
pass
# ------------------------------------------------------------------------------------------ #
def recorder_stop(self):
self.enable_transaction()
for i in range(5):
if self.config.get("rec_" + str(i)) == "y":
self.__send_lqc_command__(self.client, "recorder_" + str(i), "stop")
self.disable_transaction()
# ------------------------------------------------------------------------------------------ #
def recorder_start(self, num=-1):
self.enable_transaction()
if num == -1:
self.recorder_start_all()
else:
self.recorder_start_one(num)
# ------------------------------------------------------------------------------------------ #
def recorder_start_all(self):
for i in range(5):
self.recorder_start_one(i)
# ------------------------------------------------------------------------------------------ #
def recorder_start_one(self, num):
self.enable_transaction()
if self.config.get("rec_" + str(num)) == "y":
returnvalue = self.__send_lqc_command__(self.client, "recorder", str(num), "status")
if returnvalue == "off":
self.__send_lqc_command__(self.client, "recorder", str(num), "start")
self.disable_transaction()
# ------------------------------------------------------------------------------------------ #
def activate(self, entry):
......@@ -963,34 +996,9 @@ class LiquidSoapCommunicator(ExceptionLogger):
# we already caught and handled this error in __send_lqc_command__, but we do not want to execute this function further
pass
# ------------------------------------------------------------------------------------------ #
def recorder_start(self):
"""
Recorder starten
@rtype: string
@return: Die Antwort des Liquidsoap-Servers
"""
message = self.__send_lqc_command__(self.lqcr, "start_record")
if message.strip() == "OK":
self.success("00")
else:
self.warning("01")
self.notifyClient()
# ------------------------------------------------------------------------------------------ #
def recorder_stop(self):
"""
Recorder stoppen
"""
message = self.__send_lqc_command__(self.lqcr, "stop_record")
if message.strip() == "OK":
self.success("00")
else:
self.warning("01")
self.notifyClient()
# ------------------------------------------------------------------------------------------ #
def recorder_data(self):
......
......@@ -20,6 +20,19 @@ class LiquidSoapPlayerClient(LiquidSoapClient):
return "LiquidSoapPlayerClient does not understand mixer."+command+str(args)
# ------------------------------------------------------------------------------------------ #
def recorder(self, num, command, *args):
if command == "status":
return self.recorderstatus(num)
if command == "start":
return self.recorderstart(num)
if command == "stop":
return self.recorderstop(num)
return "LiquidSoapPlayerClient does not understand mixer." + command + str(args)
# ------------------------------------------------------------------------------------------ #
def http(self, command, *args):
if command == "url":
......@@ -112,7 +125,34 @@ class LiquidSoapPlayerClient(LiquidSoapClient):
self.command("mixer", "volume", str(pos) + " " + str(volume))
return self.message
# ------------------------------------------------------------------------------------------ #
# ------------------------------------------------------------------------------------------ #
def recorderstatus(self, num):
"""
get status of a recorder
:return:
"""
self.command("recorder_" + str(num), "status")
return self.message
# ------------------------------------------------------------------------------------------ #
def recorderstart(self, num):
"""
get status of a recorder
:return:
"""
self.command("recorder_" + str(num), "start")
return self.message
# ------------------------------------------------------------------------------------------ #
def recorderstop(self, num):
"""
get status of a recorder
:return:
"""
self.command("recorder_" + str(num), "stop")
return self.message
# ------------------------------------------------------------------------------------------ #
def skip(self, namespace="playlist", pos=""):
"""
Source skippen
......
__author__ = 'michel'
__author__ = 'gg'
import os
import smtplib
from email.message import EmailMessage
from libraries.exceptions.auraexceptions import MailingException
class AuraMailer():
config = None
def __init__(self, config):
self.config = config
self.admin_mails = config.get("admin_mail")
def __init__(self, admin_mails, from_mail):
self.admin_mails = admin_mails
self.from_mail = from_mail
def send_admin_mail(self, subject, body):
admin_mails = self.admin_mails.split()
......@@ -16,12 +21,45 @@ class AuraMailer():
self.__send(mail_to, subject, body)
def __send(self, mail_to, subject, body):
sendmail_location = "/usr/sbin/sendmail"
p = os.popen("%s -t" % sendmail_location, "w")
p.write("From: %s\n" % self.from_mail)
p.write("To: %s\n" % mail_to)
p.write("Subject: " + subject + "\n")
p.write("\n") # blank line separating headers from body
p.write(body)
status = p.close()
return status
mail_server = self.config.get("mail_server")
mail_port = self.config.get("mail_server_port")
mail_user = self.config.get("mail_user")
mail_pass = self.config.get("mail_pass")
from_mail = self.config.get("from_mail")
if mail_server == "":
raise MailingException("Mail Server not set")
if mail_port == "":
raise MailingException("Mailserver Port not set")
if mail_user == "":
raise MailingException("Mail user not set")
if mail_pass == "":
raise MailingException("No Password for mailing set")
if from_mail == "":
raise MailingException("From Mail not set")
msg = EmailMessage()
msg.set_content(body)
mail_prefix = self.config.get("mail_prefix")
if mail_prefix == "":
msg["Subject"] = subject
else:
msg["Subject"] = mail_prefix + " " + subject
msg["From"] = from_mail
msg["To"] = mail_to
server = smtplib.SMTP(mail_server, int(mail_port))
server.starttls()
server.login(mail_user, mail_pass)
server.send_message(msg)
server.quit()
# sendmail_location = "/usr/sbin/sendmail"
# p = os.popen("%s -t" % sendmail_location, "w")
# p.write("From: %s\n" % self.from_mail)
# p.write("To: %s\n" % mail_to)
# p.write("Subject: " + subject + "\n")
# p.write("\n") # blank line separating headers from body
# p.write(body)
# status = p.close()
# return status
import os
import datetime
import threading
from collections import namedtuple
from modules.communication.mail.mail import AuraMailer
from libraries.exceptions.auraexceptions import MailingException
from libraries.exceptions.auraexceptions import DiskSpaceException
class DiskSpaceWatcher(threading.Thread):
liquidsoapcommunicator = None
exit_event = None
config = None
logger = None
mailer = None
sent_a_mail = False
is_critical = False
def __init__(self, config, logger, liquidsoapcommunicator):
self.liquidsoapcommunicator = liquidsoapcommunicator
self.config = config
self.logger = logger
self.mailer = AuraMailer(self.config)
self.exit_event = threading.Event()
def run(self):
# set seconds to wait
seconds_to_wait = int(self.config.get("diskspace_check_interval"))
# while True:
while not self.exit_event.is_set():
# calc next time
next_time = datetime.datetime.now() + datetime.timedelta(seconds=seconds_to_wait)
# write to logger
self.logger.info("Diskspace watcher every " + str(seconds_to_wait) + "s started. Going to start next time " + str(next_time))
# check disk space
self.check_disk_space()
# and wait
#self.exit_event.wait(seconds_to_wait)
self.stop()
def stop(self):
self.exit_event.set()
def check_disk_space(self):
# check disk space where aure engine is writing to
self.check_recorder_disk_space()
self.check_logging_disk_space()
if self.is_critical:
self.logger.critical("Recorder STOPPED due to diskspace! FIX THIS!!!")
if self.sent_a_mail:
self.logger.warning("Recorder is going stop soon because of not enough diskspace! FIX THIS!")
if not self.is_critical and not self.sent_a_mail:
self.logger.debug("No disk space issues detected.")
self.is_critical = False
self.sent_a_mail = False
def check_recorder_disk_space(self):
for i in range(5):
if self.config.get("rec_" + str(i)) == "y":
self.check_recorder_num_disk_space(i)
def check_recorder_num_disk_space(self, num):
folder = self.config.get("rec_" + str(num) + "_folder")
try:
self.check_disk_space_of_folder(folder)
# ensure recorder is running
self.liquidsoapcommunicator.recorder_start(num)
except DiskSpaceException as e:
self.logger.critical(str(e))
# stop recorder when diskspace is critical
self.liquidsoapcommunicator.recorder_stop(num)
def check_logging_disk_space(self):
try:
self.check_disk_space_of_folder(self.config.get("logdir"))
except DiskSpaceException as e:
self.logger.critical(str(e))
def check_disk_space_of_folder(self, folder):
warning_value_raw = self.config.get("diskspace_warning_value")
critical_value_raw = self.config.get("diskspace_critical_value")
try:
warning_value = self.parse_diskspace(warning_value_raw)
except ValueError:
warning_value_raw = "2G"
warning_value = self.parse_diskspace(warning_value_raw)
try:
critical_value = self.parse_diskspace(critical_value_raw)
except ValueError:
critical_value_raw = "200M"
critical_value = self.parse_diskspace(critical_value_raw)
usage = namedtuple("usage", "total used free")
diskspace = os.statvfs(folder)
free = diskspace.f_bavail * diskspace.f_frsize
total = diskspace.f_blocks * diskspace.f_frsize
used = (diskspace.f_blocks - diskspace.f_bfree) * diskspace.f_frsize
if free < warning_value:
subj = "Diskspace warning"
msg = "Free space in " + folder + " under " + warning_value_raw + ". " + str(usage(total, used, free))
self.send_mail(subj, msg)
self.liquidsoapcommunicator.recorder_start()
self.sent_a_mail = True
elif free < critical_value:
subj = "Critical diskspace - Recorder stopped!"
msg = "Free space in " + folder + " under " + critical_value_raw + ". " + str(usage(total, used, free))
self.send_mail(subj, msg)
self.sent_a_mail = True
self.is_critical = True
raise DiskSpaceException("Diskspace in " + folder + " reached critical value!")
def send_mail(self, subj, msg):
try:
self.logger.info("Trying to send mail with subject " + subj + " and message " + msg + ".")
self.mailer.send_admin_mail(subj, msg)
except MailingException as e:
self.logger.critical("Cannot send mail with subject " + subj + " and message " + msg + ". Reason: " + str(e))
def parse_diskspace(self, value):
if value.endswith("K") or value.endswith("k"):
return int(value[:-1]) * 1024
if value.endswith("M") or value.endswith("m"):
return int(value[:-1]) * 1024 * 1024
if value.endswith("G") or value.endswith("g"):
return int(value[:-1]) * 1024 * 1024 * 1024
if value.endswith("T") or value.endswith("t"):
return int(value[:-1]) * 1024 * 1024 * 1024 * 1024
return int(value)
\ No newline at end of file
......@@ -225,7 +225,7 @@ class AuraCalendarService(threading.Thread):
if not self.datefrom:
return False
date_start = datetime.strptime(self.datefrom.replace("T"," "), "%Y-%m-%d %H:%M")
date_start = datetime.strptime(self.datefrom.replace("T"," "), "%Y-%m-%d %H:%M:%S")
time_start = date_start.strftime("%H:%M")
day_offset = 1 if (time_start > self.until) else 0
end_date = date_start + timedelta(day_offset)
......
......@@ -112,6 +112,7 @@ class AuraScheduler(ExceptionLogger, threading.Thread):
self.redismessenger.send('Scheduler started', '0000', 'success', 'initApp', None, 'appinternal')
# create exit event
self.exit_event = threading.Event()
# start loading new programm every hour
......@@ -133,7 +134,7 @@ class AuraScheduler(ExceptionLogger, threading.Thread):
def run(self):
# set seconds to wait
seconds_to_wait = 3600
seconds_to_wait = self.config.get("fetching_frequency")
#while True:
while not self.exit_event.is_set():
......
......@@ -2,8 +2,13 @@
from libraries.database.broadcasts import *
import simplejson
import logging
import sqlalchemy
from modules.communication.liquidsoap.communicator import LiquidSoapCommunicator
from modules.monitoring.diskspace_watcher import DiskSpaceWatcher
from libraries.base.config import AuraConfig
def alchemyencoder(obj):
"""JSON encoder function for SQLAlchemy special classes."""
if isinstance(obj, datetime.date):
......@@ -25,11 +30,18 @@ def select_with_relationship():
# programme_as_string = simplejson.dumps([se[0]._asdict()], default=alchemyencoder)
# print(programme_as_string)
def start_diskspace_watcher():
config = AuraConfig()
config.read_config()
diskspace_watcher = DiskSpaceWatcher(config.config, logging.getLogger("AuraEngine"), LiquidSoapCommunicator(config.config))
diskspace_watcher.run()
# # ## ## ## ## ## # #
# # ENTRY FUNCTION # #
# # ## ## ## ## ## # #
def main():
select_with_relationship()
#select_with_relationship()
start_diskspace_watcher()
# # ## ## ## ## ## ## # #
# # End ENTRY FUNCTION # #
# # ## ## ## ## ## ## # #
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment