Commit 75f50ccb authored by David Trattnig's avatar David Trattnig
Browse files

Watch disk space externally to keep Engine focused

parent 79006101
......@@ -22,14 +22,6 @@ redis_port=6379
redis_db=0
[monitoring]
# how often should i check the diskspace. defaults to 600s = 10m
diskspace_check_interval=20
# under which value should i start sending admin mails. possible values k, M, G, T or no metric prefix. defaults to 2G
diskspace_warning_value=1G
# under which value should i stop recording. defaults to 200M
diskspace_critical_value=100M
[mail]
mail_server=""
mail_server_port="587"
mail_user="aura@subsquare.at"
......
......@@ -22,14 +22,6 @@ redis_port=6379
redis_db=0
[monitoring]
# how often should i check the diskspace. defaults to 600s = 10m
diskspace_check_interval=20
# under which value should i start sending admin mails. possible values k, M, G, T or no metric prefix. defaults to 2G
diskspace_warning_value=1G
# under which value should i stop recording. defaults to 200M
diskspace_critical_value=100M
[mail]
mail_server="w00fdabd.kasserver.com"
mail_server_port="587"
mail_user="aura@subsquare.at"
......
......@@ -22,14 +22,6 @@ redis_port=6379
redis_db=0
[monitoring]
# how often should i check the diskspace. defaults to 600s = 10m
diskspace_check_interval=20
# under which value should i start sending admin mails. possible values k, M, G, T or no metric prefix. defaults to 2G
diskspace_warning_value=1G
# under which value should i stop recording. defaults to 200M
diskspace_critical_value=100M
[mail]
mail_server="w00fdabd.kasserver.com"
mail_server_port="587"
mail_user="aura@subsquare.at"
......
......@@ -13,8 +13,6 @@ This page goes into detail on what is possible to configure within the engine.
- [Fallbacks](#fallbacks)
- [Database](#database)
- [Monitoring](#monitoring)
- [Web User Interface](#web-user-interface)
- [Administrator Emails](#administrator-emails)
- [API Endpoints](#api-endpoints)
- [Fading](#fading)
- [Logging](#logging)
......@@ -110,15 +108,7 @@ Configure your engine database in the `[database]` section.
## Monitoring
Configure monitoring parameters in the `[monitoring]` section.
## Web User Interface
Configure the web interface in the `[web-ui]` section.
## Administrator Emails
Configure the email server for admin mails in the `[mail]` section.
Configure monitoring parameters such as admin emails in the `[monitoring]` section.
## API Endpoints
......
......@@ -50,9 +50,6 @@ class NoActiveEntryException(Exception):
class MailingException(Exception):
pass
class DiskSpaceException(Exception):
pass
# Liquidsoap Execeptions
......
#
# engine
#
# Playout Daemon for autoradio project
#
#
# Copyright (C) 2017-2018 Gottfried Gaisbauer <gottfried.gaisbauer@servus.at>
#
# This file is part of engine.
#
# engine is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# any later version.
#
# engine is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with engine. If not, see <http://www.gnu.org/licenses/>.
#
import os
import datetime
import threading
from collections import namedtuple
from modules.communication.mail import AuraMailer
from modules.base.exceptions import MailingException
from modules.base.exceptions import DiskSpaceException
# ------------------------------------------------------------------------------------------ #
class DiskSpaceWatcher(threading.Thread):
soundsystem = None
exit_event = None
config = None
logger = None
mailer = None
sent_a_mail = False
is_critical = False
# ------------------------------------------------------------------------------------------ #
def __init__(self, config, logger, soundsystem):
threading.Thread.__init__(self)
self.soundsystem = soundsystem
self.config = config
self.logger = logger
self.mailer = AuraMailer(self.config)
self.exit_event = threading.Event()
# ------------------------------------------------------------------------------------------ #
def run(self):
# set seconds to wait
try:
seconds_to_wait = int(self.config.get("diskspace_check_interval"))
except:
seconds_to_wait = 600
while not self.exit_event.is_set():
try:
# calc next time
next_time = datetime.datetime.now() + datetime.timedelta(seconds=seconds_to_wait)
# check disk space
self.check_disk_space()
# write to logger
self.logger.info("Diskspace checked! Going to start next time " + str(next_time))
# and wait
self.exit_event.wait(seconds_to_wait)
except BrokenPipeError as e:
self.logger.critical("Cannot check if recorder is running. It seems LiquidSoap is not running. Reason: " + str(e))
# ------------------------------------------------------------------------------------------ #
def stop(self):
self.exit_event.set()
# ------------------------------------------------------------------------------------------ #
def check_disk_space(self):
# check disk space where aure engine is writing to
self.check_recorder_disk_space()
self.check_logging_disk_space()
if self.is_critical:
self.logger.critical("Recorder STOPPED due to LOW diskspace! FIX THIS!!!")
if self.sent_a_mail:
self.logger.warning("Recorder is going stop soon because of not enough diskspace! FIX THIS!")
if not self.is_critical and not self.sent_a_mail:
self.logger.debug("No disk space issues detected.")
self.is_critical = False
self.sent_a_mail = False
# ------------------------------------------------------------------------------------------ #
def check_recorder_disk_space(self):
for i in range(5):
if self.config.get("rec_" + str(i)) == "y":
self.check_recorder_num_disk_space(i)
# ------------------------------------------------------------------------------------------ #
def check_recorder_num_disk_space(self, num):
folder = self.config.get("rec_" + str(num) + "_folder")
try:
self.check_disk_space_of_folder(folder)
# ensure recorder is running
if self.soundsystem.is_liquidsoap_running:
self.soundsystem.recorder_start(num)
else:
self.logger.warning("Cannot enable recorder. Liquidsoap is not running!")
except DiskSpaceException as e:
self.logger.critical(str(e))
# stop recorder when diskspace is critical
if self.soundsystem.is_liquidsoap_running:
self.soundsystem.recorder_stop(num)
else:
self.logger.warning("Cannot stop recorder. Liquidsoap is not running!")
# ------------------------------------------------------------------------------------------ #
def check_logging_disk_space(self):
try:
self.check_disk_space_of_folder(self.config.get("logdir"))
except DiskSpaceException as e:
self.logger.critical(str(e))
# ------------------------------------------------------------------------------------------ #
def check_disk_space_of_folder(self, folder):
warning_value_raw = self.config.get("diskspace_warning_value")
critical_value_raw = self.config.get("diskspace_critical_value")
try:
warning_value = self.parse_diskspace(warning_value_raw)
except ValueError:
warning_value_raw = "2G"
warning_value = self.parse_diskspace(warning_value_raw)
try:
critical_value = self.parse_diskspace(critical_value_raw)
except ValueError:
critical_value_raw = "200M"
critical_value = self.parse_diskspace(critical_value_raw)
usage = namedtuple("usage", "total used free")
diskspace = os.statvfs(folder)
free = diskspace.f_bavail * diskspace.f_frsize
total = diskspace.f_blocks * diskspace.f_frsize
used = (diskspace.f_blocks - diskspace.f_bfree) * diskspace.f_frsize
if free < warning_value:
subj = "Diskspace warning"
msg = "Free space in " + folder + " under " + warning_value_raw + ". " + str(usage(total, used, free))
self.send_mail(subj, msg)
if self.soundsystem.is_liquidsoap_running:
self.soundsystem.recorder_start()
else:
self.logger.warning("Cannot enable recorder. Liquidsoap is not running!")
self.sent_a_mail = True
elif free < critical_value:
subj = "Critical diskspace - Recorder stopped!"
msg = "Free space in " + folder + " under " + critical_value_raw + ". " + str(usage(total, used, free))
self.send_mail(subj, msg)
self.sent_a_mail = True
self.is_critical = True
raise DiskSpaceException("Diskspace in " + folder + " reached critical value!")
# ------------------------------------------------------------------------------------------ #
def send_mail(self, subj, msg):
try:
self.logger.info("Trying to send mail with subject " + subj + " and message " + msg + ".")
self.mailer.send_admin_mail(subj, msg)
except MailingException as e:
self.logger.critical("Cannot send mail with subject " + subj + " and message " + msg + ". Reason: " + str(e))
# ------------------------------------------------------------------------------------------ #
def parse_diskspace(self, value):
if value.endswith("K") or value.endswith("k"):
return int(value[:-1]) * 1024
if value.endswith("M") or value.endswith("m"):
return int(value[:-1]) * 1024 * 1024
if value.endswith("G") or value.endswith("g"):
return int(value[:-1]) * 1024 * 1024 * 1024
if value.endswith("T") or value.endswith("t"):
return int(value[:-1]) * 1024 * 1024 * 1024 * 1024
return int(value)
\ No newline at end of file
......@@ -31,7 +31,6 @@ import sqlalchemy
import decimal
from modules.core.engine import SoundSystem
from modules.monitoring.diskspace_watcher import DiskSpaceWatcher
from modules.base.config import AuraConfig
from libraries.database.broadcasts import Schedule, ScheduleEntry
from modules.scheduling.scheduler import AuraScheduler, AuraCalendarService
......@@ -52,14 +51,7 @@ def alchemyencoder(obj):
# programme_as_string = json.dumps([se[0]._asdict()], default=alchemyencoder)
# print(programme_as_string)
def start_diskspace_watcher():
config = AuraConfig()
config.read_config()
diskspace_watcher = DiskSpaceWatcher(config.config, logging.getLogger("AuraEngine"), SoundSystem(config.config))
diskspace_watcher.run()
def select_current_programme():
# start_diskspace_watcher()
# select_programme()
config = AuraConfig()
config.read_config()
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment