From cd9bc5fbc224100b7abf08c288fdf69516340e74 Mon Sep 17 00:00:00 2001 From: Gottfried Gaisbauer <gogo@servus.at> Date: Sat, 24 Aug 2019 17:51:11 +0200 Subject: [PATCH] greater commit than expected. informations from tank now stored into database; introduced a new switch "--recreate-database" --- aura.py | 80 ++- configuration/engine.ini | 11 +- libraries/base/config.py | 97 ++- libraries/base/logger.py | 8 +- libraries/database/broadcasts.py | 619 +++++++++++------- libraries/database/database.py | 74 --- modules/base/config.py | 127 ---- modules/communication/connection_tester.py | 18 +- .../communication/liquidsoap/communicator.py | 5 +- modules/communication/redis/adapter.py | 10 +- modules/communication/redis/messenger.py | 2 +- modules/scheduling/calendar.py | 338 +++------- modules/scheduling/calender_fetcher.py | 269 ++++++++ modules/scheduling/scheduler.py | 30 +- modules/web/routes.py | 302 ++++----- 15 files changed, 1096 insertions(+), 894 deletions(-) delete mode 100644 libraries/database/database.py delete mode 100644 modules/base/config.py create mode 100644 modules/scheduling/calender_fetcher.py diff --git a/aura.py b/aura.py index 3bd4d508..59f753dc 100755 --- a/aura.py +++ b/aura.py @@ -27,27 +27,75 @@ import os import sys import signal +import logging import unittest -from modules.scheduling.scheduler import AuraScheduler -from modules.communication.liquidsoap.communicator import LiquidSoapCommunicator -from modules.communication.redis.adapter import ServerRedisAdapter -from modules.web.routes import Routes +from flask import request, render_template, Flask, Response +from flask_sqlalchemy import SQLAlchemy +from sqlalchemy.ext.declarative import declarative_base + +#from modules.web.routes import Routes from modules.monitoring.diskspace_watcher import DiskSpaceWatcher from libraries.base.logger import AuraLogger +from libraries.base.config import AuraConfig + + +def get_config_file(): + if len(sys.argv) >= 3 and "--config-file" in sys.argv: + idx = sys.argv.index("--config-file") + return sys.argv[idx + 1] + else: + return "/etc/aura/engine.ini" + + +def get_database_uri(): + db_name = config.get("db_name") + db_user = config.get("db_user") + db_pass = config.get("db_pass") + db_host = config.get("db_host") + db_charset = config.get("db_charset", "utf8") + + #### return created database uri #### + return "mysql://" + db_user + ":" + db_pass + "@" + db_host + "/" + db_name + "?charset=" + db_charset + -from testing.test import TestConfig, TestLogger +def configure_flask(): + app.config["SQLALCHEMY_DATABASE_URI"] = get_database_uri() + app.config['BABEL_DEFAULT_LOCALE'] = 'de' + app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False -class Aura(AuraLogger): + +config = AuraConfig(get_config_file()) +app = Flask(__name__, template_folder=config.get("install_dir") + "/modules/web/templates") +configure_flask() +DB = SQLAlchemy(app) +Base = declarative_base() + + +class Aura: + logger = None + config = None server = None messenger = None controller = None # ------------------------------------------------------------------------------------------ # def __init__(self): - super(Aura, self).__init__() + # set config + self.config = config + # init logger + AuraLogger(self.config) + # use logger + self.logger = logging.getLogger("AuraEngine") def startup(self): + from modules.scheduling.scheduler import AuraScheduler + from modules.communication.liquidsoap.communicator import LiquidSoapCommunicator + from modules.communication.redis.adapter import ServerRedisAdapter + + if self.config.get("recreate_db") is not None: + AuraScheduler(self.config) # handles recreate and exits program + # self.controller = AuraController(self.config) # create scheduler and ls_communicator @@ -55,12 +103,12 @@ class Aura(AuraLogger): self.scheduler = AuraScheduler(self.config) # give both a reference of each other - self.liquidsoapcommunicator.scheduler = self.scheduler - self.scheduler.liquidsoapcommunicator = self.liquidsoapcommunicator +# self.liquidsoapcommunicator.scheduler = self.scheduler +# self.scheduler.liquidsoapcommunicator = self.liquidsoapcommunicator # create the redis adapter self.messenger = ServerRedisAdapter(self.config) - self.messenger.scheduler = self.scheduler +# self.messenger.scheduler = self.scheduler self.messenger.liquidsoapcommunicator = self.liquidsoapcommunicator #self.diskspace_watcher = DiskSpaceWatcher(self.config, self.logger, self.liquidsoapcommunicator) @@ -80,7 +128,7 @@ class Aura(AuraLogger): def start_web_service(self): try: self.logger.info("Listening on Port 5000 for API or Webcalls") - Routes(self.scheduler, self.liquidsoapcommunicator, self.messenger, self.config) +# Routes(self.scheduler, self.liquidsoapcommunicator, self.messenger, self.config) except OSError as e: self.messenger.halt() self.logger.critical("AuraEngine already running? Exception: " + e.strerror + ". Exiting...") @@ -93,8 +141,14 @@ class Aura(AuraLogger): def main(): aura = Aura() - if len(sys.argv) >= 2 and "--use-test-data" in sys.argv: - aura.config.set("use_test_data", True) + aura.logger.critical("MAKE THE STARTTIME OF A SCHEDULE TO ITS PK") + + + if len(sys.argv) >= 2: + if "--use-test-data" in sys.argv: + aura.config.set("use_test_data", True) + if "--recreate-database" in sys.argv: + aura.config.set("recreate_db", True); aura.startup() diff --git a/configuration/engine.ini b/configuration/engine.ini index 06b7961f..7e428cca 100644 --- a/configuration/engine.ini +++ b/configuration/engine.ini @@ -7,6 +7,7 @@ db_user="engine" db_name="engine" db_pass="engine" db_host="localhost" +db_charset="utf8" [redis] redis_host="localhost" @@ -38,9 +39,9 @@ mailsubject_prefix="[AURA]" [dataurls] # the url of pv/steering -calendarurl="http://localhost:8000/api/v1/playout" +calendarurl="http://localhost:8001/api/v1/playout" # the url of tank -importerurl="http://localhost:8008/api/v1/groups/_public/playlists/" +importerurl="http://localhost:8040/api/v1/shows/" # how often should the calendar be fetched in seconds (This determines the time of the last change before a specific show) fetching_frequency=3600 @@ -70,6 +71,10 @@ logdir="/var/log/aura" # possible values: debug, info, warning, error, critical loglevel="info" +[audiofolder] +audiofolder="/var/audio" + +[fallback] # track_sensitive => fallback_folder track sensitivity # max_blank => maximum time of blank from source (float) # min_noise => minimum duration of noise on source to switch back over (float) @@ -282,4 +287,4 @@ stream_4_password="source" stream_4_url="http://www.fro.at" stream_4_name="AURA Test Stream 3" stream_4_genre="mixed" -stream_4_description="Test Stream 3" \ No newline at end of file +stream_4_description="Test Stream 3" diff --git a/libraries/base/config.py b/libraries/base/config.py index 8e7cd6fe..8bac44a7 100644 --- a/libraries/base/config.py +++ b/libraries/base/config.py @@ -22,23 +22,96 @@ # along with engine. If not, see <http://www.gnu.org/licenses/>. # -from modules.base.config import ConfigReader +import os +import sys +import logging + +from configparser import ConfigParser class AuraConfig: - """ - AuraCommon handles logger, reads and stores config - """ - config = None + ini_path = "" + logger = None + + def __init__(self, ini_path): # = "/etc/aura/engine.ini"): + self.ini_path = ini_path + self.logger = logging.getLogger("AuraEngine") + self.load_config() + + def set(self, key, value): + """ + Set a property + @type key: string + @param key: The Key + @type value: mixed + @param value: Beliebiger Wert + """ + try: + self.__dict__[key] = int(value) + except: + self.__dict__[key] = str(value) + + # ------------------------------------------------------------------------------------------ # + def get(self, key, default=None): + """ + get a loaded property + @type key: string + @param key: Der Key + @type default: mixed + @param default: Beliebiger Wert + """ - def __init__(self): - super(AuraConfig, self).__init__() - self.config = ConfigReader() - self.read_config() + if key not in self.__dict__: + if default: + self.set(key, default) + else: + self.logger.warning("Key " + key + " not found in configfile " + self.ini_path + "!") + return None - def read_config(self): + if key == "loglevel": + loglvl = self.__dict__[key] + + if loglvl == "debug": + return logging.DEBUG + elif loglvl == "info": + return logging.INFO + elif loglvl == "warning": + return logging.WARNING + elif loglvl == "error": + return logging.ERROR + else: + return logging.CRITICAL + + if key == "debug": + return self.__dict__[key].count("y") + + return self.__dict__[key] + + # ------------------------------------------------------------------------------------------ # + def load_config(self): """ - reads aura.ini + Set config defaults and load settings from file :return: """ - self.config.load_config() + if not os.path.isfile(self.ini_path): + self.logger.critical(self.ini_path + " not found :(") + sys.exit(1) + + # INI einlesen + f = open(self.ini_path, 'r') + ini_str = f.read() + f.close() + + config_parser = ConfigParser() + try: + config_parser.read_string(ini_str) + except Exception as e: + self.logger.critical("Cannot read " + self.ini_path + "! Reason: " + str(e)) + sys.exit(0) + + for section in config_parser.sections(): + for key, value in config_parser.items(section): + v = config_parser.get(section, key).replace('"', '').strip() + self.set(key, v) + + self.set("install_dir", os.path.realpath(__file__ + "../../../..")) diff --git a/libraries/base/logger.py b/libraries/base/logger.py index fd28ae10..ec83f2b9 100644 --- a/libraries/base/logger.py +++ b/libraries/base/logger.py @@ -27,11 +27,13 @@ import logging from libraries.base.config import AuraConfig -class AuraLogger(AuraConfig): +class AuraLogger(): + config = None logger = None - def __init__(self): - super(AuraLogger, self).__init__() + def __init__(self, config): + self.config = config + self.__create_logger("AuraEngine") def __create_logger(self, name): diff --git a/libraries/database/broadcasts.py b/libraries/database/broadcasts.py index 062b8bd3..8f313860 100644 --- a/libraries/database/broadcasts.py +++ b/libraries/database/broadcasts.py @@ -27,14 +27,19 @@ import time import logging import datetime -from sqlalchemy import orm, func, Boolean, Column, DateTime, Integer, String, ForeignKey, ForeignKeyConstraint +from sqlalchemy import orm, func, BigInteger, Boolean, Column, DateTime, Integer, String, ForeignKey, ForeignKeyConstraint from sqlalchemy.orm import relationship from sqlalchemy.sql.expression import false, true -from libraries.database.database import DB + +from sqlalchemy.orm import relationship +from sqlalchemy import create_engine + from libraries.enum.auraenumerations import ScheduleEntryType +from aura import DB -class AuraDatabaseModel: + +class AuraDatabaseModel(): logger = None def __init__(self): @@ -69,7 +74,7 @@ class AuraDatabaseModel: # self.logger.debug("all dropped. creating...") DB.create_all() # self.logger.debug("inserting manual scheduling possibility and fallback trackservice schedule") - DB.session.add(manualschedule) +# DB.session.add(manualschedule) # db.session.add(fallback_trackservice_schedule) # self.logger.debug("all created. commiting...") DB.session.commit() @@ -79,6 +84,82 @@ class AuraDatabaseModel: sys.exit(0) +# ------------------------------------------------------------------------------------------ # +# class Schedule(DB.Model, AuraDatabaseModel): +# """ +# One specific Schedule for a show on a timeslot +# """ +# __tablename__ = 'schedule' +# +# # primary and foreign keys +# schedule_start = Column(DateTime, primary_key=True) +# +# schedule_end = Column(DateTime) +# schedule_id = Column(Integer) #, primary_key=True, autoincrement=False) +# show_id = Column(Integer) # well, in fact not needed.. +# show_name = Column(String(256)) +# show_hosts = Column(String(256)) +# funding_category = Column(String(256)) +# comment = Column(String(512)) +# languages = Column(String(256)) +# type = Column(String(256)) +# category = Column(String(256)) +# topic = Column(String(256)) +# musicfocus = Column(String(256)) +# +# is_repetition = Column(Boolean()) +# +# playlist_id = Column(Integer, ForeignKey("playlist.playlist_id")) +# timeslot_fallback_id = Column(Integer) +# show_fallback_id = Column(Integer) +# station_fallback_id = Column(Integer) +# +# playlist = relationship("Playlist", foreign_keys=[playlist_id], lazy="joined") +# # timeslot_fallback = relationship("Playlist", foreign_keys=[timeslot_fallback_id], lazy="joined") +# # show_fallback = relationship("Playlist", foreign_keys=[show_fallback_id], lazy="joined") +# # station_fallback = relationship("Playlist", foreign_keys=[station_fallback_id], lazy="joined") +# +# @staticmethod +# def select_all(): +# # fetching all entries +# all_entries = DB.session.query(Schedule).filter().order_by(Schedule.schedule_start).all() +# return all_entries +# +# @staticmethod +# def select_by_id(id): +# entry = DB.session.query(Schedule).filter(Schedule.schedule_id == id).first() +# return entry +# @staticmethod +# def select_act_programme(): +# #DB.session.query(Schedule).filter +# # fetching all from today to .. +# today = datetime.date.today() +# all_entries = DB.session.query(Schedule).filter(Schedule.schedule_start >= today).order_by(Schedule.schedule_start).all() +# +# return all_entries +# + +# +# @staticmethod +# def drop_the_future(timedelta): +# then = datetime.datetime.now() + timedelta +# +# # is this really necessary? +# future_entries = DB.session.query(Schedule).filter(Schedule.schedule_start > then) +# for e in future_entries: +# e.delete() +# DB.session.commit() +# +# def get_length(self): +# sec1 = int(datetime.datetime.strptime(self.start[0:16].replace(" ", "T"), "%Y-%m-%dT%H:%M").strftime("%s")) +# sec2 = int(datetime.datetime.strptime(self.end[0:16].replace(" ", "T"), "%Y-%m-%dT%H:%M").strftime("%s")) +# len = sec2 - sec1 +# return len +# +# # ------------------------------------------------------------------------------------------ # +# def __str__(self): +# return "ScheduleID: #" + str(self.schedule_id) + " Showname: " + self.show_name + " starts @ " + str(self.schedule_start) + # ------------------------------------------------------------------------------------------ # class Schedule(DB.Model, AuraDatabaseModel): """ @@ -87,11 +168,11 @@ class Schedule(DB.Model, AuraDatabaseModel): __tablename__ = 'schedule' # primary and foreign keys - schedule_id = Column(Integer, primary_key=True, autoincrement=False) - show_id = Column(Integer) # well, in fact not needed.. + schedule_start = Column(DateTime, primary_key=True) - schedule_start = Column(DateTime) # can be null due to manual entries - schedule_end = Column(DateTime) # can be null due to manual entries + schedule_end = Column(DateTime) + schedule_id = Column(Integer) #, primary_key=True, autoincrement=False) + show_id = Column(Integer) # well, in fact not needed.. show_name = Column(String(256)) show_hosts = Column(String(256)) funding_category = Column(String(256)) @@ -104,158 +185,48 @@ class Schedule(DB.Model, AuraDatabaseModel): is_repetition = Column(Boolean()) - playlist_id = Column(Integer) + playlist_id = Column(Integer) #, ForeignKey("playlist.playlist_id")) timeslot_fallback_id = Column(Integer) show_fallback_id = Column(Integer) station_fallback_id = Column(Integer) - @staticmethod - def select_all(): - # fetching all entries - all_entries = DB.session.query(Schedule).filter().order_by(Schedule.schedule_start).all() - return all_entries - - @staticmethod - def select_by_id(id): - entry = DB.session.query(Schedule).filter(Schedule.schedule_id == id).first() - return entry - - @staticmethod - def select_act_programme(): - #DB.session.query(Schedule).filter - # fetching all from today to .. - today = datetime.date.today() - all_entries = DB.session.query(Schedule).filter(Schedule.schedule_start >= today).order_by(Schedule.schedule_start).all() - - return all_entries + playlist = relationship("Playlist", + primaryjoin="and_(Schedule.schedule_start==Playlist.schedule_start, Schedule.playlist_id==Playlist.playlist_id, Schedule.show_name==Playlist.show_name)", + back_populates="schedule") + timeslot_fallback = relationship("Playlist", + primaryjoin="and_(Schedule.schedule_start==Playlist.schedule_start, Schedule.timeslot_fallback_id==Playlist.playlist_id, Schedule.show_name==Playlist.show_name)", + back_populates="schedule") + show_fallback = relationship("Playlist", + primaryjoin="and_(Schedule.schedule_start==Playlist.schedule_start, Schedule.show_fallback_id==Playlist.playlist_id, Schedule.show_name==Playlist.show_name)", + back_populates="schedule") + station_fallback = relationship("Playlist", + primaryjoin="and_(Schedule.schedule_start==Playlist.schedule_start, Schedule.station_fallback_id==Playlist.playlist_id, Schedule.show_name==Playlist.show_name)", + back_populates="schedule") @staticmethod def select_show_on_datetime(datetime): return DB.session.query(Schedule).filter(Schedule.schedule_start == datetime).first() - @staticmethod - def drop_the_future(timedelta): - then = datetime.datetime.now() + timedelta - - # is this really necessary? - future_entries = DB.session.query(Schedule).filter(Schedule.schedule_start > then) - for e in future_entries: - e.delete() - DB.session.commit() - - def get_length(self): - sec1 = int(datetime.datetime.strptime(self.start[0:16].replace(" ", "T"), "%Y-%m-%dT%H:%M").strftime("%s")) - sec2 = int(datetime.datetime.strptime(self.end[0:16].replace(" ", "T"), "%Y-%m-%dT%H:%M").strftime("%s")) - len = sec2 - sec1 - return len - - # ------------------------------------------------------------------------------------------ # - def __str__(self): - return "ScheduleID: #" + str(self.schedule_id) + " Showname: " + self.show_name + " starts @ " + str(self.schedule_start) - - # ------------------------------------------------------------------------------------------ # -class ScheduleEntry(DB.Model, AuraDatabaseModel): - """ - One schedule can have multiple entries - """ - __tablename__ = 'schedule_entry' +class Playlist(DB.Model, AuraDatabaseModel): + __tablename__ = 'playlist' + + # pk,fk + artificial_id = Column(Integer, primary_key=True) + schedule_start = Column(DateTime, ForeignKey("schedule.schedule_start")) + # relationships + schedule = relationship("Schedule", uselist=False, back_populates="playlist") + entries = relationship("PlaylistEntry", back_populates="playlist") + # data + playlist_id = Column(Integer, autoincrement=False) # , ForeignKey("schedule.playlist_id")) + show_name = Column(String(256)) + fallback_type = Column(Integer) + entry_count = Column(Integer) - # primary and foreign keys - playlist_id = Column(Integer, primary_key=True, nullable=False, autoincrement=False) - entry_num = Column(Integer, primary_key=True, nullable=False, autoincrement=False) - schedule_id = Column(Integer, ForeignKey("schedule.schedule_id")) - - entry_start = Column(DateTime) - source = Column(String(256)) - volume = Column(Integer, default=100) - fallback_type = Column(Integer, default=0) - cleansource = "" - cleanprotocol = "" - entry_start_unix = 0 - programme_index = -1 - type = None - fadeintimer = None - fadeouttimer = None - switchtimer = None - - # schedule = relationship("Schedule", foreign_keys=[schedule_id], lazy="joined") - - # normal constructor - def __init__(self, **kwargs): - super(ScheduleEntry, self).__init__(**kwargs) - self.calc_unix_times() - self.define_clean_source() - - # constructor like - called from sqlalchemy - @orm.reconstructor - def reconstructor(self): - self.calc_unix_times() - self.define_clean_source() - self.set_entry_type() - - def define_clean_source(self): - if self.source is None: - return None - - if self.source.startswith("http"): - self.cleanprotocol = self.source[:7] - self.cleansource = self.source - - elif self.source.startswith("linein"): - self.cleanprotocol = self.source[:9] - self.cleansource = self.source[9:] - - elif self.source.startswith("pool") or self.source.startswith("file") or self.source.startswith("live"): - self.cleanprotocol = self.source[:7] - self.cleansource = self.source[7:] - - elif self.source.startswith("playlist"): - self.cleanprotocol = self.source[:11] - self.cleansource = self.source[11:] - - else: - self.logger.error("Unknown source protocol") - - def calc_unix_times(self): - if self.entry_start is not None: - self.entry_start_unix = time.mktime(self.entry_start.timetuple()) - - def set_entry_type(self): - if self.source.startswith("http"): - self.type = ScheduleEntryType.STREAM - if self.source.startswith("pool") or self.source.startswith("playlist") or self.source.startswith("file"): - self.type = ScheduleEntryType.FILESYSTEM - if self.source.startswith("live") or self.source.startswith("linein"): - if self.cleansource == "0": - self.type = ScheduleEntryType.LIVE_0 - elif self.cleansource == "1": - self.type = ScheduleEntryType.LIVE_1 - elif self.cleansource == "2": - self.type = ScheduleEntryType.LIVE_2 - elif self.cleansource == "3": - self.type = ScheduleEntryType.LIVE_3 - elif self.cleansource == "4": - self.type = ScheduleEntryType.LIVE_4 - - # ------------------------------------------------------------------------------------------ # @staticmethod def select_all(): # fetching all entries - all_entries = DB.session.query(ScheduleEntry).filter(ScheduleEntry.fallback_type == 0).order_by(ScheduleEntry.entry_start).all() - - cnt = 0 - for entry in all_entries: - entry.programme_index = cnt - cnt = cnt + 1 - - return all_entries - - @staticmethod - def select_act_programme(include_act_playing = True): - # fetching all from today to .. - today = datetime.date.today() - all_entries = DB.session.query(ScheduleEntry).filter(ScheduleEntry.entry_start >= today, ScheduleEntry.fallback_type == 0).order_by(ScheduleEntry.entry_start).all() + all_entries = DB.session.query(Playlist).filter(Playlist.fallback_type == 0).all() cnt = 0 for entry in all_entries: @@ -264,115 +235,295 @@ class ScheduleEntry(DB.Model, AuraDatabaseModel): return all_entries - # ------------------------------------------------------------------------------------------ # @staticmethod - def truncate(): - all_entries = DB.session.query(ScheduleEntry).filter().order_by(ScheduleEntry.entry_start).all() + def select_playlist_for_schedule(datetime, playlist_id): + return DB.session.query(Playlist).filter(Playlist.schedule_start == datetime, Playlist.playlist_id == playlist_id).first() - for a in all_entries: - a.delete() - DB.session.commit() - - # ------------------------------------------------------------------------------------------ # - @staticmethod - def select_next_manual_entry_num(): - max_manual_entry_num = DB.session.query(func.max(ScheduleEntry.entry_num)).filter(ScheduleEntry.schedule_id == 0).first() +# ------------------------------------------------------------------------------------------ # +class PlaylistEntry(DB.Model, AuraDatabaseModel): + __tablename__ = 'playlist_entry' - if max_manual_entry_num[0] is None: - return 0 - else: - return int(max_manual_entry_num[0])+1 + # primary keys + artificial_id = Column(Integer, primary_key=True) - # ------------------------------------------------------------------------------------------ # - @staticmethod - def select_upcoming(datefrom=datetime.datetime.now()): - upcomingtracks = DB.session.query(ScheduleEntry).filter(ScheduleEntry.entry_start > datefrom).order_by(ScheduleEntry.entry_start).all() - return upcomingtracks + # foreign keys + artificial_playlist_id = Column(Integer, ForeignKey("playlist.artificial_id")) + entry_num = Column(Integer) # , primary_key=True) - # ------------------------------------------------------------------------------------------ # - @staticmethod - def select_one(playlist_id, entry_num): - return DB.session.query(ScheduleEntry).filter(ScheduleEntry.playlist_id == playlist_id, ScheduleEntry.entry_num == entry_num).first() + uri = Column(String(1024)) + duration = Column(BigInteger) + filename = Column(String(1024)) - # ------------------------------------------------------------------------------------------ # - @staticmethod - def select_one_playlist_entry_for_show(schedule_id, playlist_type, entry_num): - return DB.session.query(ScheduleEntry).filter(ScheduleEntry.schedule_id == schedule_id, ScheduleEntry.fallback_type == playlist_type, ScheduleEntry.entry_num == entry_num).first() + # relationships + playlist = relationship("Playlist", uselist=False, back_populates="entries") + meta_data = relationship("PlaylistEntryMetaData", back_populates="entry") - # ------------------------------------------------------------------------------------------ # @staticmethod - def select_playlist(playlist_id): - return DB.session.query(ScheduleEntry).filter(ScheduleEntry.playlist_id == playlist_id).order_by(ScheduleEntry.entry_start).all() + def select_playlistentry_for_playlist(artificial_playlist_id, entry_num): + return DB.session.query(PlaylistEntry).filter(PlaylistEntry.artificial_playlist_id == artificial_playlist_id, PlaylistEntry.entry_num == entry_num).first() - @staticmethod - def drop_the_future(timedelta): - then = datetime.datetime.now() + timedelta - #DB.session.delete(ScheduleEntry).filter(ScheduleEntry.entry_start >= then) - - # is this really necessary? - future_entries = DB.session.query(ScheduleEntry).filter(ScheduleEntry.entry_start > then) - for e in future_entries: - e.delete() - DB.session.commit() - def getChannel(self): - if self.type == self.type.FILESYSTEM: - return "fs" +class PlaylistEntryMetaData(DB.Model, AuraDatabaseModel): + __tablename__ = "playlist_entry_metadata" - if self.type == self.type.LIVE_0 or self.type == self.type.LIVE_1 or self.type == self.type.LIVE_2 or self.type == self.type.LIVE_3 or self.type == self.type.LIVE_4: - return "aura_linein_"+self.cleansource # .cleanprotocol[8] + artificial_id = Column(Integer, primary_key=True) + artificial_entry_id = Column(Integer, ForeignKey("playlist_entry.artificial_id")) - if self.type == self.type.STREAM: - return "http" + artist = Column(String(256)) + title = Column(String(256)) + album = Column(String(256)) + entry = relationship("PlaylistEntry", uselist=False, back_populates="meta_data") - # ------------------------------------------------------------------------------------------ # - def __str__(self): - return "Showentry starts @ " + str(self.entry_start) + " and plays " + self.source + @staticmethod + def select_metadata_for_entry(artificial_playlistentry_id): + return DB.session.query(PlaylistEntry).filter(PlaylistEntryMetaData.artificial_entry_id == artificial_playlistentry_id).first() # ------------------------------------------------------------------------------------------ # -class TrackService(DB.Model, AuraDatabaseModel): - __tablename__ = 'trackservice' - - trackservice_id = Column(Integer, primary_key=True, autoincrement=True) - playlist_id = Column(Integer, nullable=False) - entry_num = Column(Integer, nullable=False) - - source = Column(String(255), nullable=False) - start = Column(DateTime, nullable=False, default=func.now()) +#class PlaylistEntry(DB.Model, AuraDatabaseModel): + # __tablename__ = 'playlist_entry' + # + # # primary and foreign keys + # playlist_id = Column(Integer, ForeignKey("playlist.playlist_id"), primary_key=True, nullable=False, autoincrement=True) + # entry_num = Column(Integer, primary_key=True, nullable=False, autoincrement=False) + # + # uri = Column(String(1024)) + # + # filename = "" + # cleansource = "" + # cleanprotocol = "" + # type = None + # fadeintimer = None + # fadeouttimer = None + # switchtimer = None + # + # meta_data = relationship("PlaylistEntryMetaData", primaryjoin="and_(PlaylistEntry.playlist_id==PlaylistEntryMetaData.playlist_id, PlaylistEntry.entry_num==PlaylistEntryMetaData.entry_num)", lazy="joined") + # + # # normal constructor + # def __init__(self, **kwargs): + # super(PlaylistEntry, self).__init__(**kwargs) + # self.calc_unix_times() + # self.define_clean_source() + # + # # constructor like - called from sqlalchemy + # @orm.reconstructor + # def reconstructor(self): + # self.calc_unix_times() + # self.define_clean_source() + # self.set_entry_type() + # + # def define_clean_source(self): + # if self.uri is None: + # return None + # + # if self.uri.startswith("http"): + # self.cleanprotocol = self.uri[:7] + # self.cleansource = self.uri + # + # elif self.uri.startswith("linein"): + # self.cleanprotocol = self.uri[:9] + # self.cleansource = self.uri[9:] + # + # elif self.uri.startswith("pool") or self.uri.startswith("file") or self.uri.startswith("live"): + # self.cleanprotocol = self.uri[:7] + # self.cleansource = self.uri[7:] + # + # elif self.uri.startswith("playlist"): + # self.cleanprotocol = self.uri[:11] + # self.cleansource = self.uri[11:] + # + # else: + # self.logger.error("Unknown source protocol") + # + # def set_entry_type(self): + # if self.uri.startswith("http"): + # self.type = ScheduleEntryType.STREAM + # if self.uri.startswith("pool") or self.uri.startswith("playlist") or self.uri.startswith("file"): + # self.type = ScheduleEntryType.FILESYSTEM + # if self.uri.startswith("live") or self.uri.startswith("linein"): + # if self.cleansource == "0": + # self.type = ScheduleEntryType.LIVE_0 + # elif self.cleansource == "1": + # self.type = ScheduleEntryType.LIVE_1 + # elif self.cleansource == "2": + # self.type = ScheduleEntryType.LIVE_2 + # elif self.cleansource == "3": + # self.type = ScheduleEntryType.LIVE_3 + # elif self.cleansource == "4": + # self.type = ScheduleEntryType.LIVE_4 + + + + + # def calc_unix_times(self): + # if self.entry_start is not None: + # self.entry_start_unix = time.mktime(self.entry_start.timetuple()) + # + # + # + # # ------------------------------------------------------------------------------------------ # + # @staticmethod + # def select_all(): + # # fetching all entries + # all_entries = DB.session.query(Playlist).filter(Playlist.fallback_type == 0).order_by(Playlist.entry_start).all() + # + # cnt = 0 + # for entry in all_entries: + # entry.programme_index = cnt + # cnt = cnt + 1 + # + # return all_entries + # + # @staticmethod + # def select_act_programme(include_act_playing = True): + # # fetching all from today to .. + # today = datetime.date.today() + # all_entries = DB.session.query(Playlist).filter(Playlist.entry_start >= today, Playlist.fallback_type == 0).order_by(Playlist.entry_start).all() + # + # cnt = 0 + # for entry in all_entries: + # entry.programme_index = cnt + # cnt = cnt + 1 + # + # return all_entries + # + # # ------------------------------------------------------------------------------------------ # + # @staticmethod + # def truncate(): + # all_entries = DB.session.query(Playlist).filter().order_by(Playlist.entry_start).all() + # + # for a in all_entries: + # a.delete() + # DB.session.commit() + # + # # ------------------------------------------------------------------------------------------ # + # @staticmethod + # def select_next_manual_entry_num(): + # + # max_manual_entry_num = DB.session.query(func.max(Playlist.entry_num)).filter(Playlist.schedule_id == 0).first() + # + # if max_manual_entry_num[0] is None: + # return 0 + # else: + # return int(max_manual_entry_num[0])+1 + # + # # ------------------------------------------------------------------------------------------ # + # @staticmethod + # def select_upcoming(datefrom=datetime.datetime.now()): + # upcomingtracks = DB.session.query(Playlist).filter(Playlist.entry_start > datefrom).order_by(Playlist.entry_start).all() + # return upcomingtracks + # + # # ------------------------------------------------------------------------------------------ # + # @staticmethod + # def select_one(playlist_id, entry_num): + # return DB.session.query(Playlist).filter(Playlist.playlist_id == playlist_id, Playlist.entry_num == entry_num).first() + # + # # ------------------------------------------------------------------------------------------ # + # @staticmethod + # def select_one_playlist_entry_for_show(schedule_id, playlist_type, entry_num): + # return DB.session.query(Playlist).filter(Playlist.schedule_id == schedule_id, Playlist.fallback_type == playlist_type, Playlist.entry_num == entry_num).first() + # + # # ------------------------------------------------------------------------------------------ # + # @staticmethod + # def select_playlist(playlist_id): + # return DB.session.query(Playlist).filter(Playlist.playlist_id == playlist_id).order_by(Playlist.entry_start).all() + # + # @staticmethod + # def drop_the_future(timedelta): + # then = datetime.datetime.now() + timedelta + # #DB.session.delete(ScheduleEntry).filter(ScheduleEntry.entry_start >= then) + # + # # is this really necessary? + # future_entries = DB.session.query(Playlist).filter(Playlist.entry_start > then) + # for e in future_entries: + # e.delete() + # DB.session.commit() + # + # def getChannel(self): + # if self.type == self.type.FILESYSTEM: + # return "fs" + # + # if self.type == self.type.LIVE_0 or self.type == self.type.LIVE_1 or self.type == self.type.LIVE_2 or self.type == self.type.LIVE_3 or self.type == self.type.LIVE_4: + # return "aura_linein_"+self.cleansource # .cleanprotocol[8] + # + # if self.type == self.type.STREAM: + # return "http" + # + # + # # ------------------------------------------------------------------------------------------ # + # def __str__(self): + # return "Showentry starts @ " + str(self.entry_start) + " and plays " + self.source + + +# class ScheduleEntryFile(DB.Model, AuraDatabaseModel): +# __tablename__ = 'schedule_entry_file' +# +# # primary and foreign keys +# file_id = Column(Integer, primary_key=True, nullable=False, autoincrement=False) +# playlist_id = Column(Integer) #, ForeignKey("schedule_entry.playlist_id")) # primary_key=True, nullable=False, autoincrement=False) +# entry_num = Column(Integer) # , ForeignKey("schedule_entry.entry_num")) # primary_key=True, nullable=False, autoincrement=False) +# +# ForeignKeyConstraint(["playlist_id", "entry_num"], ["schedule_entry.playlist_id", "schedule_entry.entry_num"]) +# +# show = Column(String(512)) +# size = Column(Integer) +# duration = Column(Integer) +# +# class ScheduleEntryFileMetaData(DB.Model, AuraDatabaseModel): +# __tablename__ = "schedule_entry_file_metadata" +# +# metadata_id = Column(Integer, primary_key=True, nullable=False, autoincrement=True) +# file_id = Column(Integer, ForeignKey("schedule_entry_file.file_id")) +# +# artist = Column(String(256)) +# title = Column(String(256)) +# album = Column(String(256)) +# +# # ------------------------------------------------------------------------------------------ # +# class TrackService(DB.Model, AuraDatabaseModel): +# __tablename__ = 'trackservice' +# +# trackservice_id = Column(Integer, primary_key=True, autoincrement=True) +# schedule_entry_id = Column(Integer, ForeignKey("schedule_entry.id")) +# playlist_id = Column(Integer, nullable=False) +# entry_num = Column(Integer, nullable=False) +# +# source = Column(String(255), nullable=False) +# start = Column(DateTime, nullable=False, default=func.now()) - __table_args__ = ( - ForeignKeyConstraint(['playlist_id', 'entry_num'], ['schedule_entry.playlist_id', 'schedule_entry.entry_num']), - ) +# __table_args__ = ( +# ForeignKeyConstraint(['playlist_id', 'entry_num'], ['schedule_entry.playlist_id', 'schedule_entry.entry_num']), +# ) +# schedule_entry = relationship("ScheduleEntry", primaryjoin="and_(TrackService.playlist_id==ScheduleEntry.playlist_id, TrackService.entry_num==ScheduleEntry.entry_num)", lazy="joined") #schedule = relationship("Schedule", foreign_keys=[schedule_id], lazy="joined") # trackservice_entry = relationship("ScheduleEntry", foreign_keys=[playlist_id, entry_num], lazy="joined") - schedule_entry = relationship("ScheduleEntry", primaryjoin="and_(TrackService.playlist_id==ScheduleEntry.playlist_id, TrackService.entry_num==ScheduleEntry.entry_num)", lazy="joined") - - @staticmethod - # ------------------------------------------------------------------------------------------ # - def select_one(trackservice_id): - return DB.session.query(TrackService).filter(TrackService.trackservice_id == trackservice_id).first() - - @staticmethod - # ------------------------------------------------------------------------------------------ # - def select_by_day(day): - day_plus_one = day + datetime.timedelta(days=1) - tracks = DB.session.query(TrackService).filter(TrackService.start >= str(day), TrackService.start < str(day_plus_one)).all() - return tracks - - @staticmethod - # ------------------------------------------------------------------------------------------ # - def select_by_range(from_day, to_day): - tracks = DB.session.query(TrackService).filter(TrackService.start >= str(from_day), - TrackService.start < str(to_day)).all() - return tracks - - # ------------------------------------------------------------------------------------------ # - def __str__(self): - return "TrackServiceID: #" + str(self.trackservice_id) + " playlist_id: " + str(self.playlist_id) + " started @ " + str(self.start) + " and played " + self.source + # schedule_entry = relationship("ScheduleEntry", primaryjoin="and_(TrackService.schedule_entry_id==ScheduleEntry.id)", lazy="joined") + # + # @staticmethod + # # ------------------------------------------------------------------------------------------ # + # def select_one(trackservice_id): + # return DB.session.query(TrackService).filter(TrackService.trackservice_id == trackservice_id).first() + # + # @staticmethod + # # ------------------------------------------------------------------------------------------ # + # def select_by_day(day): + # day_plus_one = day + datetime.timedelta(days=1) + # tracks = DB.session.query(TrackService).filter(TrackService.start >= str(day), TrackService.start < str(day_plus_one)).all() + # return tracks + # + # @staticmethod + # # ------------------------------------------------------------------------------------------ # + # def select_by_range(from_day, to_day): + # tracks = DB.session.query(TrackService).filter(TrackService.start >= str(from_day), + # TrackService.start < str(to_day)).all() + # return tracks + # + # # ------------------------------------------------------------------------------------------ # + # def __str__(self): + # return "TrackServiceID: #" + str(self.trackservice_id) + " playlist_id: " + str(self.playlist_id) + " started @ " + str(self.start) + " and played " + self.source # ------------------------------------------------------------------------------------------ # # class TrackServiceSchedule(db.Model, AuraDatabaseModel): @@ -425,4 +576,4 @@ class TrackService(DB.Model, AuraDatabaseModel): # def select_all(): # return db.session.query(TrackServiceScheduleEntry).filter().all() -#AuraDatabaseModel.recreate_db(systemexit=True) +# AuraDatabaseModel.recreate_db(systemexit=True) diff --git a/libraries/database/database.py b/libraries/database/database.py deleted file mode 100644 index 56c93b19..00000000 --- a/libraries/database/database.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# engine -# -# Playout Daemon for autoradio project -# -# -# Copyright (C) 2017-2018 Gottfried Gaisbauer <gottfried.gaisbauer@servus.at> -# -# This file is part of engine. -# -# engine is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# any later version. -# -# engine is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with engine. If not, see <http://www.gnu.org/licenses/>. -# - -from sqlalchemy.ext.declarative import declarative_base -from flask_sqlalchemy import SQLAlchemy -# from flask_babel import Babel -from flask import Flask - -from modules.base.config import ConfigReader # pylint: disable=import-error - -def create_app(install_dir, uri): - """ - creates flask app context - :param install_dir: Installdir of Aura - :param uri: Database connection uri - :return: Flask object - """ - app = Flask(__name__, template_folder=install_dir + '/modules/web/templates') - app.config["SQLALCHEMY_DATABASE_URI"] = uri - app.config['BABEL_DEFAULT_LOCALE'] = 'de' - app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False - - return app - -def create_database(): - """ - creates sqlalchemy database connection - :return: SQLAlchemy object - """ - #### load config #### - config = ConfigReader() - config.load_config() - - #### read config #### - install_dir = config.get(str("install_dir")) - db_name = config.get(str("db_name")) - db_user = config.get(str("db_user")) - db_pass = config.get(str("db_pass")) - db_host = config.get(str("db_host")) - - #### create database conn #### - uri = "mysql://"+db_user+":"+db_pass+"@"+db_host+"/"+db_name+"?charset=utf8" - - app = create_app(install_dir, uri) - - database = SQLAlchemy(app) -# babel = Babel(app) - - return app, database - - -Base = declarative_base() -APP, DB = create_database() diff --git a/modules/base/config.py b/modules/base/config.py deleted file mode 100644 index c7726bb2..00000000 --- a/modules/base/config.py +++ /dev/null @@ -1,127 +0,0 @@ -# -# engine -# -# Playout Daemon for autoradio project -# -# Copyright 2014 BFR <info@freie-radios.de> -# Copyright (C) 2017-2018 Gottfried Gaisbauer <gottfried.gaisbauer@servus.at> -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; Version 3 of the License -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, the license can be downloaded here: -# -# http://www.gnu.org/licenses/gpl.html - -# Meta -__version__ = '0.1.1' -__license__ = "GNU General Public License (GPL) Version 3" -__version_info__ = (0, 1, 1) -__author__ = 'Michael Liebler <michael-liebler@radio-z.net>' - -""" -Aura Config Reader -""" -import os -import sys -import socket -import logging - -from configparser import ConfigParser - - -class ConfigReader(object): - ini_path = "" - logger = None - - def __init__(self): - self.logger = logging.getLogger("AuraEngine") - - def set(self, key, value): - """ - Set a property - @type key: string - @param key: The Key - @type value: mixed - @param value: Beliebiger Wert - """ - try: - self.__dict__[key] = int(value) - except: - self.__dict__[key] = str(value) - - # ------------------------------------------------------------------------------------------ # - def get(self, key, default=None): - """ - get a loaded property - @type key: string - @param key: Der Key - @type default: mixed - @param default: Beliebiger Wert - """ - - if key not in self.__dict__: - if default: - self.set(key, default) - else: - self.logger.warning("Key " + key + " not found in configfile " + self.ini_path + "!") - return None - - if key == "loglevel": - loglvl = self.__dict__[key] - - if loglvl == "debug": - return logging.DEBUG - elif loglvl == "info": - return logging.INFO - elif loglvl == "warning": - return logging.WARNING - elif loglvl == "error": - return logging.ERROR - else: - return logging.CRITICAL - - if key == "debug": - return self.__dict__[key].count("y") - - return self.__dict__[key] - - # ------------------------------------------------------------------------------------------ # - def load_config(self): - """ - Set config defaults and load settings from file - :return: - """ - self.ini_path = self.get('configpath', '/etc/aura/engine.ini') - - if not os.path.isfile(self.ini_path): - self.logger.critical(self.ini_path + " not found :(") - sys.exit(1) - - # INI einlesen - f = open(self.ini_path, 'r') - ini_str = f.read() - f.close() - - config_parser = ConfigParser() - try: - config_parser.read_string(ini_str) - except Exception as e: - self.logger.critical("Cannot read " + self.ini_path + "! Reason: " + str(e)) - sys.exit(0) - - for section in config_parser.sections(): - for key, value in config_parser.items(section): - v = config_parser.get(section, key).replace('"', '').strip() - self.set(key, v) - - self.set("install_dir", os.path.realpath(__file__ + "../../../..")) - - diff --git a/modules/communication/connection_tester.py b/modules/communication/connection_tester.py index fd597200..d946e656 100644 --- a/modules/communication/connection_tester.py +++ b/modules/communication/connection_tester.py @@ -27,7 +27,7 @@ import logging import json from modules.communication.liquidsoap.communicator import LiquidSoapCommunicator -from libraries.database.broadcasts import ScheduleEntry +#from libraries.database.broadcasts import ScheduleEntry from libraries.base.config import AuraConfig @@ -41,7 +41,7 @@ class ConnectionTester(AuraConfig): # ------------------------------------------------------------------------------------------ # def get_connection_status(self): status = dict() - status["db"] = self.test_db_conn() + status["db"] = False # self.test_db_conn() status["pv"] = self.test_pv_conn() status["lqs"] = self.test_lqs_conn() status["lqsr"] = False # self.test_lqsr_conn() @@ -51,13 +51,13 @@ class ConnectionTester(AuraConfig): return json.dumps(status) # ------------------------------------------------------------------------------------------ # - def test_db_conn(self): - try: - ScheduleEntry.select_all() - except: - return False - - return True +# def test_db_conn(self): +# try: +# ScheduleEntry.select_all() +# except: +# return False +# +# return True # ------------------------------------------------------------------------------------------ # def test_lqs_conn(self): diff --git a/modules/communication/liquidsoap/communicator.py b/modules/communication/liquidsoap/communicator.py index 40cfd0f0..b9ec5ae7 100644 --- a/modules/communication/liquidsoap/communicator.py +++ b/modules/communication/liquidsoap/communicator.py @@ -33,7 +33,7 @@ from modules.communication.mail.mail import AuraMailer from libraries.enum.auraenumerations import TerminalColors, ScheduleEntryType from libraries.exceptions.auraexceptions import LQConnectionError -from libraries.database.broadcasts import TrackService +#from libraries.database.broadcasts import TrackService from libraries.exceptions.exception_logger import ExceptionLogger """ @@ -306,7 +306,8 @@ class LiquidSoapCommunicator(ExceptionLogger): self.disable_transaction() # insert playlist entry - self.insert_track_service_entry(new_entry) + self.logger.critical("Trackservice entry not written here anymore") +# self.insert_track_service_entry(new_entry) except LQConnectionError as e: # we already caught and handled this error in __send_lqc_command__, but we do not want to execute this function further and pass the exception pass diff --git a/modules/communication/redis/adapter.py b/modules/communication/redis/adapter.py index b0ca8022..b89978a5 100644 --- a/modules/communication/redis/adapter.py +++ b/modules/communication/redis/adapter.py @@ -32,7 +32,7 @@ from datetime import datetime from threading import Event from modules.communication.redis.messenger import RedisMessenger -from modules.communication.connection_tester import ConnectionTester +# from modules.communication.connection_tester import ConnectionTester from libraries.database.statestore import RedisStateStore from libraries.exceptions.auraexceptions import RedisConnectionException from libraries.enum.auraenumerations import RedisChannel, TerminalColors, FallbackType @@ -47,7 +47,7 @@ class ServerRedisAdapter(threading.Thread, RedisMessenger): channel = "" scheduler = None redisclient = None - connection_tester = None +# connection_tester = None liquidsoapcommunicator = None # ------------------------------------------------------------------------------------------ # @@ -69,7 +69,7 @@ class ServerRedisAdapter(threading.Thread, RedisMessenger): self.adminMails = '' self.redisclient = ClientRedisAdapter(config) - self.connection_tester = ConnectionTester() +# self.connection_tester = ConnectionTester() # ------------------------------------------------------------------------------------------ # def run(self): @@ -157,8 +157,8 @@ class ServerRedisAdapter(threading.Thread, RedisMessenger): elif item["data"] == "get_act_programme": self.execute(RedisChannel.GAP_REPLY.value, self.scheduler.get_act_programme_as_string) - elif item["data"] == "get_connection_status": - self.execute(RedisChannel.GCS_REPLY.value, self.connection_tester.get_connection_status) +# elif item["data"] == "get_connection_status": +# self.execute(RedisChannel.GCS_REPLY.value, self.connection_tester.get_connection_status) elif item["data"] == "print_message_queue": self.execute(RedisChannel.PMQ_REPLY.value, self.scheduler.print_message_queue) diff --git a/modules/communication/redis/messenger.py b/modules/communication/redis/messenger.py index 190c4f68..be3bf587 100644 --- a/modules/communication/redis/messenger.py +++ b/modules/communication/redis/messenger.py @@ -107,7 +107,7 @@ class RedisMessenger(): @param section: Globale Sektion überschreiben """ section = self.section if section == '' else section - self.time = str(datetime.datetime.now().strftime('%Y-%m-%dT%H:%M:%S:%f')) + self.time = str(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S:%f')) self.utime = time.time() state = {'message':message.strip().replace("'","\\'"), 'code':self.errnr + str(code),'job':job,'value':value} self.rstore.set_section(section) diff --git a/modules/scheduling/calendar.py b/modules/scheduling/calendar.py index 1a5e06ba..f40b1965 100644 --- a/modules/scheduling/calendar.py +++ b/modules/scheduling/calendar.py @@ -34,9 +34,10 @@ import logging from mutagen.flac import FLAC from datetime import datetime, timedelta -from libraries.database.broadcasts import Schedule, ScheduleEntry +from libraries.database.broadcasts import Schedule, Playlist, PlaylistEntry, PlaylistEntryMetaData from libraries.enum.auraenumerations import ScheduleEntryType from modules.communication.redis.messenger import RedisMessenger +from modules.scheduling.calender_fetcher import CalendarFetcher class AuraCalendarService(threading.Thread): @@ -45,16 +46,15 @@ class AuraCalendarService(threading.Thread): playlistdir = "" xmlplaylist = range(0) - has_already_fetched = False queue = None config = None debug = False _stop_event = None logger = None + fetched_schedule_data = None url = dict() data = dict() - # another crutch because of the missing TANK - used_random_playlist_ids = list() + calendar_fetcher = None """ Fetching playlist data, write it into the database and notify service @@ -78,6 +78,8 @@ class AuraCalendarService(threading.Thread): self.__set_url__("calendar") self.__set_url__("importer") + self.calendar_fetcher = CalendarFetcher(config) + # ------------------------------------------------------------------------------------------ # def set_date_from(self, date): self.datefrom = str(date).replace(" ", "T") @@ -126,46 +128,43 @@ class AuraCalendarService(threading.Thread): """ try: - # fetch upcoming schedules from STEERING - self.logger.debug("Fetching schedules from STEERING") - self.__fetch_schedule_data__() - # fetch playlist and fallbacks to the schedules from TANK - self.logger.debug("Fetching playlists from TANK") - self.__fetch_schedule_entry_data__() - - # drop everything what is more than 30 minutes in the future to avoid strange sync errors - # the programme is still in the memory of engine and reloaded, when this fetching is finished. - self.drop_the_future(timedelta(minutes=30)) - - for schedule in self.fetched_schedule_data: - if "start" not in schedule: - self.logger.warning("No start of schedule given. skipping the schedule: "+str(schedule)) - continue - if "end" not in schedule: - self.logger.warning("No end of schedule given. skipping the schedule: "+str(schedule)) - continue - + fetched_schedule_data = self.calendar_fetcher.fetch() + + ret_schedule = [] + # for schedule in self.fetched_schedule_data: + # if "start" not in schedule: + # self.logger.warning("No start of schedule given. skipping the schedule: "+str(schedule)) + # continue + # if "end" not in schedule: + # self.logger.warning("No end of schedule given. skipping the schedule: "+str(schedule)) + # continue + + for schedule in fetched_schedule_data: # store the schedule schedule_db = self.store_schedule(schedule) # store playlists to play - self.store_schedule_playlist(schedule_db, schedule, "playlist") - self.store_schedule_playlist(schedule_db, schedule, "schedule_fallback", 1) - self.store_schedule_playlist(schedule_db, schedule, "show_fallback", 2) - self.store_schedule_playlist(schedule_db, schedule, "station_fallback", 3) + self.logger.warning("only storing playlist") + self.store_playlist(schedule_db, schedule_db.playlist_id, schedule["playlist"]) +# self.store_schedule_playlist(schedule_db, schedule, "schedule_fallback", 1) +# self.store_schedule_playlist(schedule_db, schedule, "show_fallback", 2) +# self.store_schedule_playlist(schedule_db, schedule, "station_fallback", 3) + + ret_schedule.append(schedule_db) # release the mutex - self.queue.put(schedule) #"fetching_finished") + self.queue.put(ret_schedule) except Exception as e: + # release the mutex self.queue.put("fetching_aborted " + str(e)) # terminate the thread return # ------------------------------------------------------------------------------------------ # - def drop_the_future(self, time_in_the_future): - ScheduleEntry.drop_the_future(time_in_the_future) - Schedule.drop_the_future(time_in_the_future) +# def drop_the_future(self, time_in_the_future): +# ScheduleEntry.drop_the_future(time_in_the_future) +# Schedule.drop_the_future(time_in_the_future) # ------------------------------------------------------------------------------------------ # def store_schedule(self, schedule): @@ -195,6 +194,8 @@ class AuraCalendarService(threading.Thread): schedule_db.topic = schedule["show_topics"] schedule_db.musicfocus = schedule["show_musicfocus"] + if schedule["playlist_id"] is None: + schedule["playlist_id"] = 1 schedule_db.playlist_id = schedule["playlist_id"] schedule_db.schedule_fallback_id = schedule["schedule_fallback_id"] schedule_db.show_fallback_id = schedule["show_fallback_id"] @@ -205,41 +206,76 @@ class AuraCalendarService(threading.Thread): return schedule_db # ------------------------------------------------------------------------------------------ # - def store_schedule_playlist(self, schedule_db, schedule, playlistname, fallbackplaylist_type=0): - playlist = schedule[playlistname] + def store_playlist(self, schedule_db, playlist_id, fetched_playlist, fallbackplaylist_type=0): + playlist_db = Playlist.select_playlist_for_schedule(schedule_db.schedule_start, playlist_id) + havetoadd = False + if not playlist_db: + playlist_db = Playlist() + havetoadd = True - debug = "Schedule playlist (" + playlistname + ") for " + schedule_db.show_name + " stored" - warning = "No scheduleentries for playlist #" + str(playlist['playlist_id']) + " in schedule #" + str(schedule_db.schedule_id) + " found" - entrynum = 0 + playlist_db.playlist_id = playlist_id + playlist_db.schedule_start = schedule_db.schedule_start + playlist_db.show_name = schedule_db.show_name + playlist_db.fallback_type = fallbackplaylist_type + playlist_db.entry_count = len(fetched_playlist["entries"]) - if "entries" in playlist: - lastentry = None + playlist_db.store(havetoadd, True) - for entry in playlist["entries"]: - lastentry = self.store_playlist_entry(schedule_db, playlist, entry, lastentry, entrynum, fallbackplaylist_type) - entrynum = entrynum + 1 + self.store_playlist_entries(playlist_db, fetched_playlist) - if lastentry is None: - self.logger.warning(warning) - else: - self.logger.debug(debug) - else: - self.logger.warning(warning) + return playlist_db + + + def store_playlist_entries(self, playlist_db, fetched_playlist): + entry_num = 0 + for entry in fetched_playlist["entries"]: + playlistentry_db = PlaylistEntry.select_playlistentry_for_playlist(playlist_db.artificial_id, entry_num) + havetoadd = False + if not playlistentry_db: + playlistentry_db = PlaylistEntry() + havetoadd = True + + playlistentry_db.artificial_playlist_id = playlist_db.artificial_id + playlistentry_db.entry_num = entry_num + playlistentry_db.uri = entry["uri"] + playlistentry_db.filename = entry["filename"] + playlistentry_db.duration = entry["file"]["duration"] + + playlistentry_db.store(havetoadd, True) + + self.store_playlist_entry_metadata(playlistentry_db, entry["file"]["metadata"]) + + entry_num = entry_num + 1 + + def store_playlist_entry_metadata(self, playlistentry_db, metadata): + playlistentrymetadata_db = PlaylistEntryMetaData.select_metadata_for_entry(playlistentry_db.artificial_id) + havetoadd = False + if not playlistentrymetadata_db: + playlistentrymetadata_db = PlaylistEntryMetaData() + havetoadd = True + + playlistentrymetadata_db.artificial_entry_id = playlistentry_db.artificial_id + playlistentrymetadata_db.artist = metadata["artist"] + playlistentrymetadata_db.title = metadata["title"] + playlistentrymetadata_db.album = metadata["album"] + + playlistentrymetadata_db.store(havetoadd, True); # ------------------------------------------------------------------------------------------ # def store_playlist_entry(self, schedule_db, playlist, entry, lastentry, entrynum, fallbackplaylist_type=0): - schedule_entry_db = ScheduleEntry.select_one_playlist_entry_for_show(schedule_db.schedule_id, fallbackplaylist_type, entrynum) + schedule_entry_db = Playlist.select_one().select_one_playlist_entry_for_show(schedule_db.schedule_id, fallbackplaylist_type, entrynum) havetoadd = False if not schedule_entry_db: - self.logger.debug("no scheduleentry with id " + str(playlist["playlist_id"]) + " and pos " + str(entrynum) + " in database => creating a new one") + self.logger.debug("no scheduleentry with id " + str(playlist["id"]) + " and pos " + str(entrynum) + " in database => creating a new one") schedule_entry_db = ScheduleEntry() havetoadd = True - schedule_entry_db.playlist_id = playlist["playlist_id"] + + schedule_entry_db.playlist_id = playlist["id"] schedule_entry_db.entry_num = entrynum schedule_entry_db.schedule_id = schedule_db.schedule_id - schedule_entry_db.source = entry["source"] + schedule_entry_db.uri = entry["uri"] schedule_entry_db.fallback_type = fallbackplaylist_type schedule_entry_db.entry_start = schedule_db.schedule_start + timedelta(seconds=self.get_length(lastentry)) @@ -247,7 +283,7 @@ class AuraCalendarService(threading.Thread): if havetoadd: schedule_entry_db.define_clean_source() - self.logger.debug("Storing entries... playlist_id: " + str(playlist["playlist_id"]) + " schedule_id: " + str(schedule_db.schedule_id) + " num: " + str(entrynum)) + self.logger.debug("Storing entries... playlist_id: " + str(playlist["id"]) + " schedule_id: " + str(schedule_db.schedule_id) + " num: " + str(entrynum)) schedule_entry_db.store(add=havetoadd, commit=True) @@ -286,204 +322,6 @@ class AuraCalendarService(threading.Thread): sec2 = int(datetime.strptime(end[0:16].replace(" ","T"),"%Y-%m-%dT%H:%M").strftime("%s")); return (sec2 - sec1); - # ------------------------------------------------------------------------------------------ # - def __fetch_schedule_entry_data__(self): - # store fetched entries => do not have to fetch playlist_id more than once - fetched_entries=[] - - try: - for schedule in self.fetched_schedule_data: - # retrieve playlist and the fallbacks for every schedule - # if a playlist (like station_fallback) is already fetched, it is not fetched again but reused - schedule["playlist"] = self.__fetch_schedule_entries__(schedule, "playlist_id", fetched_entries) - schedule["schedule_fallback"] = self.__fetch_schedule_entries__(schedule, "schedule_fallback_id", fetched_entries) - schedule["show_fallback"] = self.__fetch_schedule_entries__(schedule, "show_fallback_id", fetched_entries) - schedule["station_fallback"] = self.__fetch_schedule_entries__(schedule, "station_fallback_id", fetched_entries) - - self.logger.info(str(schedule)) - - except Exception as e: - self.logger.error(str(e)) - - # ------------------------------------------------------------------------------------------ # - def __fetch_schedule_entries__(self, schedule, id_name, fetched_schedule_entries): - servicetype = "importer" - use_testdata = False - - # fetch data from importer - json_response = self.__fetch_data__(servicetype) - if not json_response and self.config.get("use_test_data"): - use_testdata = True - - # if a playlist is already fetched, do not fetch it again - for entry in fetched_schedule_entries: - if entry["playlist_id"] == schedule[id_name]: - self.logger.debug("playlist #" + str(schedule[id_name]) + " already fetched") - return entry - - # generate testdata - if use_testdata: - json_response = self.create_test_data(id_name, schedule) - - # convert to list - try: - schedule_entries = json.loads(json_response) - except Exception as e: - self.logger.critical("Cannot convert playlist from importer into list") - schedule_entries = list() - - if "entries" in schedule_entries: - for entry in schedule_entries["entries"]: - if entry["source"].startswith("file"): - e = entry["source"][7:] # filter file:// out - if not os.path.isfile(e): - self.logger.warning("File", e, "does not exist!") - - fetched_schedule_entries.append(schedule_entries) - - return schedule_entries - - def create_test_data(self, id_name, schedule): - import random - rand_id = random.randint(1, 10000) - - while rand_id in self.used_random_playlist_ids: - rand_id = random.randint(1, 10000) - - self.used_random_playlist_ids.append(rand_id) - - # HARDCODED Testdata - if id_name != "playlist_id": - # FALLBACK TESTDATA - - if rand_id % 3 == 0: # playlist fallback - json_response = '{"playlist_id":' + str( - rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}' - elif rand_id % 2 == 0: # stream fallback - json_response = '{"playlist_id":' + str( - rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}' - else: # pool fallback - json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}' - - schedule[id_name] = rand_id - - elif schedule[id_name] == 0 or schedule[id_name] is None: - # this happens when playlist id is not filled out in pv - # json_response = '{"playlist_id": 0}' - - if rand_id % 4 == 0: # playlist with two files - json_response = '{"playlist_id":' + str( - rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}' - elif rand_id % 3 == 0: # playlist with jingle and then linein - json_response = '{"playlist_id":' + str( - rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}' - elif rand_id % 2 == 0: # playlist with jingle and then http stream - json_response = '{"playlist_id":' + str( - rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}' - else: # pool playlist - json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}' - - schedule[id_name] = rand_id - - elif schedule[id_name] % 4 == 0: # playlist with two files - json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}' - elif schedule[id_name] % 3 == 0: # playlist with jingle and then http stream - json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}' - elif schedule[id_name] % 2 == 0: # playlist with jingle and then linein - json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}' - else: # pool playlist - json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}' - - self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"]) - - return json_response - - # ------------------------------------------------------------------------------------------ # - def __fetch_schedule_data__(self): - servicetype = "calendar" - use_testdata = False - - html_response = self.__fetch_data__(servicetype) - if not html_response or html_response == b"[]": - self.logger.debug("Got no response: Using testdata") - use_testdata = True - - # if an error occours => use testdata - if use_testdata: - html_response = '[{"schedule_id":1,"start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%d %H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%d %H:00:00') + '","show_id":9,"show_name":"FROzine","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"},{"schedule_id":2,"schedule_start":"' + (datetime.now()+timedelta(hours=1)).strftime('%Y-%m-%d %H:00:00') + '","schedule_end":"' + (datetime.now()+timedelta(hours=2)).strftime('%Y-%m-%d %H:00:00') + '","show_id":10,"show_name":"FROMat","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"},{"schedule_id":3,"schedule_start":"' + (datetime.now()+timedelta(hours=2)).strftime('%Y-%m-%d %H:00:00') + '","schedule_end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%d %H:00:00') + '","show_id":11,"show_name":"Radio für Senioren","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"}]' - - try: - schedule_from_pv = json.loads(html_response) - except Exception as e: - self.logger.critical("Cannot fetch schedule entries from PV") - sys.exit() - - # check data - self.logger.critical("Hardcoded Response && no JSON data checks. I believe what i get here") - - d = self.remove_data_more_than_24h_in_the_future(schedule_from_pv) - self.fetched_schedule_data = self.remove_data_in_the_past(d) - - return self.fetched_schedule_data - - # ------------------------------------------------------------------------------------------ # - def remove_data_more_than_24h_in_the_future(self, schedule_from_pv): - act_list = [] - now = datetime.now() - now_plus_24hours = now + timedelta(hours=24) - - for s in schedule_from_pv: - date_start = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S") - - # append only elements which are close enough to now - if date_start <= now_plus_24hours and date_start >= now - timedelta(hours=1): - act_list.append(s) - - return act_list - - # ------------------------------------------------------------------------------------------ # - def remove_data_in_the_past(self, schedule_from_pv): - act_list = [] - now = datetime.now() - - for index,curr in enumerate(schedule_from_pv[:-1]): - date_start = datetime.strptime(curr["start"], "%Y-%m-%dT%H:%M:%S") - date_next_start = datetime.strptime(schedule_from_pv[index+1]["start"], "%Y-%m-%dT%H:%M:%S") - - # append all elements in the future - if date_start >= now: - act_list.append(curr) - # append the one which is now playing - if date_start <= now and date_next_start >= now: - act_list.append(curr) - - return act_list - - # ------------------------------------------------------------------------------------------ # - def __fetch_data__(self, type): - # init html_response - html_response = "" - - # open an url and read the data - try: - if type not in self.data: - if self.url[type] == "": - return False - request = urllib.request.Request(self.url[type]) - else: - request = urllib.request.Request(self.url[type], self.data[type]) - - response = urllib.request.urlopen(request) - html_response = response.read() - - except (urllib.error.URLError, IOError, ValueError) as e: - self.logger.error("Cannot connect to " + self.url[type] + "! reason: " + str(e.reason)) - if not self.has_already_fetched: # first fetch - sys.exit() - - self.has_already_fetched = True - return html_response - # ------------------------------------------------------------------------------------------ # def get_length(self, entry): if entry is None or entry.type == ScheduleEntryType.STREAM or entry.type == ScheduleEntryType.LIVE_0 or entry.type == ScheduleEntryType.LIVE_1 or entry.type == ScheduleEntryType.LIVE_2 or entry.type == ScheduleEntryType.LIVE_3 or entry.type == ScheduleEntryType.LIVE_4: diff --git a/modules/scheduling/calender_fetcher.py b/modules/scheduling/calender_fetcher.py new file mode 100644 index 00000000..e3be5fe6 --- /dev/null +++ b/modules/scheduling/calender_fetcher.py @@ -0,0 +1,269 @@ +import os +import sys +import urllib +import logging +import simplejson + +from datetime import datetime, timedelta +#from modules.models.schedule import Schedule + + +class CalendarFetcher: + url = dict() + url_parameter = dict() + config = None + logging = None + has_already_fetched = False + fetched_schedule_data = None + # another crutch because of the missing TANK + used_random_playlist_ids = list() + + def __init__(self, config): + self.config = config + self.logger = logging.getLogger("AuraEngine") + self.__set_url__("calendar") + self.__set_url__("importer") + + def fetch(self): + # fetch upcoming schedules from STEERING + try: + self.logger.debug("Fetching schedules from STEERING") + self.__fetch_schedule_data__() + except urllib.error.HTTPError as e: + self.logger.critical("Cannot fetch from " + self.url["calendar"] + "! Reason: " + str(e)) + except (urllib.error.URLError, IOError, ValueError) as e: + self.logger.critical("Cannot connect to " + self.url["calendar"] + "! Reason: " + str(e)) + + # fetch playlist and fallbacks to the schedules from TANK + try: + self.logger.debug("Fetching playlists from TANK") + self.__fetch_schedule_playlists__() + except urllib.error.HTTPError as e: + self.logger.critical("Cannot fetch from " + self.url["importer"] + "! Reason: " + str(e)) + except (urllib.error.URLError, IOError, ValueError) as e: + self.logger.critical("Cannot connect to " + self.url["importer"] + "! Reason: " + str(e)) + + return_data = [] + # gather returndata + try: + for schedule in self.fetched_schedule_data: + # skip schedule if no start or end is given + if "start" not in schedule: + self.logger.warning("No start of schedule given. skipping schedule: " + str(schedule)) + schedule = None + if "end" not in schedule: + self.logger.warning("No end of schedule given. skipping schedule: " + str(schedule)) + schedule = None + if "playlist" not in schedule: + self.logger.warning("No playlist for schedule given. skipping schedule: " + str(schedule)) + schedule = None + + if schedule: + return_data.append(schedule) + except TypeError as e: + self.logger.error("Nothing fetched...") + + return return_data + + # ------------------------------------------------------------------------------------------ # + def __set_url__(self, type): + url = self.config.get(type+"url") + pos = url.find("?") + + if pos > 0: + self.url[type] = url[0:pos] + self.url_parameter[type] = url[pos:] + else: + self.url[type] = url + + # ------------------------------------------------------------------------------------------ # + def __fetch_schedule_data__(self): + schedule = None + + # fetch data from steering + html_response = self.__fetch_data__(self.url["calendar"]) + + # response fails or is empty + if not html_response: + self.logger.debug("Got no response from pv!") + + # use testdata if wanted + if self.config.get("use_test_data"): + html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \ + '{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \ + '{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]' + self.logger.critical("Hardcoded Response") + else: + html_response = "{}" + + # convert to dict + schedule = simplejson.loads(html_response) + + # check data + self.logger.critical("no JSON data checks. I believe what i get here") + + self.fetched_schedule_data = self.remove_unnecessary_data(schedule) + + # ------------------------------------------------------------------------------------------ # + def __fetch_schedule_playlists__(self): + # store fetched entries => do not have to fetch playlist_id more than once + fetched_entries=[] + + self.logger.warning("only fetching normal playlists. no fallbacks") + for schedule in self.fetched_schedule_data: + # retrieve playlist and the fallbacks for every schedule + # if a playlist (like station_fallback) is already fetched, it is not fetched again but reused + schedule["playlist"] = self.__fetch_schedule_playlist__(schedule, "playlist_id", fetched_entries) +# schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, "schedule_fallback_id", fetched_entries) +# schedule["show_fallback"] = self.__fetch_schedule_playlist__(schedule, "show_fallback_id", fetched_entries) +# schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, "station_fallback_id", fetched_entries) + + # ------------------------------------------------------------------------------------------ # + def __fetch_schedule_playlist__(self, schedule, id_name, fetched_schedule_entries): + # set playlist_id (in testenvironment always null => no idea) + if id_name not in schedule or schedule[id_name] is None: + playlist_id = 1 + else: + playlist_id = schedule[id_name] + + # set url + #url = self.url["importer"] + schedule["show_name"] + "/playlists/" + str(playlist_id) + url = self.url["importer"] + "public" + "/playlists/" + str(playlist_id) + + # fetch data from importer + json_response = self.__fetch_data__(url) + + # use testdata if wanted + if not json_response and self.config.get("use_test_data"): + json_response = self.create_test_data(id_name, schedule) + + # convert to list + schedule_entries = simplejson.loads(json_response) + + if "entries" in schedule_entries: + for entry in schedule_entries["entries"]: + if entry["uri"].startswith("file"): + entry["filename"] = self.convert_to_filename(entry["uri"]) + + fetched_schedule_entries.append(schedule_entries) + + return schedule_entries + + def convert_to_filename(self, uri): + # convert to normal filename + e = self.config.get("audiofolder") + "/" + uri[7:] + ".flac" + if not os.path.isfile(e): + self.logger.warning("File", e, "does not exist!") + return e + + # ------------------------------------------------------------------------------------------ # + def __fetch_data__(self, url, parameter = ""): + # init html_response + html_response = b'' + + # open an url and read the data + if parameter == "": + request = urllib.request.Request(url) + else: + request = urllib.request.Request(url, parameter) + + response = urllib.request.urlopen(request) + html_response = response.read() + + self.has_already_fetched = True + return html_response.decode("utf-8") + + # ------------------------------------------------------------------------------------------ # + def remove_unnecessary_data(self, schedule): + reduced_schedule = self.remove_data_more_than_24h_in_the_future(self.remove_data_in_the_past(schedule)) + return reduced_schedule + + # ------------------------------------------------------------------------------------------ # + def remove_data_more_than_24h_in_the_future(self, schedule_from_pv): + act_list = [] + now = datetime.now() + now_plus_24hours = now + timedelta(hours=24) + + for s in schedule_from_pv: + date_start = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S") + + # append only elements which are close enough to now + if date_start <= now_plus_24hours and date_start >= now - timedelta(hours=1): + act_list.append(s) + + return act_list + + # ------------------------------------------------------------------------------------------ # + def remove_data_in_the_past(self, schedule_from_pv): + act_list = [] + now = datetime.now() + + for index,curr in enumerate(schedule_from_pv[:-1]): + print(curr["start"]) + date_start = datetime.strptime(curr["start"], "%Y-%m-%dT%H:%M:%S") + date_next_start = datetime.strptime(schedule_from_pv[index+1]["start"], "%Y-%m-%dT%H:%M:%S") + + # append all elements in the future + if date_start >= now: + act_list.append(curr) + # append the one which is now playing + if date_start <= now and date_next_start >= now: + act_list.append(curr) + + return act_list + + # ------------------------------------------------------------------------------------------ # + def create_test_data(self, id_name, schedule): + import random + rand_id = random.randint(1, 10000) + + while rand_id in self.used_random_playlist_ids: + rand_id = random.randint(1, 10000) + + self.used_random_playlist_ids.append(rand_id) + + # HARDCODED Testdata + if id_name != "playlist_id": + # FALLBACK TESTDATA + + if rand_id % 3 == 0: # playlist fallback + json_response = '{"playlist_id":' + str( + rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}' + elif rand_id % 2 == 0: # stream fallback + json_response = '{"playlist_id":' + str( + rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}' + else: # pool fallback + json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}' + + schedule[id_name] = rand_id + + elif schedule[id_name] == 0 or schedule[id_name] is None: + # this happens when playlist id is not filled out in pv + # json_response = '{"playlist_id": 0}' + + if rand_id % 4 == 0: # playlist with two files + json_response = '{"playlist_id":' + str( + rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}' + elif rand_id % 3 == 0: # playlist with jingle and then linein + json_response = '{"playlist_id":' + str( + rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}' + elif rand_id % 2 == 0: # playlist with jingle and then http stream + json_response = '{"playlist_id":' + str( + rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}' + else: # pool playlist + json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}' + + schedule[id_name] = rand_id + + elif schedule[id_name] % 4 == 0: # playlist with two files + json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}' + elif schedule[id_name] % 3 == 0: # playlist with jingle and then http stream + json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}' + elif schedule[id_name] % 2 == 0: # playlist with jingle and then linein + json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}' + else: # pool playlist + json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}' + + self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"]) + + return json_response \ No newline at end of file diff --git a/modules/scheduling/scheduler.py b/modules/scheduling/scheduler.py index 60026856..d544a08a 100644 --- a/modules/scheduling/scheduler.py +++ b/modules/scheduling/scheduler.py @@ -46,7 +46,7 @@ from operator import attrgetter from modules.communication.redis.messenger import RedisMessenger from modules.scheduling.calendar import AuraCalendarService -from libraries.database.broadcasts import Schedule, ScheduleEntry, AuraDatabaseModel +from libraries.database.broadcasts import Schedule, Playlist, AuraDatabaseModel from libraries.exceptions.exception_logger import ExceptionLogger from libraries.enum.auraenumerations import ScheduleEntryType, TimerType @@ -94,6 +94,10 @@ class AuraScheduler(ExceptionLogger, threading.Thread): @param config: read engine.ini """ self.config = config + + # init database ? + self.init_database() + self.redismessenger = RedisMessenger(config) self.logger = logging.getLogger("AuraEngine") @@ -110,9 +114,6 @@ class AuraScheduler(ExceptionLogger, threading.Thread): self.error_data = json.load(f) f.close() - # init database ? - self.init_database() - #self.redismessenger.send('Scheduler started', '0000', 'success', 'initApp', None, 'appinternal') # create exit event @@ -123,9 +124,12 @@ class AuraScheduler(ExceptionLogger, threading.Thread): # ------------------------------------------------------------------------------------------ # def init_database(self): + if self.config.get("recreate_db") is not None: + AuraDatabaseModel.recreate_db(systemexit=True) + # check if tables do exist. if not create them try: - ScheduleEntry.select_all() + Playlist.select_all() except sqlalchemy.exc.ProgrammingError as e: errcode = e.orig.args[0] @@ -191,13 +195,13 @@ class AuraScheduler(ExceptionLogger, threading.Thread): for schedule in self.programme: # playlist to play - schedule.playlist = ScheduleEntry.select_playlist(schedule.playlist_id) + schedule.playlist = Playlist.select_playlist(schedule.playlist_id) # show fallback is played when playlist fails - schedule.showfallback = ScheduleEntry.select_playlist(schedule.show_fallback_id) + schedule.showfallback = Playlist.select_playlist(schedule.show_fallback_id) # timeslot fallback is played when show fallback fails - schedule.timeslotfallback = ScheduleEntry.select_playlist(schedule.timeslot_fallback_id) + schedule.timeslotfallback = Playlist.select_playlist(schedule.timeslot_fallback_id) # station fallback is played when timeslot fallback fails - schedule.stationfallback = ScheduleEntry.select_playlist(schedule.station_fallback_id) + schedule.stationfallback = Playlist.select_playlist(schedule.station_fallback_id) for p in schedule.playlist: planned_entries.append(p) @@ -358,8 +362,12 @@ class AuraScheduler(ExceptionLogger, threading.Thread): # wait for the end response = queue.get() - if type(response) is dict: - self.load_programme_from_db() + if response is None: + self.logger.critical("Got an EMPTY response from AuraCalendarService: " + str(response)) + elif type(response) is list: + self.logger.critical("not loading from db") + self.programme = response + # self.load_programme_from_db() if self.programme is not None and len(self.programme) > 0: self.tried_fetching = 0 diff --git a/modules/web/routes.py b/modules/web/routes.py index fee80aff..fcd7377e 100644 --- a/modules/web/routes.py +++ b/modules/web/routes.py @@ -21,154 +21,156 @@ # You should have received a copy of the GNU General Public License # along with engine. If not, see <http://www.gnu.org/licenses/>. # - -import json -import decimal -import traceback -import sqlalchemy -import datetime -import logging - -from flask import request, render_template - -from libraries.database.database import APP -from libraries.database.broadcasts import TrackService, Schedule, ScheduleEntry - - - -def alchemyencoder(obj): - """JSON encoder function for SQLAlchemy special classes.""" - if isinstance(obj, datetime.date): - return obj.isoformat() - elif isinstance(obj, decimal.Decimal): - return float(obj) - elif isinstance(obj, sqlalchemy.orm.state.InstanceState): - return "" - elif isinstance(obj, Schedule): - return json.dumps([obj._asdict()], default=alchemyencoder) - else: - return str(obj) - - -class Routes: - error = None - scheduler = None - messenger = None - lqs_communicator = None - - def __init__(self, scheduler, lqs_communicator, messenger, config): - self.scheduler = scheduler - self.messenger = messenger - self.lqs_communicator = lqs_communicator - - # when debug is enabled => logging messages appear twice - APP.run(port=config.get("web_port")) #debug=True) - - @staticmethod - @APP.route('/') - @APP.route('/index') - def index(): - return render_template("index.html") - - @staticmethod - @APP.route("/trackservice", methods=["GET"]) - def track_service(): - from_time = request.args.get("from") - to_time = request.args.get("to") - last = request.args.get("last") - now = request.args.get("now") - - # nothing set => today's playlist - if from_time == None and to_time == None and now == None: - selected_date = datetime.date.today() - trackservice_entries = TrackService.select_by_day(selected_date) - - # from and end time set - elif from_time is not None and to_time is not None: - to_time = datetime.datetime.strptime(to_time, "%Y-%m-%d") - from_time = datetime.datetime.strptime(from_time, "%Y-%m-%d") - trackservice_entries = TrackService.select_by_range(from_time, to_time) - - # now set - elif now == "": - datetime.date.today() - trackservice_entries = TrackService.now_playing() - - return render_template("trackservice.html", - length=len(trackservice_entries), - trackservice_entries=trackservice_entries, - selected_date=selected_date) - - @staticmethod - @APP.route("/test") - def test(): - return render_template("index2.html") - - @staticmethod - @APP.route("/login") - def login(): - return "login" - #return render_template("index.html") - - @staticmethod - @APP.route("/logout") - def logout(): - #session.pop("logged_in", None) - return "logout" - #return render_template("index.html") - - @staticmethod - @APP.route("/api/v1/trackservice/<selected_date>", methods=["GET"]) - def api_trackservice(selected_date): - try: - # convert date - selected_date = datetime.datetime.strptime(selected_date, "%Y-%m-%d").date() - # select from database - tracks_on_selected_date = TrackService.select_by_day(selected_date) - # return as json - return json.dumps([tracks._asdict() for tracks in tracks_on_selected_date], default=alchemyencoder) - except Exception as e: - import traceback - traceback.print_exc() - - error = "Cannot transform programme into JSON String. Reason: " + str(e) - - logger = logging.getLogger("AuraEngine") - logger.error(error) - - return json.dumps({"Error": error}) - - @staticmethod - @APP.route("/api/v1/soundserver_state", methods=["GET"]) - def soundserver_settings(): - from modules.communication.liquidsoap.communicator import LiquidSoapCommunicator - from modules.base.config import ConfigReader - - try: - cr = ConfigReader() - cr.load_config() - lqs = LiquidSoapCommunicator(cr) - return lqs.auraengine_state() - except Exception as e: - error = "Unable to fetch state from Liquidsoap. Is Soundserver running? Reason: " + str(e) - logger = logging.getLogger("AuraEngine") - logger.error(error) - return json.dumps({"Error": error}) - - - @staticmethod - @APP.route("/api/v1/trackservice/", methods=["GET"]) - def api_trackservice_now(): - return json.dumps({'reached': True}) - - @staticmethod - @APP.route("/api/v1/upcoming/", methods=["GET"]) - def api_clock(): - servertime = datetime.datetime.now() - # get upcoming tracks - upcoming = ScheduleEntry.select_upcoming() - # convert to json string - upcoming_as_json = json.dumps([tracks._asdict() for tracks in upcoming], default=alchemyencoder) - # add servertime and return it - return upcoming_as_json.replace('[{', '[{"servertime":'+str(servertime)+"},{", 1) +# +# import json +# import decimal +# import traceback +# import sqlalchemy +# import datetime +# import logging +# +# from flask import request, render_template +# +# from aura import app +# #from libraries.database.broadcasts import TrackService, Schedule, ScheduleEntry +# +# +# +# def alchemyencoder(obj): +# """JSON encoder function for SQLAlchemy special classes.""" +# if isinstance(obj, datetime.date): +# return obj.isoformat() +# elif isinstance(obj, decimal.Decimal): +# return float(obj) +# elif isinstance(obj, sqlalchemy.orm.state.InstanceState): +# return "" +# elif isinstance(obj, Schedule): +# return json.dumps([obj._asdict()], default=alchemyencoder) +# else: +# return str(obj) +# +# +# class Routes: +# error = None +# scheduler = None +# messenger = None +# lqs_communicator = None +# +# def __init__(self, scheduler, lqs_communicator, messenger, config): +# self.scheduler = scheduler +# self.messenger = messenger +# self.lqs_communicator = lqs_communicator +# +# # when debug is enabled => logging messages appear twice +# app.run(port=config.get("web_port")) #debug=True) +# +# @staticmethod +# @APP.route('/') +# @APP.route('/index') +# def index(): +# return render_template("index.html") +# +# @staticmethod +# @APP.route("/trackservice", methods=["GET"]) +# def track_service(): +# from_time = request.args.get("from") +# to_time = request.args.get("to") +# last = request.args.get("last") +# now = request.args.get("now") +# +# # nothing set => today's playlist +# if from_time == None and to_time == None and now == None: +# selected_date = datetime.date.today() +# trackservice_entries = [] # TrackService.select_by_day(selected_date) +# +# # from and end time set +# elif from_time is not None and to_time is not None: +# to_time = datetime.datetime.strptime(to_time, "%Y-%m-%d") +# from_time = datetime.datetime.strptime(from_time, "%Y-%m-%d") +# trackservice_entries = [] # TrackService.select_by_range(from_time, to_time) +# +# # now set +# elif now == "": +# datetime.date.today() +# trackservice_entries = [] # TrackService.now_playing() +# +# return render_template("trackservice.html", +# length=len(trackservice_entries), +# trackservice_entries=trackservice_entries, +# selected_date=selected_date) +# +# @staticmethod +# @APP.route("/test") +# def test(): +# return render_template("index2.html") +# +# @staticmethod +# @APP.route("/login") +# def login(): +# return "login" +# #return render_template("index.html") +# +# @staticmethod +# @APP.route("/logout") +# def logout(): +# #session.pop("logged_in", None) +# return "logout" +# #return render_template("index.html") +# +# @staticmethod +# @APP.route("/api/v1/trackservice/<selected_date>", methods=["GET"]) +# def api_trackservice(selected_date): +# try: +# # convert date +# selected_date = datetime.datetime.strptime(selected_date, "%Y-%m-%d").date() +# # select from database +# tracks_on_selected_date = [] # TrackService.select_by_day(selected_date) +# # return as json +# return json.dumps([tracks._asdict() for tracks in tracks_on_selected_date], default=alchemyencoder) +# except Exception as e: +# import traceback +# traceback.print_exc() +# +# error = "Cannot transform programme into JSON String. Reason: " + str(e) +# +# logger = logging.getLogger("AuraEngine") +# logger.error(error) +# +# return json.dumps({"Error": error}) +# +# @staticmethod +# @APP.route("/api/v1/soundserver_state", methods=["GET"]) +# def soundserver_settings(): +# logger = logging.getLogger("AuraEngine") +# logger.critical("soundserver_state removed!") +# #from modules.communication.liquidsoap.communicator import LiquidSoapCommunicator +# #from modules.base.config import ConfigReader +# +# try: +# #cr = ConfigReader() +# #cr.load_config() +# #lqs = LiquidSoapCommunicator(cr) +# return "check removed!" #lqs.auraengine_state() +# except Exception as e: +# error = "Unable to fetch state from Liquidsoap. Is Soundserver running? Reason: " + str(e) +# logger = logging.getLogger("AuraEngine") +# logger.error(error) +# return json.dumps({"Error": error}) +# +# +# @staticmethod +# @APP.route("/api/v1/trackservice/", methods=["GET"]) +# def api_trackservice_now(): +# return json.dumps({'reached': True}) +# +# @staticmethod +# @APP.route("/api/v1/upcoming/", methods=["GET"]) +# def api_clock(): +# servertime = datetime.datetime.now() +# # get upcoming tracks +# upcoming = ScheduleEntry.select_upcoming() +# # convert to json string +# upcoming_as_json = json.dumps([tracks._asdict() for tracks in upcoming], default=alchemyencoder) +# # add servertime and return it +# return upcoming_as_json.replace('[{', '[{"servertime":'+str(servertime)+"},{", 1) -- GitLab