From f33bba162d7465a0a8ccfe9867fe5d54f23b5aaa Mon Sep 17 00:00:00 2001
From: David Trattnig <david.trattnig@gmail.com>
Date: Thu, 24 Oct 2019 13:34:08 +0200
Subject: [PATCH] Refactored calendar fetcher to work with latest Steering and
Tank API.
---
configuration/engine.ini | 10 +++--
modules/scheduling/calender_fetcher.py | 58 ++++++++++++++++++++------
2 files changed, 51 insertions(+), 17 deletions(-)
diff --git a/configuration/engine.ini b/configuration/engine.ini
index 1a82de62..59bd9f78 100644
--- a/configuration/engine.ini
+++ b/configuration/engine.ini
@@ -29,10 +29,12 @@ from_mail="monitor@aura.py"
mailsubject_prefix="[AURA]"
[dataurls]
-# the url of pv/steering
+# The URL to get the Calendar via PV/Steering
calendarurl="http://localhost:8000/api/v1/playout"
-# the url of tank
-importerurl="http://localhost:8040/api/v1/groups/_public/playlists/"
+# The URL to get show details via PV/Steering
+api_show_url="http://localhost:8000/api/v1/shows/${ID}/"
+# The URL to get playlist details via Tank
+importerurl="http://localhost:8040/api/v1/shows/${SLUG}/playlists"
# how often should the calendar be fetched in seconds (This determines the time of the last change before a specific show)
fetching_frequency=3600
@@ -55,7 +57,7 @@ daemongroup="gg"
daemonuser="gg"
[socket]
-socketdir="/home/gg/PycharmProjects/engine/modules/liquidsoap"
+socketdir="~/code/aura/aura-engine/modules/liquidsoap"
[logging]
logdir="/var/log/aura"
diff --git a/modules/scheduling/calender_fetcher.py b/modules/scheduling/calender_fetcher.py
index 42978ece..d5a4316d 100644
--- a/modules/scheduling/calender_fetcher.py
+++ b/modules/scheduling/calender_fetcher.py
@@ -21,11 +21,12 @@ class CalendarFetcher:
self.logger = logging.getLogger("AuraEngine")
self.__set_url__("calendar")
self.__set_url__("importer")
+ self.__set_url__("api_show_")
def fetch(self):
# fetch upcoming schedules from STEERING
self.logger.debug("Fetching schedules from STEERING")
- self.__fetch_schedule_data__()
+ self.fetched_schedule_data = self.__fetch_schedule_data__()
# fetch playlist and fallbacks to the schedules from TANK
self.logger.debug("Fetching playlists from TANK")
self.__fetch_schedule_entry_data__()
@@ -61,6 +62,7 @@ class CalendarFetcher:
# fetch data from steering
html_response = self.__fetch_data__(servicetype)
+ # FIXME move hardcoded test-data to separate testing logic.
# use testdata if response fails or is empty
if not html_response or html_response == b"[]":
self.logger.debug("Got no response: Using testdata")
@@ -76,7 +78,7 @@ class CalendarFetcher:
# check data
self.logger.critical("Hardcoded Response && no JSON data checks. I believe what i get here")
- return self.remove_unnecessary_data(self.fetched_schedule_data)
+ return self.remove_unnecessary_data(schedule_from_pv)
# ------------------------------------------------------------------------------------------ #
def __fetch_schedule_entry_data__(self):
@@ -85,6 +87,9 @@ class CalendarFetcher:
try:
for schedule in self.fetched_schedule_data:
+
+ # Enhance schedule with details of show (e.g. slug)
+ schedule = self.__fetch_show_details__(schedule)
# retrieve playlist and the fallbacks for every schedule
# if a playlist (like station_fallback) is already fetched, it is not fetched again but reused
schedule["playlist"] = self.__fetch_schedule_entries__(schedule, "playlist_id", fetched_entries)
@@ -95,15 +100,32 @@ class CalendarFetcher:
self.logger.info(str(schedule))
except Exception as e:
- self.logger.error(str(e))
+ self.logger.error("Error: "+str(e))
+
+ # ------------------------------------------------------------------------------------------ #
+ def __fetch_show_details__(self, schedule):
+ servicetype = "api_show_"
+
+ json_response = self.__fetch_data__(servicetype, "${ID}", str(schedule["show_id"]))
+ show_details = simplejson.loads(json_response)
+
+ # Augment "schedules" with details of "show"
+ schedule["show_slug"] = show_details["slug"]
+ ### ... add more properties here, if needed ... ###
+
+ return schedule
# ------------------------------------------------------------------------------------------ #
def __fetch_schedule_entries__(self, schedule, id_name, fetched_schedule_entries):
servicetype = "importer"
use_testdata = False
- # fetch data from importer
- json_response = self.__fetch_data__(servicetype)
+ # fetch playlists from TANK
+ if not "show_slug" in schedule:
+ raise ValueError("Missing 'show_slug' for schedule", schedule)
+
+ slug = str(schedule["show_slug"])
+ json_response = self.__fetch_data__(servicetype, "${SLUG}", slug)
if not json_response:
use_testdata = True
@@ -113,6 +135,7 @@ class CalendarFetcher:
self.logger.debug("playlist #" + str(schedule[id_name]) + " already fetched")
return entry
+ # FIXME move hardcoded test-data to separate testing logic.
# generate testdata
if use_testdata:
json_response = self.create_test_data(id_name, schedule)
@@ -136,31 +159,39 @@ class CalendarFetcher:
return schedule_entries
# ------------------------------------------------------------------------------------------ #
- def __fetch_data__(self, type):
- # init html_response
+ def __fetch_data__(self, type, placeholder=None, value=None):
+ # Init html_response
html_response = ""
+ url = self.__build_url__(type, placeholder, value)
- # open an url and read the data
+ # Send request to the API and read the data
try:
if type not in self.data:
if self.url[type] == "":
return False
- request = urllib.request.Request(self.url[type])
+ request = urllib.request.Request(url)
else:
- request = urllib.request.Request(self.url[type], self.data[type])
+ request = urllib.request.Request(url, self.data[type])
response = urllib.request.urlopen(request)
html_response = response.read()
except (urllib.error.URLError, IOError, ValueError) as e:
- self.logger.error("Cannot connect to " + self.url[type] + "! reason: " + str(e.reason))
+ self.logger.error("Cannot connect to " + self.url[type] +
+ " (type: " + type + ")! Reason: " + str(e.reason))
if not self.has_already_fetched: # first fetch
self.logger.critical("exiting fetch data thread..")
sys.exit()
self.has_already_fetched = True
return html_response
-
+ # ------------------------------------------------------------------------------------------ #
+ def __build_url__(self, type, placeholder=None, value=None):
+ url = self.url[type]
+ if placeholder:
+ url = url.replace(placeholder, value)
+ # print("built URL: "+url)
+ return url
# ------------------------------------------------------------------------------------------ #
def remove_unnecessary_data(self, schedule):
return self.remove_data_more_than_24h_in_the_future(self.remove_data_in_the_past(schedule))
@@ -208,6 +239,7 @@ class CalendarFetcher:
self.used_random_playlist_ids.append(rand_id)
+ # FIXME move hardcoded test-data to separate testing logic.
# HARDCODED Testdata
if id_name != "playlist_id":
# FALLBACK TESTDATA
@@ -252,4 +284,4 @@ class CalendarFetcher:
self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])
- return json_response
\ No newline at end of file
+ return json_response
--
GitLab