Commit 0d1f509c authored by David Trattnig's avatar David Trattnig Committed by David Trattnig
Browse files

Refactored calendar fetcher to work with latest Steering and Tank API.

parent 8e06b6b9
......@@ -35,7 +35,6 @@ calendarurl="http://localhost:8000/api/v1/playout"
api_show_url="http://localhost:8000/api/v1/shows/${ID}/"
# The URL to get playlist details via Tank
importerurl="http://localhost:8040/api/v1/shows/${SLUG}/playlists"
# how often should the calendar be fetched in seconds (This determines the time of the last change before a specific show)
fetching_frequency=3600
......@@ -58,7 +57,7 @@ daemongroup="david"
daemonuser="david"
[socket]
socketdir="/home/david/Code/aura/engine"
socketdir="/home/david/code/aura/aura-engine/modules/liquidsoap"
[logging]
logdir="/var/log/aura"
......
......@@ -15,7 +15,7 @@ class CalendarFetcher:
logging = None
has_already_fetched = False
fetched_schedule_data = None
# another crutch because of the missing TANK
# FIXME another crutch because of the missing TANK
used_random_playlist_ids = list()
def __init__(self, config):
......@@ -23,6 +23,7 @@ class CalendarFetcher:
self.logger = logging.getLogger("AuraEngine")
self.__set_url__("calendar")
self.__set_url__("importer")
self.__set_url__("api_show_")
def fetch(self):
# fetch upcoming schedules from STEERING
......@@ -91,45 +92,74 @@ class CalendarFetcher:
schedule = None
# fetch data from steering
html_response = self.__fetch_data__(self.url["calendar"])
html_response = self.__fetch_data__(servicetype)
# response fails or is empty
if not html_response:
self.logger.debug("Got no response from pv!")
# FIXME move hardcoded test-data to separate testing logic.
# use testdata if response fails or is empty
if not html_response or html_response == b"[]":
self.logger.critical("Got no response from Steering!")
#html_response = '[{"schedule_id":1,"start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%d %H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%d %H:00:00') + '","show_id":9,"show_name":"FROzine","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"},{"schedule_id":2,"schedule_start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%d %H:00:00') + '","schedule_end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%d %H:00:00') + '","show_id":10,"show_name":"FROMat","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"},{"schedule_id":3,"schedule_start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%d %H:00:00') + '","schedule_end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%d %H:00:00') + '","show_id":11,"show_name":"Radio für Senioren","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"}]'
# use testdata if wanted
if self.config.get("use_test_data"):
# FIXME move hardcoded test-data to separate testing logic.
html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
'{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
'{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]'
self.logger.critical("Hardcoded Response")
self.logger.critical("Using hardcoded Response!")
else:
html_response = "{}"
# convert to dict
schedule = simplejson.loads(html_response)
# check data
self.logger.critical("no JSON data checks. I believe what i get here")
self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
#self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
return self.remove_unnecessary_data(schedule_from_pv)
# ------------------------------------------------------------------------------------------ #
def __fetch_schedule_playlists__(self):
# store fetched entries => do not have to fetch playlist_id more than once
fetched_entries=[]
self.logger.warning("only fetching normal playlists. no fallbacks")
for schedule in self.fetched_schedule_data:
# retrieve playlist and the fallbacks for every schedule
# if a playlist (like station_fallback) is already fetched, it is not fetched again but reused
schedule["playlist"] = self.__fetch_schedule_playlist__(schedule, "playlist_id", fetched_entries)
# schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, "schedule_fallback_id", fetched_entries)
# schedule["show_fallback"] = self.__fetch_schedule_playlist__(schedule, "show_fallback_id", fetched_entries)
# schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, "station_fallback_id", fetched_entries)
try:
self.logger.warning("only fetching normal playlists. no fallbacks")
for schedule in self.fetched_schedule_data:
# Enhance schedule with details of show (e.g. slug)
schedule = self.__fetch_show_details__(schedule)
# retrieve playlist and the fallbacks for every schedule
# if a playlist (like station_fallback) is already fetched, it is not fetched again but reused
schedule["playlist"] = self.__fetch_schedule_playlist__(schedule, "playlist_id", fetched_entries)
#schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, "schedule_fallback_id", fetched_entries)
#schedule["show_fallback"] = self.__fetch_schedule_playlist__(schedule, "show_fallback_id", fetched_entries)
#schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, "station_fallback_id", fetched_entries)
self.logger.info(str(schedule))
except Exception as e:
self.logger.error("Error: "+str(e))
# ------------------------------------------------------------------------------------------ #
def __fetch_show_details__(self, schedule):
servicetype = "api_show_"
json_response = self.__fetch_data__(servicetype, "${ID}", str(schedule["show_id"]))
show_details = simplejson.loads(json_response)
# Augment "schedules" with details of "show"
schedule["show_slug"] = show_details["slug"]
### ... add more properties here, if needed ... ###
return schedule
# ------------------------------------------------------------------------------------------ #
def __fetch_schedule_playlist__(self, schedule, id_name, fetched_schedule_entries):
servicetype = "importer"
# set playlist_id (in testenvironment always null => no idea)
if id_name not in schedule or schedule[id_name] is None:
playlist_id = 1
......@@ -138,13 +168,28 @@ class CalendarFetcher:
# set url
#url = self.url["importer"] + schedule["show_name"] + "/playlists/" + str(playlist_id)
url = self.url["importer"] + "public" + "/playlists/" + str(playlist_id)
#url = self.url["importer"] + "public" + "/playlists/" + str(playlist_id)
# fetch playlists from TANK
if not "show_slug" in schedule:
raise ValueError("Missing 'show_slug' for schedule", schedule)
# fetch data from importer
json_response = self.__fetch_data__(url)
slug = str(schedule["show_slug"])
json_response = self.__fetch_data__(servicetype, "${SLUG}", slug)
# use testdata if wanted
if not json_response and self.config.get("use_test_data"):
self.logger.warn("Using test-data for fetch-schedule-playlist")
use_testdata = True
# if a playlist is already fetched, do not fetch it again
for entry in fetched_schedule_entries:
if entry["playlist_id"] == schedule[id_name]:
self.logger.debug("playlist #" + str(schedule[id_name]) + " already fetched")
return entry
if use_testdata:
# FIXME move hardcoded test-data to separate testing logic.
json_response = self.create_test_data(id_name, schedule)
# convert to list
......@@ -167,27 +212,48 @@ class CalendarFetcher:
return e
# ------------------------------------------------------------------------------------------ #
def __fetch_data__(self, url, parameter = ""):
# init html_response
def __fetch_data__(self, type, placeholder=None, value=None):
# Init html_response
html_response = b''
url = self.__build_url__(type, placeholder, value)
# open an url and read the data
if parameter == "":
request = urllib.request.Request(url)
else:
request = urllib.request.Request(url, parameter)
# Send request to the API and read the data
try:
if type not in self.data:
if self.url[type] == "":
return False
request = urllib.request.Request(url)
else:
request = urllib.request.Request(url, self.data[type])
response = urllib.request.urlopen(request)
html_response = response.read()
response = urllib.request.urlopen(request)
html_response = response.read()
except (urllib.error.URLError, IOError, ValueError) as e:
self.logger.error("Cannot connect to " + self.url[type] +
" (type: " + type + ")! Reason: " + str(e.reason))
#if not self.has_already_fetched: # first fetch
# self.logger.critical("exiting fetch data thread..")
# sys.exit()
self.has_already_fetched = True
return html_response.decode("utf-8")
# ------------------------------------------------------------------------------------------ #
def __build_url__(self, type, placeholder=None, value=None):
url = self.url[type]
if placeholder:
url = url.replace(placeholder, value)
# print("built URL: "+url)
return url
# ------------------------------------------------------------------------------------------ #
def remove_unnecessary_data(self, schedule):
reduced_schedule = self.remove_data_more_than_24h_in_the_future(self.remove_data_in_the_past(schedule))
return reduced_schedule
count_before = len(schedule)
schedule = self.remove_data_more_than_24h_in_the_future(schedule)
schedule = self.remove_data_in_the_past(schedule)
count_after = len(schedule)
self.logger.info("Removed %d unnecessary schedules from response." % (count_before - count_after))
return schedule
# ------------------------------------------------------------------------------------------ #
def remove_data_more_than_24h_in_the_future(self, schedule_from_pv):
act_list = []
......@@ -231,6 +297,7 @@ class CalendarFetcher:
self.used_random_playlist_ids.append(rand_id)
# FIXME move hardcoded test-data to separate testing logic.
# HARDCODED Testdata
if id_name != "playlist_id":
# FALLBACK TESTDATA
......@@ -275,4 +342,4 @@ class CalendarFetcher:
self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])
return json_response
\ No newline at end of file
return json_response
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment