From 861c16110feac29641c30a7b4f4f6502601e9e57 Mon Sep 17 00:00:00 2001
From: David Trattnig <david.trattnig@o94.at>
Date: Thu, 17 Sep 2020 23:02:01 +0200
Subject: [PATCH] Bearer auth against Tank. Refactoring. #14
---
modules/scheduling/calender_fetcher.py | 351 ++++++-------------------
1 file changed, 77 insertions(+), 274 deletions(-)
diff --git a/modules/scheduling/calender_fetcher.py b/modules/scheduling/calender_fetcher.py
index e8a4ecba..087dbec8 100644
--- a/modules/scheduling/calender_fetcher.py
+++ b/modules/scheduling/calender_fetcher.py
@@ -19,31 +19,29 @@
-import os
-import sys
-import urllib
import logging
-import simplejson
+import requests
-from datetime import datetime, timedelta
-#from modules.models.schedule import Schedule
-from modules.base.utils import SimpleUtil
+from datetime import datetime
+from modules.base.utils import SimpleUtil as SU
class CalendarFetcher:
"""
Fetches the schedules, playlists and playlist entries as JSON
- via the API endpoints.
+ via the API endpoints of Steering and Tank.
"""
- url = dict()
- url_parameter = dict()
config = None
logging = None
has_already_fetched = False
fetched_schedule_data = None
- # FIXME another crutch because of the missing TANK
- used_random_playlist_ids = list()
+ # Config for API Endpoints
+ steering_calendar_url = None
+ tank_playlist_url = None
+ tank_session = None
+ tank_secret = None
+
def __init__(self, config):
@@ -55,9 +53,11 @@ class CalendarFetcher:
"""
self.config = config
self.logger = logging.getLogger("AuraEngine")
- self.__set_url__("api_steering_calendar")
- self.__set_url__("api_tank_playlist")
- self.__set_url__("api_steering_show")
+ self.steering_calendar_url = self.config.get("api_steering_calendar")
+ self.tank_playlist_url = self.config.get("api_tank_playlist")
+ self.tank_session = self.config.get("api_tank_session")
+ self.tank_secret = self.config.get("api_tank_secret")
+
#
# PUBLIC METHODS
@@ -68,38 +68,17 @@ class CalendarFetcher:
"""
Retrieve all required data from the API.
"""
+ return_data = []
- # Fetch upcoming schedules from STEERING
- try:
- self.logger.debug("Fetching schedules from STEERING")
- self.fetched_schedule_data = self.__fetch_schedule_data__()
- if not self.fetched_schedule_data:
- self.logger.critical(SimpleUtil.red("No schedules fetched from API!"))
- return None
- except urllib.error.HTTPError as e:
- self.logger.critical("Cannot fetch from " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
- self.fetched_schedule_data = None
- return None
- except (urllib.error.URLError, IOError, ValueError) as e:
- self.logger.critical("Cannot connect to " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
- self.fetched_schedule_data = None
+ self.logger.debug("Fetching schedules from STEERING")
+ self.fetched_schedule_data = self.fetch_schedule_data()
+ if not self.fetched_schedule_data:
+ self.logger.critical(SU.red("No schedules fetched from API!"))
return None
- # Fetch playlist and fallbacks to the schedules from TANK
- try:
- self.logger.debug("Fetching playlists from TANK")
- self.__fetch_schedule_playlists__()
- except urllib.error.HTTPError as e:
- self.logger.critical("Cannot fetch from " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
- self.fetched_schedule_data = None
- return None
- except (urllib.error.URLError, IOError, ValueError) as e:
- self.logger.critical("Cannot connect to " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
- self.fetched_schedule_data = None
- return None
+ self.logger.debug("Fetching playlists from TANK")
+ self.fetch_playlists()
- return_data = []
- # Gather returndata
try:
for schedule in self.fetched_schedule_data:
# Skip schedule if no start or end is given
@@ -119,8 +98,8 @@ class CalendarFetcher:
if schedule:
return_data.append(schedule)
- except TypeError as e:
- self.logger.error("Nothing fetched ...")
+ except TypeError:
+ self.logger.error(SU.red("Nothing fetched ..."))
self.fetched_schedule_data = None
return None
@@ -133,57 +112,37 @@ class CalendarFetcher:
#
- # FIXME Refactor for more transparent API requests.
- def __set_url__(self, type):
- """
- Initializes URLs and parameters for API calls.
- """
- url = self.config.get(type)
- pos = url.find("?")
-
- if pos > 0:
- self.url[type] = url[0:pos]
- self.url_parameter[type] = url[pos:]
- else:
- self.url[type] = url
-
-
-
- def __fetch_schedule_data__(self):
+ def fetch_schedule_data(self):
"""
Fetches schedule data from Steering.
Returns:
([Schedule]): An array of schedules
"""
- servicetype = "api_steering_calendar"
schedule = None
+ headers = { "content-type": "application/json" }
+
+ try:
+ self.logger.debug("Fetch schedules from Steering API...")
+ response = requests.get(self.steering_calendar_url, data=None, headers=headers)
+ self.logger.debug("Steering API response: %s" % response.status_code)
+ if not response.status_code == 200:
+ self.logger.critical(SU.red("HTTP Status: %s | Schedules could not be fetched! Response: %s" % \
+ (str(response.status_code), response.text)))
+ return None
+ schedule = response.json()
+ except Exception as e:
+ self.logger.critical(SU.red("Error while requesting schedules from Steering!"), e)
- # fetch data from steering
- url = self.__build_url__(servicetype)
- html_response = self.__fetch_data__(servicetype, url)
-
- # use testdata if response fails or is empty
- if not html_response or html_response == b"[]":
- self.logger.critical("Got no response from Steering!")
-
- # FIXME move hardcoded test-data to separate testing logic.
- html_response = self.get_test_schedules()
-
- # convert to dict
- schedule = simplejson.loads(html_response)
-
- # check data
if not schedule:
- self.logger.warn("Got no schedule via Playout API (Steering)!")
+ self.logger.error(SU.red("Got no schedule via Playout API (Steering)!"))
return None
- #self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
return self.remove_unnecessary_data(schedule)
- def __fetch_schedule_playlists__(self):
+ def fetch_playlists(self):
"""
Fetches all playlists including fallback playlists for every schedule.
This method used the class member `fetched_schedule_data`` to iterate
@@ -196,9 +155,6 @@ class CalendarFetcher:
try:
for schedule in self.fetched_schedule_data:
- # Extend schedule with details of show (e.g. slug)
- schedule = self.__fetch_show_details__(schedule)
-
# Get IDs of playlists
playlist_id = self.get_playlist_id(schedule, "playlist_id")
schedule_fallback_id = self.get_playlist_id(schedule, "schedule_fallback_id")
@@ -207,15 +163,15 @@ class CalendarFetcher:
# Retrieve playlist and the fallback playlists for every schedule.
# If a playlist (like station_fallback) is already fetched, it is not fetched again but reused
- schedule["playlist"] = self.__fetch_schedule_playlist__(schedule, playlist_id, fetched_entries)
- schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, schedule_fallback_id, fetched_entries)
- schedule["show_fallback"] = self.__fetch_schedule_playlist__(schedule, show_fallback_id, fetched_entries)
- schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, station_fallback_id, fetched_entries)
+ schedule["playlist"] = self.fetch_playlist(playlist_id, fetched_entries)
+ schedule["schedule_fallback"] = self.fetch_playlist(schedule_fallback_id, fetched_entries)
+ schedule["show_fallback"] = self.fetch_playlist(show_fallback_id, fetched_entries)
+ schedule["station_fallback"] = self.fetch_playlist(station_fallback_id, fetched_entries)
# If Steering doesn't provide a station fallback, the local one is used
if not schedule["station_fallback"] and int(local_station_fallback_id) > 0:
schedule["station_fallback_id"] = local_station_fallback_id
- schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, local_station_fallback_id, fetched_entries)
+ schedule["station_fallback"] = self.fetch_playlist(local_station_fallback_id, fetched_entries)
if schedule["station_fallback"]:
self.logger.info("Assigned playlist #%s as local station fallback to schedule #%s" % (local_station_fallback_id, schedule["schedule_id"]))
@@ -224,133 +180,60 @@ class CalendarFetcher:
- def __fetch_show_details__(self, schedule):
- """
- Fetches details of a show from Steering.
-
- Args:
- schedule (Schedule): A schedule holding a valid `show_id`
-
- Returns:
- (Schedule): The given schedule with additional show fields set.
- """
- servicetype = "api_steering_show"
-
- url = self.__build_url__(servicetype, "${ID}", str(schedule["show_id"]))
- json_response = self.__fetch_data__(servicetype, url)
- show_details = simplejson.loads(json_response)
-
- # Extend "schedules" with details of "show"
- schedule["show_slug"] = show_details["slug"]
- ### ...
- ### ... Add more properties here, if needed
- ### ...
-
- return schedule
-
-
-
- def get_playlist_id(self, schedule, id_name):
- """
- Extracts the playlist ID for a given playlist (fallback) type.
- """
- playlist_id = str(schedule[id_name])
- if not playlist_id or playlist_id == "None":
- self.logger.debug("No value defined for '%s' in schedule '#%s'" % (id_name, schedule["schedule_id"]))
- return None
-
- return playlist_id
-
-
-
- def __fetch_schedule_playlist__(self, schedule, playlist_id, fetched_playlists):
+ def fetch_playlist(self, playlist_id, fetched_playlists):
"""
Fetches the playlist for a given schedule.
Args:
- schedule (Schedule): The schedule to fetch playlists for
id_name (String): The type of playlist to fetch (e.g. normal vs. fallback)
fetched_playlists ([]): Previously fetched playlists to avoid re-fetching
Returns:
(Playlist): Playlist of type `id_name`
"""
- servicetype = "api_tank_playlist"
- playlist = None
-
if not playlist_id:
return None
+ playlist = None
+ url = self.tank_playlist_url.replace("${ID}", playlist_id)
+ headers = {
+ "Authorization": "Bearer %s:%s" % (self.tank_session, self.tank_secret),
+ "content-type": "application/json"
+ }
- # If playlist is already fetched, use the existing one
+ # If playlist is already fetched in this round, use the existing one
for playlist in fetched_playlists:
if playlist["id"] == playlist_id:
self.logger.debug("Playlist #%s already fetched" % playlist_id)
return playlist
-
- url = self.__build_url__(servicetype, "${ID}", playlist_id)
- json_response = self.__fetch_data__(servicetype, url)
-
- if not json_response:
- self.logger.critical(SimpleUtil.red("Playlist #%s could not be fetched or is not available! JSON response: '%s'" % (playlist_id, json_response)))
- return None
-
+
try:
- playlist = simplejson.loads(json_response)
+ self.logger.debug("Fetch playlist from Tank API...")
+ response = requests.get(url, data=None, headers=headers)
+ self.logger.info("Tank API response: %s" % response.status_code)
+ if not response.status_code == 200:
+ self.logger.critical(SU.red("HTTP Status: %s | Playlist #%s could not be fetched or is not available! Response: %s" % \
+ (str(response.status_code), str(playlist_id), response.text)))
+ return None
+ playlist = response.json()
except Exception as e:
- self.logger.critical(SimpleUtil.red("Error while parsing JSON response for playlist: %s" % json_response), e)
+ self.logger.critical(SU.red("Error while requesting playlist #%s from Tank" % str(playlist_id)), e)
+ return None
fetched_playlists.append(playlist)
return playlist
-
- def __fetch_data__(self, type, url):
- """
- Fetches JSON data for the given URL.
-
- Args:
- url (String): The API endpoint to call
-
- Returns:
- (Byte[]): An UTF-8 encoded byte object holding the response
- """
- html_response = b''
-
- # Send request to the API and read the data
- try:
- if type not in self.url_parameter:
- if self.url[type] == "":
- return False
- request = urllib.request.Request(url)
- else:
- request = urllib.request.Request(url, self.url_parameter[type])
-
- response = urllib.request.urlopen(request)
- html_response = response.read()
-
- except (urllib.error.URLError, IOError, ValueError) as e:
- self.logger.error("Cannot connect to " + self.url[type] +
- " (type: " + type + ")! Reason: " + str(e.reason))
- #if not self.has_already_fetched: # first fetch
- # self.logger.critical("exiting fetch data thread..")
- # sys.exit()
-
- self.has_already_fetched = True
- return html_response.decode("utf-8")
-
-
-
- def __build_url__(self, type, placeholder=None, value=None):
+ def get_playlist_id(self, schedule, id_name):
"""
- Builds an API request URL using passed placeholder and value.
+ Extracts the playlist ID for a given playlist (fallback) type.
"""
- url = self.url[type]
- if placeholder:
- url = url.replace(placeholder, value)
- self.logger.debug("Built API URL: "+url)
- return url
-
+ playlist_id = str(schedule[id_name])
+ if not playlist_id or playlist_id == "None":
+ self.logger.debug("No value defined for '%s' in schedule '#%s'" % (id_name, schedule["schedule_id"]))
+ return None
+
+ return playlist_id
def remove_unnecessary_data(self, schedule):
@@ -367,7 +250,6 @@ class CalendarFetcher:
return schedule
-
def remove_data_more_than_24h_in_the_future(self, schedules):
"""
Removes entries 24h in the future and 12 hours in the past.
@@ -376,13 +258,13 @@ class CalendarFetcher:
Think e.g. live broadcasts.
"""
items = []
- now = SimpleUtil.timestamp()
+ now = SU.timestamp()
now_plus_24hours = now + (12*60*60)
now_minus_12hours = now - (12*60*60)
for s in schedules:
start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
- start_time = SimpleUtil.timestamp(start_time)
+ start_time = SU.timestamp(start_time)
if start_time <= now_plus_24hours and start_time >= now_minus_12hours:
items.append(s)
@@ -396,12 +278,12 @@ class CalendarFetcher:
currently playing.
"""
items = []
- now = SimpleUtil.timestamp()
+ now = SU.timestamp()
for s in schedules:
start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
- start_time = SimpleUtil.timestamp(start_time)
+ start_time = SU.timestamp(start_time)
end_time = datetime.strptime(s["end"], "%Y-%m-%dT%H:%M:%S")
- end_time = SimpleUtil.timestamp(end_time)
+ end_time = SU.timestamp(end_time)
# Append all elements in the future
if start_time >= now:
@@ -411,82 +293,3 @@ class CalendarFetcher:
items.append(s)
return items
-
-
-
-
- #
- # TESTING
- #
-
-
- def get_test_schedules(self):
- html_response = "{}"
-
- # use testdata if wanted
- if self.config.get("use_test_data"):
-
- html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
- '{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
- '{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]'
- self.logger.critical("Using hardcoded Response!")
-
- return html_response
-
-
-
- def create_test_data(self, id_name, schedule):
- import random
- rand_id = random.randint(1, 10000)
-
- while rand_id in self.used_random_playlist_ids:
- rand_id = random.randint(1, 10000)
-
- self.used_random_playlist_ids.append(rand_id)
-
- # FIXME move hardcoded test-data to separate testing logic.
- # HARDCODED Testdata
- if id_name != "playlist_id":
- # FALLBACK TESTDATA
-
- if rand_id % 3 == 0: # playlist fallback
- json_response = '{"playlist_id":' + str(
- rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
- elif rand_id % 2 == 0: # stream fallback
- json_response = '{"playlist_id":' + str(
- rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
- else: # pool fallback
- json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}'
-
- schedule[id_name] = rand_id
-
- elif schedule[id_name] == 0 or schedule[id_name] is None:
- # this happens when playlist id is not filled out in pv
- # json_response = '{"playlist_id": 0}'
-
- if rand_id % 4 == 0: # playlist with two files
- json_response = '{"playlist_id":' + str(
- rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
- elif rand_id % 3 == 0: # playlist with jingle and then linein
- json_response = '{"playlist_id":' + str(
- rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}'
- elif rand_id % 2 == 0: # playlist with jingle and then http stream
- json_response = '{"playlist_id":' + str(
- rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
- else: # pool playlist
- json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}'
-
- schedule[id_name] = rand_id
-
- elif schedule[id_name] % 4 == 0: # playlist with two files
- json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}'
- elif schedule[id_name] % 3 == 0: # playlist with jingle and then http stream
- json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}'
- elif schedule[id_name] % 2 == 0: # playlist with jingle and then linein
- json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}'
- else: # pool playlist
- json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}'
-
- self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])
-
- return json_response
--
GitLab