Skip to content
Snippets Groups Projects
Commit 861c1611 authored by David Trattnig's avatar David Trattnig
Browse files

Bearer auth against Tank. Refactoring. #14

parent f679ae7e
No related branches found
No related tags found
No related merge requests found
Pipeline #817 passed
Stage: test
Hide whitespace changes
Inline Side-by-side
modules/scheduling/calender_fetcher.py
+ 77
− 274
View file @ 861c1611
......@@ -19,31 +19,29 @@
import os
import sys
import urllib
import logging
import simplejson
import requests
from datetime import datetime, timedelta
#from modules.models.schedule import Schedule
from modules.base.utils import SimpleUtil
from datetime import datetime
from modules.base.utils import SimpleUtil as SU
class CalendarFetcher:
"""
Fetches the schedules, playlists and playlist entries as JSON
via the API endpoints.
via the API endpoints of Steering and Tank.
"""
url = dict()
url_parameter = dict()
config = None
logging = None
has_already_fetched = False
fetched_schedule_data = None
# FIXME another crutch because of the missing TANK
used_random_playlist_ids = list()
# Config for API Endpoints
steering_calendar_url = None
tank_playlist_url = None
tank_session = None
tank_secret = None
def __init__(self, config):
......@@ -55,9 +53,11 @@ class CalendarFetcher:
"""
self.config = config
self.logger = logging.getLogger("AuraEngine")
self.__set_url__("api_steering_calendar")
self.__set_url__("api_tank_playlist")
self.__set_url__("api_steering_show")
self.steering_calendar_url = self.config.get("api_steering_calendar")
self.tank_playlist_url = self.config.get("api_tank_playlist")
self.tank_session = self.config.get("api_tank_session")
self.tank_secret = self.config.get("api_tank_secret")
#
# PUBLIC METHODS
......@@ -68,38 +68,17 @@ class CalendarFetcher:
"""
Retrieve all required data from the API.
"""
return_data = []
# Fetch upcoming schedules from STEERING
try:
self.logger.debug("Fetching schedules from STEERING")
self.fetched_schedule_data = self.__fetch_schedule_data__()
if not self.fetched_schedule_data:
self.logger.critical(SimpleUtil.red("No schedules fetched from API!"))
return None
except urllib.error.HTTPError as e:
self.logger.critical("Cannot fetch from " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
self.fetched_schedule_data = None
return None
except (urllib.error.URLError, IOError, ValueError) as e:
self.logger.critical("Cannot connect to " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
self.fetched_schedule_data = None
self.logger.debug("Fetching schedules from STEERING")
self.fetched_schedule_data = self.fetch_schedule_data()
if not self.fetched_schedule_data:
self.logger.critical(SU.red("No schedules fetched from API!"))
return None
# Fetch playlist and fallbacks to the schedules from TANK
try:
self.logger.debug("Fetching playlists from TANK")
self.__fetch_schedule_playlists__()
except urllib.error.HTTPError as e:
self.logger.critical("Cannot fetch from " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
self.fetched_schedule_data = None
return None
except (urllib.error.URLError, IOError, ValueError) as e:
self.logger.critical("Cannot connect to " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
self.fetched_schedule_data = None
return None
self.logger.debug("Fetching playlists from TANK")
self.fetch_playlists()
return_data = []
# Gather returndata
try:
for schedule in self.fetched_schedule_data:
# Skip schedule if no start or end is given
......@@ -119,8 +98,8 @@ class CalendarFetcher:
if schedule:
return_data.append(schedule)
except TypeError as e:
self.logger.error("Nothing fetched ...")
except TypeError:
self.logger.error(SU.red("Nothing fetched ..."))
self.fetched_schedule_data = None
return None
......@@ -133,57 +112,37 @@ class CalendarFetcher:
#
# FIXME Refactor for more transparent API requests.
def __set_url__(self, type):
"""
Initializes URLs and parameters for API calls.
"""
url = self.config.get(type)
pos = url.find("?")
if pos > 0:
self.url[type] = url[0:pos]
self.url_parameter[type] = url[pos:]
else:
self.url[type] = url
def __fetch_schedule_data__(self):
def fetch_schedule_data(self):
"""
Fetches schedule data from Steering.
Returns:
([Schedule]): An array of schedules
"""
servicetype = "api_steering_calendar"
schedule = None
headers = { "content-type": "application/json" }
try:
self.logger.debug("Fetch schedules from Steering API...")
response = requests.get(self.steering_calendar_url, data=None, headers=headers)
self.logger.debug("Steering API response: %s" % response.status_code)
if not response.status_code == 200:
self.logger.critical(SU.red("HTTP Status: %s | Schedules could not be fetched! Response: %s" % \
(str(response.status_code), response.text)))
return None
schedule = response.json()
except Exception as e:
self.logger.critical(SU.red("Error while requesting schedules from Steering!"), e)
# fetch data from steering
url = self.__build_url__(servicetype)
html_response = self.__fetch_data__(servicetype, url)
# use testdata if response fails or is empty
if not html_response or html_response == b"[]":
self.logger.critical("Got no response from Steering!")
# FIXME move hardcoded test-data to separate testing logic.
html_response = self.get_test_schedules()
# convert to dict
schedule = simplejson.loads(html_response)
# check data
if not schedule:
self.logger.warn("Got no schedule via Playout API (Steering)!")
self.logger.error(SU.red("Got no schedule via Playout API (Steering)!"))
return None
#self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
return self.remove_unnecessary_data(schedule)
def __fetch_schedule_playlists__(self):
def fetch_playlists(self):
"""
Fetches all playlists including fallback playlists for every schedule.
This method used the class member `fetched_schedule_data`` to iterate
......@@ -196,9 +155,6 @@ class CalendarFetcher:
try:
for schedule in self.fetched_schedule_data:
# Extend schedule with details of show (e.g. slug)
schedule = self.__fetch_show_details__(schedule)
# Get IDs of playlists
playlist_id = self.get_playlist_id(schedule, "playlist_id")
schedule_fallback_id = self.get_playlist_id(schedule, "schedule_fallback_id")
......@@ -207,15 +163,15 @@ class CalendarFetcher:
# Retrieve playlist and the fallback playlists for every schedule.
# If a playlist (like station_fallback) is already fetched, it is not fetched again but reused
schedule["playlist"] = self.__fetch_schedule_playlist__(schedule, playlist_id, fetched_entries)
schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, schedule_fallback_id, fetched_entries)
schedule["show_fallback"] = self.__fetch_schedule_playlist__(schedule, show_fallback_id, fetched_entries)
schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, station_fallback_id, fetched_entries)
schedule["playlist"] = self.fetch_playlist(playlist_id, fetched_entries)
schedule["schedule_fallback"] = self.fetch_playlist(schedule_fallback_id, fetched_entries)
schedule["show_fallback"] = self.fetch_playlist(show_fallback_id, fetched_entries)
schedule["station_fallback"] = self.fetch_playlist(station_fallback_id, fetched_entries)
# If Steering doesn't provide a station fallback, the local one is used
if not schedule["station_fallback"] and int(local_station_fallback_id) > 0:
schedule["station_fallback_id"] = local_station_fallback_id
schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, local_station_fallback_id, fetched_entries)
schedule["station_fallback"] = self.fetch_playlist(local_station_fallback_id, fetched_entries)
if schedule["station_fallback"]:
self.logger.info("Assigned playlist #%s as local station fallback to schedule #%s" % (local_station_fallback_id, schedule["schedule_id"]))
......@@ -224,133 +180,60 @@ class CalendarFetcher:
def __fetch_show_details__(self, schedule):
"""
Fetches details of a show from Steering.
Args:
schedule (Schedule): A schedule holding a valid `show_id`
Returns:
(Schedule): The given schedule with additional show fields set.
"""
servicetype = "api_steering_show"
url = self.__build_url__(servicetype, "${ID}", str(schedule["show_id"]))
json_response = self.__fetch_data__(servicetype, url)
show_details = simplejson.loads(json_response)
# Extend "schedules" with details of "show"
schedule["show_slug"] = show_details["slug"]
### ...
### ... Add more properties here, if needed
### ...
return schedule
def get_playlist_id(self, schedule, id_name):
"""
Extracts the playlist ID for a given playlist (fallback) type.
"""
playlist_id = str(schedule[id_name])
if not playlist_id or playlist_id == "None":
self.logger.debug("No value defined for '%s' in schedule '#%s'" % (id_name, schedule["schedule_id"]))
return None
return playlist_id
def __fetch_schedule_playlist__(self, schedule, playlist_id, fetched_playlists):
def fetch_playlist(self, playlist_id, fetched_playlists):
"""
Fetches the playlist for a given schedule.
Args:
schedule (Schedule): The schedule to fetch playlists for
id_name (String): The type of playlist to fetch (e.g. normal vs. fallback)
fetched_playlists ([]): Previously fetched playlists to avoid re-fetching
Returns:
(Playlist): Playlist of type `id_name`
"""
servicetype = "api_tank_playlist"
playlist = None
if not playlist_id:
return None
playlist = None
url = self.tank_playlist_url.replace("${ID}", playlist_id)
headers = {
"Authorization": "Bearer %s:%s" % (self.tank_session, self.tank_secret),
"content-type": "application/json"
}
# If playlist is already fetched, use the existing one
# If playlist is already fetched in this round, use the existing one
for playlist in fetched_playlists:
if playlist["id"] == playlist_id:
self.logger.debug("Playlist #%s already fetched" % playlist_id)
return playlist
url = self.__build_url__(servicetype, "${ID}", playlist_id)
json_response = self.__fetch_data__(servicetype, url)
if not json_response:
self.logger.critical(SimpleUtil.red("Playlist #%s could not be fetched or is not available! JSON response: '%s'" % (playlist_id, json_response)))
return None
try:
playlist = simplejson.loads(json_response)
self.logger.debug("Fetch playlist from Tank API...")
response = requests.get(url, data=None, headers=headers)
self.logger.info("Tank API response: %s" % response.status_code)
if not response.status_code == 200:
self.logger.critical(SU.red("HTTP Status: %s | Playlist #%s could not be fetched or is not available! Response: %s" % \
(str(response.status_code), str(playlist_id), response.text)))
return None
playlist = response.json()
except Exception as e:
self.logger.critical(SimpleUtil.red("Error while parsing JSON response for playlist: %s" % json_response), e)
self.logger.critical(SU.red("Error while requesting playlist #%s from Tank" % str(playlist_id)), e)
return None
fetched_playlists.append(playlist)
return playlist
def __fetch_data__(self, type, url):
"""
Fetches JSON data for the given URL.
Args:
url (String): The API endpoint to call
Returns:
(Byte[]): An UTF-8 encoded byte object holding the response
"""
html_response = b''
# Send request to the API and read the data
try:
if type not in self.url_parameter:
if self.url[type] == "":
return False
request = urllib.request.Request(url)
else:
request = urllib.request.Request(url, self.url_parameter[type])
response = urllib.request.urlopen(request)
html_response = response.read()
except (urllib.error.URLError, IOError, ValueError) as e:
self.logger.error("Cannot connect to " + self.url[type] +
" (type: " + type + ")! Reason: " + str(e.reason))
#if not self.has_already_fetched: # first fetch
# self.logger.critical("exiting fetch data thread..")
# sys.exit()
self.has_already_fetched = True
return html_response.decode("utf-8")
def __build_url__(self, type, placeholder=None, value=None):
def get_playlist_id(self, schedule, id_name):
"""
Builds an API request URL using passed placeholder and value.
Extracts the playlist ID for a given playlist (fallback) type.
"""
url = self.url[type]
if placeholder:
url = url.replace(placeholder, value)
self.logger.debug("Built API URL: "+url)
return url
playlist_id = str(schedule[id_name])
if not playlist_id or playlist_id == "None":
self.logger.debug("No value defined for '%s' in schedule '#%s'" % (id_name, schedule["schedule_id"]))
return None
return playlist_id
def remove_unnecessary_data(self, schedule):
......@@ -367,7 +250,6 @@ class CalendarFetcher:
return schedule
def remove_data_more_than_24h_in_the_future(self, schedules):
"""
Removes entries 24h in the future and 12 hours in the past.
......@@ -376,13 +258,13 @@ class CalendarFetcher:
Think e.g. live broadcasts.
"""
items = []
now = SimpleUtil.timestamp()
now = SU.timestamp()
now_plus_24hours = now + (12*60*60)
now_minus_12hours = now - (12*60*60)
for s in schedules:
start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
start_time = SimpleUtil.timestamp(start_time)
start_time = SU.timestamp(start_time)
if start_time <= now_plus_24hours and start_time >= now_minus_12hours:
items.append(s)
......@@ -396,12 +278,12 @@ class CalendarFetcher:
currently playing.
"""
items = []
now = SimpleUtil.timestamp()
now = SU.timestamp()
for s in schedules:
start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
start_time = SimpleUtil.timestamp(start_time)
start_time = SU.timestamp(start_time)
end_time = datetime.strptime(s["end"], "%Y-%m-%dT%H:%M:%S")
end_time = SimpleUtil.timestamp(end_time)
end_time = SU.timestamp(end_time)
# Append all elements in the future
if start_time >= now:
......@@ -411,82 +293,3 @@ class CalendarFetcher:
items.append(s)
return items
#
# TESTING
#
def get_test_schedules(self):
html_response = "{}"
# use testdata if wanted
if self.config.get("use_test_data"):
html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
'{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
'{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]'
self.logger.critical("Using hardcoded Response!")
return html_response
def create_test_data(self, id_name, schedule):
import random
rand_id = random.randint(1, 10000)
while rand_id in self.used_random_playlist_ids:
rand_id = random.randint(1, 10000)
self.used_random_playlist_ids.append(rand_id)
# FIXME move hardcoded test-data to separate testing logic.
# HARDCODED Testdata
if id_name != "playlist_id":
# FALLBACK TESTDATA
if rand_id % 3 == 0: # playlist fallback
json_response = '{"playlist_id":' + str(
rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
elif rand_id % 2 == 0: # stream fallback
json_response = '{"playlist_id":' + str(
rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
else: # pool fallback
json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}'
schedule[id_name] = rand_id
elif schedule[id_name] == 0 or schedule[id_name] is None:
# this happens when playlist id is not filled out in pv
# json_response = '{"playlist_id": 0}'
if rand_id % 4 == 0: # playlist with two files
json_response = '{"playlist_id":' + str(
rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
elif rand_id % 3 == 0: # playlist with jingle and then linein
json_response = '{"playlist_id":' + str(
rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}'
elif rand_id % 2 == 0: # playlist with jingle and then http stream
json_response = '{"playlist_id":' + str(
rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
else: # pool playlist
json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}'
schedule[id_name] = rand_id
elif schedule[id_name] % 4 == 0: # playlist with two files
json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}'
elif schedule[id_name] % 3 == 0: # playlist with jingle and then http stream
json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}'
elif schedule[id_name] % 2 == 0: # playlist with jingle and then linein
json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}'
else: # pool playlist
json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}'
self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])
return json_response
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment