calender_fetcher.py 20.7 KB
Newer Older
David Trattnig's avatar
David Trattnig committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

#
# Aura Engine (https://gitlab.servus.at/aura/engine)
#
# Copyright (C) 2017-2020 - The Aura Engine Team.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.



22
23
24
25
26
27
28
29
import os
import sys
import urllib
import logging
import simplejson

from datetime import datetime, timedelta
#from modules.models.schedule import Schedule
David Trattnig's avatar
David Trattnig committed
30
from modules.base.utils import SimpleUtil
31

David Trattnig's avatar
David Trattnig committed
32

33
class CalendarFetcher:
David Trattnig's avatar
David Trattnig committed
34
35
36
37
    """
    Fetches the schedules, playlists and playlist entries as JSON
    via the API endpoints.
    """
38
39
40
41
42
43
    url = dict()
    url_parameter = dict()
    config = None
    logging = None
    has_already_fetched = False
    fetched_schedule_data = None
David Trattnig's avatar
David Trattnig committed
44

45
    # FIXME another crutch because of the missing TANK
46
47
    used_random_playlist_ids = list()

David Trattnig's avatar
David Trattnig committed
48

49
    def __init__(self, config):
David Trattnig's avatar
David Trattnig committed
50
51
52
53
54
55
        """
        Constructor

        Args:
            config (AuraConfig):    Holds the engine configuration
        """
56
57
        self.config = config
        self.logger = logging.getLogger("AuraEngine")
58
59
60
        self.__set_url__("api_steering_calendar")
        self.__set_url__("api_tank_playlist")
        self.__set_url__("api_steering_show")
David Trattnig's avatar
David Trattnig committed
61
62
63
64
65

    #
    #   PUBLIC METHODS
    #

66
67

    def fetch(self):
David Trattnig's avatar
David Trattnig committed
68
69
70
71
72
        """
        Retrieve all required data from the API.
        """

        # Fetch upcoming schedules from STEERING
73
74
        try:
            self.logger.debug("Fetching schedules from STEERING")
75
            self.fetched_schedule_data = self.__fetch_schedule_data__()
David Trattnig's avatar
David Trattnig committed
76
77
78
            if not self.fetched_schedule_data:
                self.logger.critical(SimpleUtil.red("No schedules fetched from API!"))
                return None
79
        except urllib.error.HTTPError as e:
80
            self.logger.critical("Cannot fetch from " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
81
82
            self.fetched_schedule_data = None
            return None
83
        except (urllib.error.URLError, IOError, ValueError) as e:
84
            self.logger.critical("Cannot connect to " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
85
86
            self.fetched_schedule_data = None
            return None
87

David Trattnig's avatar
David Trattnig committed
88
        # Fetch playlist and fallbacks to the schedules from TANK
89
90
91
92
        try:
            self.logger.debug("Fetching playlists from TANK")
            self.__fetch_schedule_playlists__()
        except urllib.error.HTTPError as e:
93
            self.logger.critical("Cannot fetch from " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
94
95
            self.fetched_schedule_data = None
            return None
96
        except (urllib.error.URLError, IOError, ValueError) as e:
97
            self.logger.critical("Cannot connect to " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
98
99
            self.fetched_schedule_data = None
            return None
100
101

        return_data = []
David Trattnig's avatar
David Trattnig committed
102
        # Gather returndata
103
104
        try:
            for schedule in self.fetched_schedule_data:
David Trattnig's avatar
David Trattnig committed
105
                # Skip schedule if no start or end is given
106
                if "start" not in schedule:
David Trattnig's avatar
David Trattnig committed
107
                    self.logger.warning("No start of schedule given. Skipping schedule: " + str(schedule))
108
109
                    schedule = None
                if "end" not in schedule:
David Trattnig's avatar
David Trattnig committed
110
                    self.logger.warning("No end of schedule given. Skipping schedule: " + str(schedule))
111
                    schedule = None
112
113
114
115
116
                if "playlist" not in schedule \
                    and "show_fallback" not in schedule \
                    and "schedule_fallback" not in schedule \
                    and "station_fallback" not in schedule:
                    
David Trattnig's avatar
David Trattnig committed
117
                    self.logger.warning("No playlist for schedule given. Skipping schedule: " + str(schedule))
118
119
120
121
122
                    schedule = None

                if schedule:
                    return_data.append(schedule)
        except TypeError as e:
David Trattnig's avatar
David Trattnig committed
123
            self.logger.error("Nothing fetched ...")
124
125
            self.fetched_schedule_data = None
            return None
126
127
128

        return return_data

David Trattnig's avatar
David Trattnig committed
129
130
131
132
133
134
135
136


    #
    #   PRIVATE METHODS
    #


    # FIXME Refactor for more transparent API requests.
137
    def __set_url__(self, type):
David Trattnig's avatar
David Trattnig committed
138
139
140
141
        """
        Initializes URLs and parameters for API calls.
        """
        url = self.config.get(type)
142
143
144
145
146
147
148
149
        pos = url.find("?")

        if pos > 0:
            self.url[type] = url[0:pos]
            self.url_parameter[type] = url[pos:]
        else:
            self.url[type] = url

David Trattnig's avatar
David Trattnig committed
150
151


152
    def __fetch_schedule_data__(self):
David Trattnig's avatar
David Trattnig committed
153
154
155
156
157
158
        """
        Fetches schedule data from Steering.

        Returns:
            ([Schedule]):   An array of schedules
        """
159
        servicetype = "api_steering_calendar"
160
161
162
        schedule = None

        # fetch data from steering
David Trattnig's avatar
David Trattnig committed
163
164
        url = self.__build_url__(servicetype)
        html_response = self.__fetch_data__(servicetype, url)
165

166
167
168
        # use testdata if response fails or is empty
        if not html_response or html_response == b"[]":
            self.logger.critical("Got no response from Steering!")
169

David Trattnig's avatar
David Trattnig committed
170
171
            # FIXME move hardcoded test-data to separate testing logic.
            html_response = self.get_test_schedules()
172

173
174
175
176
        # convert to dict
        schedule = simplejson.loads(html_response)

        # check data
177
178
179
        if not schedule:
            self.logger.warn("Got no schedule via Playout API (Steering)!")
            return None
180

181
        #self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
182
        return self.remove_unnecessary_data(schedule)
183

David Trattnig's avatar
David Trattnig committed
184
185


186
    def __fetch_schedule_playlists__(self):
David Trattnig's avatar
David Trattnig committed
187
188
189
190
191
        """
        Fetches all playlists including fallback playlists for every schedule.
        This method used the class member `fetched_schedule_data`` to iterate
        over and extend schedule data.
        """
192
193
        # store fetched entries => do not have to fetch playlist_id more than once
        fetched_entries=[]
194
        local_station_fallback_id = str(self.config.get("scheduling_station_fallback_id"))
195

196
197
198
        try:
            for schedule in self.fetched_schedule_data:

David Trattnig's avatar
David Trattnig committed
199
                # Extend schedule with details of show (e.g. slug)
200
201
                schedule = self.__fetch_show_details__(schedule)

202
203
204
205
206
207
                # Get IDs of playlists
                playlist_id = self.get_playlist_id(schedule, "playlist_id")
                schedule_fallback_id = self.get_playlist_id(schedule, "schedule_fallback_id")
                show_fallback_id = self.get_playlist_id(schedule, "show_fallback_id")
                station_fallback_id = self.get_playlist_id(schedule, "station_fallback_id")

David Trattnig's avatar
David Trattnig committed
208
209
                # Retrieve playlist and the fallback playlists for every schedule.
                # If a playlist (like station_fallback) is already fetched, it is not fetched again but reused
210
211
212
213
214
215
216
                schedule["playlist"]          = self.__fetch_schedule_playlist__(schedule, playlist_id,          fetched_entries)
                schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, schedule_fallback_id, fetched_entries)
                schedule["show_fallback"]     = self.__fetch_schedule_playlist__(schedule, show_fallback_id,     fetched_entries)
                schedule["station_fallback"]  = self.__fetch_schedule_playlist__(schedule, station_fallback_id,  fetched_entries)

                # If Steering doesn't provide a station fallback, the local one is used
                if not schedule["station_fallback"] and int(local_station_fallback_id) > 0:
217
218
                    schedule["station_fallback_id"] = local_station_fallback_id 
                    schedule["station_fallback"] = self.__fetch_schedule_playlist__(schedule, local_station_fallback_id, fetched_entries)
219
                    if schedule["station_fallback"]:
David Trattnig's avatar
David Trattnig committed
220
                        self.logger.info("Assigned playlist #%s as local station fallback to schedule #%s" % (local_station_fallback_id, schedule["schedule_id"]))
221
222

        except Exception as e:
223
            self.logger.error("Error while fetching playlists from API endpoints: " + str(e), e)
224

David Trattnig's avatar
David Trattnig committed
225
226


227
    def __fetch_show_details__(self, schedule):
David Trattnig's avatar
David Trattnig committed
228
229
        """
        Fetches details of a show from Steering.
230

David Trattnig's avatar
David Trattnig committed
231
232
233
234
235
236
        Args:
            schedule (Schedule):    A schedule holding a valid `show_id`

        Returns:
            (Schedule):             The given schedule with additional show fields set.
        """
237
        servicetype = "api_steering_show"
David Trattnig's avatar
David Trattnig committed
238
239
240

        url = self.__build_url__(servicetype, "${ID}", str(schedule["show_id"]))
        json_response = self.__fetch_data__(servicetype, url)
241
242
        show_details = simplejson.loads(json_response)

David Trattnig's avatar
David Trattnig committed
243
        # Extend "schedules" with details of "show"
244
        schedule["show_slug"] = show_details["slug"]
David Trattnig's avatar
David Trattnig committed
245
246
247
        ### ...
        ### ... Add more properties here, if needed 
        ### ...
248
249

        return schedule
250

David Trattnig's avatar
David Trattnig committed
251
252


253
254
255
256
257
258
    def get_playlist_id(self, schedule, id_name):
        """
        Extracts the playlist ID for a given playlist (fallback) type.
        """
        playlist_id = str(schedule[id_name])
        if not playlist_id or playlist_id == "None":
David Trattnig's avatar
David Trattnig committed
259
            self.logger.debug("No value defined for '%s' in schedule '#%s'" % (id_name, schedule["schedule_id"]))
260
261
262
263
264
265
266
            return None
        
        return playlist_id



    def __fetch_schedule_playlist__(self, schedule, playlist_id, fetched_playlists):
David Trattnig's avatar
David Trattnig committed
267
268
269
270
271
272
273
274
275
        """
        Fetches the playlist for a given schedule.

        Args:
            schedule (Schedule):    The schedule to fetch playlists for
            id_name (String):       The type of playlist to fetch (e.g. normal vs. fallback)
            fetched_playlists ([]): Previously fetched playlists to avoid re-fetching

        Returns:
276
            (Playlist):             Playlist of type `id_name`
David Trattnig's avatar
David Trattnig committed
277
        """
278
        servicetype = "api_tank_playlist"
279
        playlist = None
280

281
282
283
        if not playlist_id:
            return None

284
        # If playlist is already fetched, use the existing one
David Trattnig's avatar
David Trattnig committed
285
        for playlist in fetched_playlists:
286
287
            if playlist["id"] == playlist_id:
                self.logger.debug("Playlist #%s already fetched" % playlist_id)
David Trattnig's avatar
David Trattnig committed
288
                return playlist
289

290
291
292
293
294
295
296
297
298
299
300
        url = self.__build_url__(servicetype, "${ID}", playlist_id)
        json_response = self.__fetch_data__(servicetype, url)

        if not json_response:
            self.logger.critical(SimpleUtil.red("Playlist #%s could not be fetched or is not available! JSON response: '%s'" % (playlist_id, json_response)))
            return None
        
        try:
            playlist = simplejson.loads(json_response)
        except Exception as e:
            self.logger.critical(SimpleUtil.red("Error while parsing JSON response for playlist: %s" % json_response), e)
301

302
303
304
        fetched_playlists.append(playlist)
        return playlist
 
305
306


David Trattnig's avatar
David Trattnig committed
307
308
309
310
311
312
313
314
315
316
317

    def __fetch_data__(self, type, url):
        """
        Fetches JSON data for the given URL.

        Args:
            url (String):       The API endpoint to call
        
        Returns:
            (Byte[]):           An UTF-8 encoded byte object holding the response
        """
318
319
        html_response = b''

320
321
        # Send request to the API and read the data
        try:
322
            if type not in self.url_parameter:
323
324
325
326
                if self.url[type] == "":
                    return False
                request = urllib.request.Request(url)
            else:
327
                request = urllib.request.Request(url, self.url_parameter[type])
328
329
330

            response = urllib.request.urlopen(request)
            html_response = response.read()
331

332
333
334
335
336
337
        except (urllib.error.URLError, IOError, ValueError) as e:
            self.logger.error("Cannot connect to " + self.url[type] +
                " (type: " + type + ")! Reason: " + str(e.reason))
            #if not self.has_already_fetched:  # first fetch
            #    self.logger.critical("exiting fetch data thread..")
            #    sys.exit()
338
339
340

        self.has_already_fetched = True
        return html_response.decode("utf-8")
341
342
343



344
    def __build_url__(self, type, placeholder=None, value=None):
345
346
347
        """
        Builds an API request URL using passed placeholder and value.
        """
348
349
350
        url = self.url[type]
        if placeholder:
            url = url.replace(placeholder, value)
351
            self.logger.debug("Built API URL: "+url)
352
        return url
353
354
355



356
    def remove_unnecessary_data(self, schedule):
357
358
359
360
        """
        Removes all schedules which are not relevant for 
        further processing.
        """
361
362
363
364
        count_before = len(schedule)
        schedule = self.remove_data_more_than_24h_in_the_future(schedule)
        schedule = self.remove_data_in_the_past(schedule)
        count_after = len(schedule)
365

David Trattnig's avatar
David Trattnig committed
366
        self.logger.debug("Removed %d unnecessary schedules from response. Entries left: %d" % ((count_before - count_after), count_after))
367
        return schedule
368
369
370



371
    def remove_data_more_than_24h_in_the_future(self, schedules):
372
373
374
375
376
377
        """ 
        Removes entries 24h in the future and 12 hours in the past.
        Note: This might influence resuming (in case of a crash)  
        single schedules which are longer than 12 hours long.
        Think e.g. live broadcasts.
        """
378
379
380
381
        items = []
        now = SimpleUtil.timestamp()
        now_plus_24hours = now + (12*60*60)
        now_minus_12hours = now - (12*60*60)
382

383
384
385
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
386

387
388
            if start_time <= now_plus_24hours and start_time >= now_minus_12hours:
                items.append(s)
389

390
        return items
391
392


393
    def remove_data_in_the_past(self, schedules):
394
395
396
397
        """
        Removes all schedules from the past, except the one which is 
        currently playing.
        """
398
399
400
401
402
403
404
        items = []
        now = SimpleUtil.timestamp()
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
            end_time = datetime.strptime(s["end"], "%Y-%m-%dT%H:%M:%S")
            end_time = SimpleUtil.timestamp(end_time)
405

406
            # Append all elements in the future
407
408
409
410
411
            if start_time >= now:
                items.append(s)
             # Append the one which is playing now
            elif start_time < now < end_time:
                items.append(s)
412

413
        return items
414
415


David Trattnig's avatar
David Trattnig committed
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437


    #
    #   TESTING
    #


    def get_test_schedules(self):
        html_response = "{}"

        # use testdata if wanted
        if self.config.get("use_test_data"):
            
            html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                '{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                '{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]'
            self.logger.critical("Using hardcoded Response!")

        return html_response
            
        

438
439
440
441
442
443
444
445
446
    def create_test_data(self, id_name, schedule):
        import random
        rand_id = random.randint(1, 10000)

        while rand_id in self.used_random_playlist_ids:
            rand_id = random.randint(1, 10000)

        self.used_random_playlist_ids.append(rand_id)

447
        # FIXME move hardcoded test-data to separate testing logic.
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
        # HARDCODED Testdata
        if id_name != "playlist_id":
            # FALLBACK TESTDATA

            if rand_id % 3 == 0:  # playlist fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 2 == 0:  # stream fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool fallback
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] == 0 or schedule[id_name] is None:
            # this happens when playlist id is not filled out in pv
            # json_response = '{"playlist_id": 0}'

            if rand_id % 4 == 0:  # playlist with two files
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 3 == 0:  # playlist with jingle and then linein
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}'
            elif rand_id % 2 == 0:  # playlist with jingle and then http stream
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool playlist
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] % 4 == 0:  # playlist with two files
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}'
        elif schedule[id_name] % 3 == 0:  # playlist with jingle and then http stream
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}'
        elif schedule[id_name] % 2 == 0:  # playlist with jingle and then linein
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}'
        else:  # pool playlist
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}'

        self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])

492
        return json_response