calender_fetcher.py 20.3 KB
Newer Older
1
2
3
4
5
6
7
8
import os
import sys
import urllib
import logging
import simplejson

from datetime import datetime, timedelta
#from modules.models.schedule import Schedule
David Trattnig's avatar
David Trattnig committed
9
from modules.base.utils import SimpleUtil
10

David Trattnig's avatar
David Trattnig committed
11

12
class CalendarFetcher:
David Trattnig's avatar
David Trattnig committed
13
14
15
16
    """
    Fetches the schedules, playlists and playlist entries as JSON
    via the API endpoints.
    """
17
18
19
20
21
22
    url = dict()
    url_parameter = dict()
    config = None
    logging = None
    has_already_fetched = False
    fetched_schedule_data = None
David Trattnig's avatar
David Trattnig committed
23

24
    # FIXME another crutch because of the missing TANK
25
26
    used_random_playlist_ids = list()

David Trattnig's avatar
David Trattnig committed
27

28
    def __init__(self, config):
David Trattnig's avatar
David Trattnig committed
29
30
31
32
33
34
        """
        Constructor

        Args:
            config (AuraConfig):    Holds the engine configuration
        """
35
36
        self.config = config
        self.logger = logging.getLogger("AuraEngine")
37
38
39
        self.__set_url__("api_steering_calendar")
        self.__set_url__("api_tank_playlist")
        self.__set_url__("api_steering_show")
David Trattnig's avatar
David Trattnig committed
40
41
42
43
44

    #
    #   PUBLIC METHODS
    #

45
46

    def fetch(self):
David Trattnig's avatar
David Trattnig committed
47
48
49
50
51
        """
        Retrieve all required data from the API.
        """

        # Fetch upcoming schedules from STEERING
52
53
        try:
            self.logger.debug("Fetching schedules from STEERING")
54
            self.fetched_schedule_data = self.__fetch_schedule_data__()
55
        except urllib.error.HTTPError as e:
56
            self.logger.critical("Cannot fetch from " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
57
58
            self.fetched_schedule_data = None
            return None
59
        except (urllib.error.URLError, IOError, ValueError) as e:
60
            self.logger.critical("Cannot connect to " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
61
62
            self.fetched_schedule_data = None
            return None
63

David Trattnig's avatar
David Trattnig committed
64
        # Fetch playlist and fallbacks to the schedules from TANK
65
66
67
68
        try:
            self.logger.debug("Fetching playlists from TANK")
            self.__fetch_schedule_playlists__()
        except urllib.error.HTTPError as e:
69
            self.logger.critical("Cannot fetch from " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
70
71
            self.fetched_schedule_data = None
            return None
72
        except (urllib.error.URLError, IOError, ValueError) as e:
73
            self.logger.critical("Cannot connect to " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
74
75
            self.fetched_schedule_data = None
            return None
76
77

        return_data = []
David Trattnig's avatar
David Trattnig committed
78
        # Gather returndata
79
80
        try:
            for schedule in self.fetched_schedule_data:
David Trattnig's avatar
David Trattnig committed
81
                # Skip schedule if no start or end is given
82
                if "start" not in schedule:
David Trattnig's avatar
David Trattnig committed
83
                    self.logger.warning("No start of schedule given. Skipping schedule: " + str(schedule))
84
85
                    schedule = None
                if "end" not in schedule:
David Trattnig's avatar
David Trattnig committed
86
                    self.logger.warning("No end of schedule given. Skipping schedule: " + str(schedule))
87
88
                    schedule = None
                if "playlist" not in schedule:
David Trattnig's avatar
David Trattnig committed
89
                    self.logger.warning("No playlist for schedule given. Skipping schedule: " + str(schedule))
90
91
92
93
94
95
                    schedule = None

                if schedule:
                    return_data.append(schedule)
        except TypeError as e:
            self.logger.error("Nothing fetched...")
96
97
            self.fetched_schedule_data = None
            return None
98
99
100

        return return_data

David Trattnig's avatar
David Trattnig committed
101
102
103
104
105
106
107
108


    #
    #   PRIVATE METHODS
    #


    # FIXME Refactor for more transparent API requests.
109
    def __set_url__(self, type):
David Trattnig's avatar
David Trattnig committed
110
111
112
113
        """
        Initializes URLs and parameters for API calls.
        """
        url = self.config.get(type)
114
115
116
117
118
119
120
121
        pos = url.find("?")

        if pos > 0:
            self.url[type] = url[0:pos]
            self.url_parameter[type] = url[pos:]
        else:
            self.url[type] = url

David Trattnig's avatar
David Trattnig committed
122
123


124
    def __fetch_schedule_data__(self):
David Trattnig's avatar
David Trattnig committed
125
126
127
128
129
130
        """
        Fetches schedule data from Steering.

        Returns:
            ([Schedule]):   An array of schedules
        """
131
        servicetype = "api_steering_calendar"
132
133
134
        schedule = None

        # fetch data from steering
David Trattnig's avatar
David Trattnig committed
135
136
        url = self.__build_url__(servicetype)
        html_response = self.__fetch_data__(servicetype, url)
137

138
139
140
        # use testdata if response fails or is empty
        if not html_response or html_response == b"[]":
            self.logger.critical("Got no response from Steering!")
141

David Trattnig's avatar
David Trattnig committed
142
143
            # FIXME move hardcoded test-data to separate testing logic.
            html_response = self.get_test_schedules()
144

145
146
147
148
        # convert to dict
        schedule = simplejson.loads(html_response)

        # check data
149
150
151
        if not schedule:
            self.logger.warn("Got no schedule via Playout API (Steering)!")
            return None
152

153
        #self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
154
        return self.remove_unnecessary_data(schedule)
155

David Trattnig's avatar
David Trattnig committed
156
157


158
    def __fetch_schedule_playlists__(self):
David Trattnig's avatar
David Trattnig committed
159
160
161
162
163
        """
        Fetches all playlists including fallback playlists for every schedule.
        This method used the class member `fetched_schedule_data`` to iterate
        over and extend schedule data.
        """
164
165
166
        # store fetched entries => do not have to fetch playlist_id more than once
        fetched_entries=[]

167
168
169
        try:
            for schedule in self.fetched_schedule_data:

David Trattnig's avatar
David Trattnig committed
170
                # Extend schedule with details of show (e.g. slug)
171
172
                schedule = self.__fetch_show_details__(schedule)

David Trattnig's avatar
David Trattnig committed
173
174
175
176
177
178
                # Retrieve playlist and the fallback playlists for every schedule.
                # If a playlist (like station_fallback) is already fetched, it is not fetched again but reused
                schedule["playlist"]          = self.__fetch_schedule_playlist__(schedule, "playlist_id",          fetched_entries)
                schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, "schedule_fallback_id", fetched_entries)
                schedule["show_fallback"]     = self.__fetch_schedule_playlist__(schedule, "show_fallback_id",     fetched_entries)
                schedule["station_fallback"]  = self.__fetch_schedule_playlist__(schedule, "station_fallback_id",  fetched_entries)
179
180
181
182

        except Exception as e:
            self.logger.error("Error: "+str(e))

David Trattnig's avatar
David Trattnig committed
183
184


185
    def __fetch_show_details__(self, schedule):
David Trattnig's avatar
David Trattnig committed
186
187
        """
        Fetches details of a show from Steering.
188

David Trattnig's avatar
David Trattnig committed
189
190
191
192
193
194
        Args:
            schedule (Schedule):    A schedule holding a valid `show_id`

        Returns:
            (Schedule):             The given schedule with additional show fields set.
        """
195
        servicetype = "api_steering_show"
David Trattnig's avatar
David Trattnig committed
196
197
198

        url = self.__build_url__(servicetype, "${ID}", str(schedule["show_id"]))
        json_response = self.__fetch_data__(servicetype, url)
199
200
        show_details = simplejson.loads(json_response)

David Trattnig's avatar
David Trattnig committed
201
        # Extend "schedules" with details of "show"
202
        schedule["show_slug"] = show_details["slug"]
David Trattnig's avatar
David Trattnig committed
203
204
205
        ### ...
        ### ... Add more properties here, if needed 
        ### ...
206
207

        return schedule
208

David Trattnig's avatar
David Trattnig committed
209
210
211
212
213
214
215
216
217
218
219
220
221
222


    def __fetch_schedule_playlist__(self, schedule, id_name, fetched_playlists):
        """
        Fetches the playlist for a given schedule.

        Args:
            schedule (Schedule):    The schedule to fetch playlists for
            id_name (String):       The type of playlist to fetch (e.g. normal vs. fallback)
            fetched_playlists ([]): Previously fetched playlists to avoid re-fetching

        Returns:
            ([Schedule]):   Array of playlists
        """
223
        servicetype = "api_tank_playlist"
224
225
226
227

        # fetch playlists from TANK
        if not "show_slug" in schedule:
            raise ValueError("Missing 'show_slug' for schedule", schedule)
228

229
        slug = str(schedule["show_slug"])
David Trattnig's avatar
David Trattnig committed
230
231
        url = self.__build_url__(servicetype, "${SLUG}", slug)
        json_response = self.__fetch_data__(servicetype, url)
232

233
        # if a playlist is already fetched, do not fetch it again
David Trattnig's avatar
David Trattnig committed
234
235
236
237
238
        for playlist in fetched_playlists:
            # FIXME schedule["playlist_id"] is always None, review if playlist["id"] is valid
            if playlist["id"] == schedule[id_name]:
                self.logger.debug("Playlist #" + str(schedule[id_name]) + " already fetched")
                return playlist
239

240
        if self.config.get("use_test_data"):
241
            # FIXME move hardcoded test-data to separate testing logic.
242
            self.logger.warn("Using test-data for fetch-schedule-playlist")
243
244
245
            json_response = self.create_test_data(id_name, schedule)

        # convert to list
David Trattnig's avatar
David Trattnig committed
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
        playlists = simplejson.loads(json_response)
        pl = None

        if "results" in playlists:
            # FIXME Currently we use hardcoded playlist and fallback assignments due to issues in Dashboard/Steering/Tank
            self.logger.warn("FIXME Currently we use hardcoded playlist and fallback assignments due to issues in Dashboard/Steering/Tank")
            i = 0
            for playlist in playlists["results"]:
                
                pl = playlist
                # FIXME Always use the first playlist, since the schedule.playlist_id is currently not set via Dashboard:
                if i == 0 and id_name == "playlist_id":
                    schedule["playlist_id"] = playlist["id"]
                    break
                # FIXME Currently it's not possible to set & query the fallback for a timeslot/show/station; therefore hardcode it:
                elif i == 1 and id_name == "schedule_fallback_id":
                    schedule["schedule_fallback_id"] = playlist["id"]
                    break
                elif i == 2 and id_name == "show_fallback_id":
                    schedule["show_fallback_id"] = playlist["id"]
                    break
                elif i == 3 and id_name == "station_fallback_id":
                    schedule["station_fallback_id"] = playlist["id"]
                    break   
                else:
                    pl = None             
                i += 1

            if pl:
                # Note: playlists without entries are allowed -> will trigger fallbacks
                if "entries" in pl:
                    for entry in pl["entries"]:
                        if entry["uri"].startswith("file"):
                            entry["filename"] = self.convert_to_filename(entry["uri"])

                fetched_playlists.append(pl)


        return pl
285
286
287
288



    def convert_to_filename(self, uri):
David Trattnig's avatar
David Trattnig committed
289
290
291
292
293
294
295
296
297
        """
        Converts a file-system URI to an actual, absolute path to the file.

        Args:
            uri (String):   The URI of the file

        Returns:
            path (String):  Absolute file path
        """
298
299
        e = self.config.get("audiofolder") + "/" + uri[7:] + ".flac"
        if not os.path.isfile(e):
300
            self.logger.warning("File %s does not exist!" % e)
301
302
        return e

David Trattnig's avatar
David Trattnig committed
303
304
305
306
307
308
309
310
311
312
313
314


    def __fetch_data__(self, type, url):
        """
        Fetches JSON data for the given URL.

        Args:
            url (String):       The API endpoint to call
        
        Returns:
            (Byte[]):           An UTF-8 encoded byte object holding the response
        """
315
316
        html_response = b''

317
318
        # Send request to the API and read the data
        try:
319
            if type not in self.url_parameter:
320
321
322
323
                if self.url[type] == "":
                    return False
                request = urllib.request.Request(url)
            else:
324
                request = urllib.request.Request(url, self.url_parameter[type])
325
326
327

            response = urllib.request.urlopen(request)
            html_response = response.read()
328

329
330
331
332
333
334
        except (urllib.error.URLError, IOError, ValueError) as e:
            self.logger.error("Cannot connect to " + self.url[type] +
                " (type: " + type + ")! Reason: " + str(e.reason))
            #if not self.has_already_fetched:  # first fetch
            #    self.logger.critical("exiting fetch data thread..")
            #    sys.exit()
335
336
337

        self.has_already_fetched = True
        return html_response.decode("utf-8")
338
339
340



341
    def __build_url__(self, type, placeholder=None, value=None):
342
343
344
        """
        Builds an API request URL using passed placeholder and value.
        """
345
346
347
        url = self.url[type]
        if placeholder:
            url = url.replace(placeholder, value)
David Trattnig's avatar
David Trattnig committed
348
            # self.logger.info("built URL: "+url)
349
        return url
350
351
352



353
    def remove_unnecessary_data(self, schedule):
354
355
356
357
        """
        Removes all schedules which are not relevant for 
        further processing.
        """
358
359
360
361
        count_before = len(schedule)
        schedule = self.remove_data_more_than_24h_in_the_future(schedule)
        schedule = self.remove_data_in_the_past(schedule)
        count_after = len(schedule)
362

363
        self.logger.info("Removed %d unnecessary schedules from response. Entries left: %d" % ((count_before - count_after), count_after))
364
        return schedule
365
366
367



368
    def remove_data_more_than_24h_in_the_future(self, schedules):
369
370
371
372
373
374
        """ 
        Removes entries 24h in the future and 12 hours in the past.
        Note: This might influence resuming (in case of a crash)  
        single schedules which are longer than 12 hours long.
        Think e.g. live broadcasts.
        """
375
376
377
378
        items = []
        now = SimpleUtil.timestamp()
        now_plus_24hours = now + (12*60*60)
        now_minus_12hours = now - (12*60*60)
379

380
381
382
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
383

384
385
            if start_time <= now_plus_24hours and start_time >= now_minus_12hours:
                items.append(s)
386

387
        return items
388
389


390
    def remove_data_in_the_past(self, schedules):
391
392
393
394
        """
        Removes all schedules from the past, except the one which is 
        currently playing.
        """
395
396
397
398
399
400
401
        items = []
        now = SimpleUtil.timestamp()
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
            end_time = datetime.strptime(s["end"], "%Y-%m-%dT%H:%M:%S")
            end_time = SimpleUtil.timestamp(end_time)
402

403
            # Append all elements in the future
404
405
406
407
408
            if start_time >= now:
                items.append(s)
             # Append the one which is playing now
            elif start_time < now < end_time:
                items.append(s)
409

410
        return items
411
412


David Trattnig's avatar
David Trattnig committed
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434


    #
    #   TESTING
    #


    def get_test_schedules(self):
        html_response = "{}"

        # use testdata if wanted
        if self.config.get("use_test_data"):
            
            html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                '{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                '{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]'
            self.logger.critical("Using hardcoded Response!")

        return html_response
            
        

435
436
437
438
439
440
441
442
443
    def create_test_data(self, id_name, schedule):
        import random
        rand_id = random.randint(1, 10000)

        while rand_id in self.used_random_playlist_ids:
            rand_id = random.randint(1, 10000)

        self.used_random_playlist_ids.append(rand_id)

444
        # FIXME move hardcoded test-data to separate testing logic.
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
        # HARDCODED Testdata
        if id_name != "playlist_id":
            # FALLBACK TESTDATA

            if rand_id % 3 == 0:  # playlist fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 2 == 0:  # stream fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool fallback
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] == 0 or schedule[id_name] is None:
            # this happens when playlist id is not filled out in pv
            # json_response = '{"playlist_id": 0}'

            if rand_id % 4 == 0:  # playlist with two files
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 3 == 0:  # playlist with jingle and then linein
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}'
            elif rand_id % 2 == 0:  # playlist with jingle and then http stream
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool playlist
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] % 4 == 0:  # playlist with two files
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}'
        elif schedule[id_name] % 3 == 0:  # playlist with jingle and then http stream
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}'
        elif schedule[id_name] % 2 == 0:  # playlist with jingle and then linein
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}'
        else:  # pool playlist
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}'

        self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])

489
        return json_response