calender_fetcher.py 21.1 KB
Newer Older
David Trattnig's avatar
David Trattnig committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21

#
# Aura Engine (https://gitlab.servus.at/aura/engine)
#
# Copyright (C) 2017-2020 - The Aura Engine Team.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.



22
23
24
25
26
27
28
29
import os
import sys
import urllib
import logging
import simplejson

from datetime import datetime, timedelta
#from modules.models.schedule import Schedule
David Trattnig's avatar
David Trattnig committed
30
from modules.base.utils import SimpleUtil
31

David Trattnig's avatar
David Trattnig committed
32

33
class CalendarFetcher:
David Trattnig's avatar
David Trattnig committed
34
35
36
37
    """
    Fetches the schedules, playlists and playlist entries as JSON
    via the API endpoints.
    """
38
39
40
41
42
43
    url = dict()
    url_parameter = dict()
    config = None
    logging = None
    has_already_fetched = False
    fetched_schedule_data = None
David Trattnig's avatar
David Trattnig committed
44

45
    # FIXME another crutch because of the missing TANK
46
47
    used_random_playlist_ids = list()

David Trattnig's avatar
David Trattnig committed
48

49
    def __init__(self, config):
David Trattnig's avatar
David Trattnig committed
50
51
52
53
54
55
        """
        Constructor

        Args:
            config (AuraConfig):    Holds the engine configuration
        """
56
57
        self.config = config
        self.logger = logging.getLogger("AuraEngine")
58
59
60
        self.__set_url__("api_steering_calendar")
        self.__set_url__("api_tank_playlist")
        self.__set_url__("api_steering_show")
David Trattnig's avatar
David Trattnig committed
61
62
63
64
65

    #
    #   PUBLIC METHODS
    #

66
67

    def fetch(self):
David Trattnig's avatar
David Trattnig committed
68
69
70
71
72
        """
        Retrieve all required data from the API.
        """

        # Fetch upcoming schedules from STEERING
73
74
        try:
            self.logger.debug("Fetching schedules from STEERING")
75
            self.fetched_schedule_data = self.__fetch_schedule_data__()
76
        except urllib.error.HTTPError as e:
77
            self.logger.critical("Cannot fetch from " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
78
79
            self.fetched_schedule_data = None
            return None
80
        except (urllib.error.URLError, IOError, ValueError) as e:
81
            self.logger.critical("Cannot connect to " + self.url["api_steering_calendar"] + "! Reason: " + str(e))
82
83
            self.fetched_schedule_data = None
            return None
84

David Trattnig's avatar
David Trattnig committed
85
        # Fetch playlist and fallbacks to the schedules from TANK
86
87
88
89
        try:
            self.logger.debug("Fetching playlists from TANK")
            self.__fetch_schedule_playlists__()
        except urllib.error.HTTPError as e:
90
            self.logger.critical("Cannot fetch from " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
91
92
            self.fetched_schedule_data = None
            return None
93
        except (urllib.error.URLError, IOError, ValueError) as e:
94
            self.logger.critical("Cannot connect to " + self.url["api_tank_playlist"] + "! Reason: " + str(e))
95
96
            self.fetched_schedule_data = None
            return None
97
98

        return_data = []
David Trattnig's avatar
David Trattnig committed
99
        # Gather returndata
100
101
        try:
            for schedule in self.fetched_schedule_data:
David Trattnig's avatar
David Trattnig committed
102
                # Skip schedule if no start or end is given
103
                if "start" not in schedule:
David Trattnig's avatar
David Trattnig committed
104
                    self.logger.warning("No start of schedule given. Skipping schedule: " + str(schedule))
105
106
                    schedule = None
                if "end" not in schedule:
David Trattnig's avatar
David Trattnig committed
107
                    self.logger.warning("No end of schedule given. Skipping schedule: " + str(schedule))
108
109
                    schedule = None
                if "playlist" not in schedule:
David Trattnig's avatar
David Trattnig committed
110
                    self.logger.warning("No playlist for schedule given. Skipping schedule: " + str(schedule))
111
112
113
114
115
116
                    schedule = None

                if schedule:
                    return_data.append(schedule)
        except TypeError as e:
            self.logger.error("Nothing fetched...")
117
118
            self.fetched_schedule_data = None
            return None
119
120
121

        return return_data

David Trattnig's avatar
David Trattnig committed
122
123
124
125
126
127
128
129


    #
    #   PRIVATE METHODS
    #


    # FIXME Refactor for more transparent API requests.
130
    def __set_url__(self, type):
David Trattnig's avatar
David Trattnig committed
131
132
133
134
        """
        Initializes URLs and parameters for API calls.
        """
        url = self.config.get(type)
135
136
137
138
139
140
141
142
        pos = url.find("?")

        if pos > 0:
            self.url[type] = url[0:pos]
            self.url_parameter[type] = url[pos:]
        else:
            self.url[type] = url

David Trattnig's avatar
David Trattnig committed
143
144


145
    def __fetch_schedule_data__(self):
David Trattnig's avatar
David Trattnig committed
146
147
148
149
150
151
        """
        Fetches schedule data from Steering.

        Returns:
            ([Schedule]):   An array of schedules
        """
152
        servicetype = "api_steering_calendar"
153
154
155
        schedule = None

        # fetch data from steering
David Trattnig's avatar
David Trattnig committed
156
157
        url = self.__build_url__(servicetype)
        html_response = self.__fetch_data__(servicetype, url)
158

159
160
161
        # use testdata if response fails or is empty
        if not html_response or html_response == b"[]":
            self.logger.critical("Got no response from Steering!")
162

David Trattnig's avatar
David Trattnig committed
163
164
            # FIXME move hardcoded test-data to separate testing logic.
            html_response = self.get_test_schedules()
165

166
167
168
169
        # convert to dict
        schedule = simplejson.loads(html_response)

        # check data
170
171
172
        if not schedule:
            self.logger.warn("Got no schedule via Playout API (Steering)!")
            return None
173

174
        #self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
175
        return self.remove_unnecessary_data(schedule)
176

David Trattnig's avatar
David Trattnig committed
177
178


179
    def __fetch_schedule_playlists__(self):
David Trattnig's avatar
David Trattnig committed
180
181
182
183
184
        """
        Fetches all playlists including fallback playlists for every schedule.
        This method used the class member `fetched_schedule_data`` to iterate
        over and extend schedule data.
        """
185
186
187
        # store fetched entries => do not have to fetch playlist_id more than once
        fetched_entries=[]

188
189
190
        try:
            for schedule in self.fetched_schedule_data:

David Trattnig's avatar
David Trattnig committed
191
                # Extend schedule with details of show (e.g. slug)
192
193
                schedule = self.__fetch_show_details__(schedule)

David Trattnig's avatar
David Trattnig committed
194
195
196
197
198
199
                # Retrieve playlist and the fallback playlists for every schedule.
                # If a playlist (like station_fallback) is already fetched, it is not fetched again but reused
                schedule["playlist"]          = self.__fetch_schedule_playlist__(schedule, "playlist_id",          fetched_entries)
                schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, "schedule_fallback_id", fetched_entries)
                schedule["show_fallback"]     = self.__fetch_schedule_playlist__(schedule, "show_fallback_id",     fetched_entries)
                schedule["station_fallback"]  = self.__fetch_schedule_playlist__(schedule, "station_fallback_id",  fetched_entries)
200
201
202
203

        except Exception as e:
            self.logger.error("Error: "+str(e))

David Trattnig's avatar
David Trattnig committed
204
205


206
    def __fetch_show_details__(self, schedule):
David Trattnig's avatar
David Trattnig committed
207
208
        """
        Fetches details of a show from Steering.
209

David Trattnig's avatar
David Trattnig committed
210
211
212
213
214
215
        Args:
            schedule (Schedule):    A schedule holding a valid `show_id`

        Returns:
            (Schedule):             The given schedule with additional show fields set.
        """
216
        servicetype = "api_steering_show"
David Trattnig's avatar
David Trattnig committed
217
218
219

        url = self.__build_url__(servicetype, "${ID}", str(schedule["show_id"]))
        json_response = self.__fetch_data__(servicetype, url)
220
221
        show_details = simplejson.loads(json_response)

David Trattnig's avatar
David Trattnig committed
222
        # Extend "schedules" with details of "show"
223
        schedule["show_slug"] = show_details["slug"]
David Trattnig's avatar
David Trattnig committed
224
225
226
        ### ...
        ### ... Add more properties here, if needed 
        ### ...
227
228

        return schedule
229

David Trattnig's avatar
David Trattnig committed
230
231
232
233
234
235
236
237
238
239
240
241
242
243


    def __fetch_schedule_playlist__(self, schedule, id_name, fetched_playlists):
        """
        Fetches the playlist for a given schedule.

        Args:
            schedule (Schedule):    The schedule to fetch playlists for
            id_name (String):       The type of playlist to fetch (e.g. normal vs. fallback)
            fetched_playlists ([]): Previously fetched playlists to avoid re-fetching

        Returns:
            ([Schedule]):   Array of playlists
        """
244
        servicetype = "api_tank_playlist"
245
246
247
248

        # fetch playlists from TANK
        if not "show_slug" in schedule:
            raise ValueError("Missing 'show_slug' for schedule", schedule)
249

250
        slug = str(schedule["show_slug"])
David Trattnig's avatar
David Trattnig committed
251
252
        url = self.__build_url__(servicetype, "${SLUG}", slug)
        json_response = self.__fetch_data__(servicetype, url)
253

254
        # if a playlist is already fetched, do not fetch it again
David Trattnig's avatar
David Trattnig committed
255
256
257
258
259
        for playlist in fetched_playlists:
            # FIXME schedule["playlist_id"] is always None, review if playlist["id"] is valid
            if playlist["id"] == schedule[id_name]:
                self.logger.debug("Playlist #" + str(schedule[id_name]) + " already fetched")
                return playlist
260

261
        if self.config.get("use_test_data"):
262
            # FIXME move hardcoded test-data to separate testing logic.
263
            self.logger.warn("Using test-data for fetch-schedule-playlist")
264
265
266
            json_response = self.create_test_data(id_name, schedule)

        # convert to list
David Trattnig's avatar
David Trattnig committed
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
        playlists = simplejson.loads(json_response)
        pl = None

        if "results" in playlists:
            # FIXME Currently we use hardcoded playlist and fallback assignments due to issues in Dashboard/Steering/Tank
            self.logger.warn("FIXME Currently we use hardcoded playlist and fallback assignments due to issues in Dashboard/Steering/Tank")
            i = 0
            for playlist in playlists["results"]:
                
                pl = playlist
                # FIXME Always use the first playlist, since the schedule.playlist_id is currently not set via Dashboard:
                if i == 0 and id_name == "playlist_id":
                    schedule["playlist_id"] = playlist["id"]
                    break
                # FIXME Currently it's not possible to set & query the fallback for a timeslot/show/station; therefore hardcode it:
                elif i == 1 and id_name == "schedule_fallback_id":
                    schedule["schedule_fallback_id"] = playlist["id"]
                    break
                elif i == 2 and id_name == "show_fallback_id":
                    schedule["show_fallback_id"] = playlist["id"]
                    break
                elif i == 3 and id_name == "station_fallback_id":
                    schedule["station_fallback_id"] = playlist["id"]
                    break   
                else:
                    pl = None             
                i += 1

            if pl:
                # Note: playlists without entries are allowed -> will trigger fallbacks
                if "entries" in pl:
                    for entry in pl["entries"]:
                        if entry["uri"].startswith("file"):
                            entry["filename"] = self.convert_to_filename(entry["uri"])

                fetched_playlists.append(pl)


        return pl
306
307
308
309



    def convert_to_filename(self, uri):
David Trattnig's avatar
David Trattnig committed
310
311
312
313
314
315
316
317
318
        """
        Converts a file-system URI to an actual, absolute path to the file.

        Args:
            uri (String):   The URI of the file

        Returns:
            path (String):  Absolute file path
        """
319
320
        e = self.config.get("audiofolder") + "/" + uri[7:] + ".flac"
        if not os.path.isfile(e):
321
            self.logger.warning("File %s does not exist!" % e)
322
323
        return e

David Trattnig's avatar
David Trattnig committed
324
325
326
327
328
329
330
331
332
333
334
335


    def __fetch_data__(self, type, url):
        """
        Fetches JSON data for the given URL.

        Args:
            url (String):       The API endpoint to call
        
        Returns:
            (Byte[]):           An UTF-8 encoded byte object holding the response
        """
336
337
        html_response = b''

338
339
        # Send request to the API and read the data
        try:
340
            if type not in self.url_parameter:
341
342
343
344
                if self.url[type] == "":
                    return False
                request = urllib.request.Request(url)
            else:
345
                request = urllib.request.Request(url, self.url_parameter[type])
346
347
348

            response = urllib.request.urlopen(request)
            html_response = response.read()
349

350
351
352
353
354
355
        except (urllib.error.URLError, IOError, ValueError) as e:
            self.logger.error("Cannot connect to " + self.url[type] +
                " (type: " + type + ")! Reason: " + str(e.reason))
            #if not self.has_already_fetched:  # first fetch
            #    self.logger.critical("exiting fetch data thread..")
            #    sys.exit()
356
357
358

        self.has_already_fetched = True
        return html_response.decode("utf-8")
359
360
361



362
    def __build_url__(self, type, placeholder=None, value=None):
363
364
365
        """
        Builds an API request URL using passed placeholder and value.
        """
366
367
368
        url = self.url[type]
        if placeholder:
            url = url.replace(placeholder, value)
David Trattnig's avatar
David Trattnig committed
369
            # self.logger.info("built URL: "+url)
370
        return url
371
372
373



374
    def remove_unnecessary_data(self, schedule):
375
376
377
378
        """
        Removes all schedules which are not relevant for 
        further processing.
        """
379
380
381
382
        count_before = len(schedule)
        schedule = self.remove_data_more_than_24h_in_the_future(schedule)
        schedule = self.remove_data_in_the_past(schedule)
        count_after = len(schedule)
383

384
        self.logger.info("Removed %d unnecessary schedules from response. Entries left: %d" % ((count_before - count_after), count_after))
385
        return schedule
386
387
388



389
    def remove_data_more_than_24h_in_the_future(self, schedules):
390
391
392
393
394
395
        """ 
        Removes entries 24h in the future and 12 hours in the past.
        Note: This might influence resuming (in case of a crash)  
        single schedules which are longer than 12 hours long.
        Think e.g. live broadcasts.
        """
396
397
398
399
        items = []
        now = SimpleUtil.timestamp()
        now_plus_24hours = now + (12*60*60)
        now_minus_12hours = now - (12*60*60)
400

401
402
403
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
404

405
406
            if start_time <= now_plus_24hours and start_time >= now_minus_12hours:
                items.append(s)
407

408
        return items
409
410


411
    def remove_data_in_the_past(self, schedules):
412
413
414
415
        """
        Removes all schedules from the past, except the one which is 
        currently playing.
        """
416
417
418
419
420
421
422
        items = []
        now = SimpleUtil.timestamp()
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
            end_time = datetime.strptime(s["end"], "%Y-%m-%dT%H:%M:%S")
            end_time = SimpleUtil.timestamp(end_time)
423

424
            # Append all elements in the future
425
426
427
428
429
            if start_time >= now:
                items.append(s)
             # Append the one which is playing now
            elif start_time < now < end_time:
                items.append(s)
430

431
        return items
432
433


David Trattnig's avatar
David Trattnig committed
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455


    #
    #   TESTING
    #


    def get_test_schedules(self):
        html_response = "{}"

        # use testdata if wanted
        if self.config.get("use_test_data"):
            
            html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                '{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                '{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]'
            self.logger.critical("Using hardcoded Response!")

        return html_response
            
        

456
457
458
459
460
461
462
463
464
    def create_test_data(self, id_name, schedule):
        import random
        rand_id = random.randint(1, 10000)

        while rand_id in self.used_random_playlist_ids:
            rand_id = random.randint(1, 10000)

        self.used_random_playlist_ids.append(rand_id)

465
        # FIXME move hardcoded test-data to separate testing logic.
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
        # HARDCODED Testdata
        if id_name != "playlist_id":
            # FALLBACK TESTDATA

            if rand_id % 3 == 0:  # playlist fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 2 == 0:  # stream fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool fallback
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] == 0 or schedule[id_name] is None:
            # this happens when playlist id is not filled out in pv
            # json_response = '{"playlist_id": 0}'

            if rand_id % 4 == 0:  # playlist with two files
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 3 == 0:  # playlist with jingle and then linein
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}'
            elif rand_id % 2 == 0:  # playlist with jingle and then http stream
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool playlist
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] % 4 == 0:  # playlist with two files
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}'
        elif schedule[id_name] % 3 == 0:  # playlist with jingle and then http stream
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}'
        elif schedule[id_name] % 2 == 0:  # playlist with jingle and then linein
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}'
        else:  # pool playlist
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}'

        self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])

510
        return json_response