calender_fetcher.py 19.2 KB
Newer Older
1
2
3
4
5
6
7
8
import os
import sys
import urllib
import logging
import simplejson

from datetime import datetime, timedelta
#from modules.models.schedule import Schedule
9
from modules.base.simpleutil import SimpleUtil
10
11
12
13
14
15
16
17

class CalendarFetcher:
    url = dict()
    url_parameter = dict()
    config = None
    logging = None
    has_already_fetched = False
    fetched_schedule_data = None
18
    # FIXME another crutch because of the missing TANK
19
20
21
22
23
24
25
    used_random_playlist_ids = list()

    def __init__(self, config):
        self.config = config
        self.logger = logging.getLogger("AuraEngine")
        self.__set_url__("calendar")
        self.__set_url__("importer")
26
        self.__set_url__("api_show_")
27
28
29
30
31

    def fetch(self):
        # fetch upcoming schedules from STEERING
        try:
            self.logger.debug("Fetching schedules from STEERING")
32
            self.fetched_schedule_data = self.__fetch_schedule_data__()
33
34
        except urllib.error.HTTPError as e:
            self.logger.critical("Cannot fetch from " + self.url["calendar"] + "! Reason: " + str(e))
35
36
            self.fetched_schedule_data = None
            return None
37
38
        except (urllib.error.URLError, IOError, ValueError) as e:
            self.logger.critical("Cannot connect to " + self.url["calendar"] + "! Reason: " + str(e))
39
40
            self.fetched_schedule_data = None
            return None
41
42
43
44
45
46
47

        # fetch playlist and fallbacks to the schedules from TANK
        try:
            self.logger.debug("Fetching playlists from TANK")
            self.__fetch_schedule_playlists__()
        except urllib.error.HTTPError as e:
            self.logger.critical("Cannot fetch from " + self.url["importer"] + "! Reason: " + str(e))
48
49
            self.fetched_schedule_data = None
            return None
50
51
        except (urllib.error.URLError, IOError, ValueError) as e:
            self.logger.critical("Cannot connect to " + self.url["importer"] + "! Reason: " + str(e))
52
53
            self.fetched_schedule_data = None
            return None
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73

        return_data = []
        # gather returndata
        try:
            for schedule in self.fetched_schedule_data:
                # skip schedule if no start or end is given
                if "start" not in schedule:
                    self.logger.warning("No start of schedule given. skipping schedule: " + str(schedule))
                    schedule = None
                if "end" not in schedule:
                    self.logger.warning("No end of schedule given. skipping schedule: " + str(schedule))
                    schedule = None
                if "playlist" not in schedule:
                    self.logger.warning("No playlist for schedule given. skipping schedule: " + str(schedule))
                    schedule = None

                if schedule:
                    return_data.append(schedule)
        except TypeError as e:
            self.logger.error("Nothing fetched...")
74
75
            self.fetched_schedule_data = None
            return None
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91

        return return_data

    # ------------------------------------------------------------------------------------------ #
    def __set_url__(self, type):
        url = self.config.get(type+"url")
        pos = url.find("?")

        if pos > 0:
            self.url[type] = url[0:pos]
            self.url_parameter[type] = url[pos:]
        else:
            self.url[type] = url

    # ------------------------------------------------------------------------------------------ #
    def __fetch_schedule_data__(self):
92
        servicetype = "calendar"
93
94
95
        schedule = None

        # fetch data from steering
96
        html_response = self.__fetch_data__(servicetype)
97

98
99
100
101
102
        # FIXME move hardcoded test-data to separate testing logic.
        # use testdata if response fails or is empty
        if not html_response or html_response == b"[]":
            self.logger.critical("Got no response from Steering!")
            #html_response = '[{"schedule_id":1,"start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%d %H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%d %H:00:00') + '","show_id":9,"show_name":"FROzine","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"},{"schedule_id":2,"schedule_start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%d %H:00:00') + '","schedule_end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%d %H:00:00') + '","show_id":10,"show_name":"FROMat","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"},{"schedule_id":3,"schedule_start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%d %H:00:00') + '","schedule_end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%d %H:00:00') + '","show_id":11,"show_name":"Radio für Senioren","show_hosts":"Sandra Hochholzer, Martina Schweiger","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"string","comment":"Kommentar","languages":"Sprachen","type":"Typ","category":"Kategorie","topic":"Topic","musicfocus":"Fokus"}]'
103
104
105

            # use testdata if wanted
            if self.config.get("use_test_data"):
106
                # FIXME move hardcoded test-data to separate testing logic.
107
108
109
                html_response = '[{"id":1,"schedule_id":1,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=0)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":9,"show_name":"TestData: FROzine","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":2,"schedule_fallback_id":12,"show_fallback_id":92,"station_fallback_id":1,"rtr_category":"string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                 '{"id":2,"schedule_id":2,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=1)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":10,"show_name":"TestData: FROMat","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":4,"schedule_fallback_id":22,"show_fallback_id":102,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"},' \
                                 '{"id":3,"schedule_id":3,"automation-id":1,"className":"TestData","memo":"TestData","show_fundingcategory":"TestData","start":"' + (datetime.now() + timedelta(hours=2)).strftime('%Y-%m-%dT%H:00:00') + '","end":"' + (datetime.now() + timedelta(hours=3)).strftime('%Y-%m-%dT%H:00:00') + '","show_id":11,"show_name":"TestData: Radio für Senioren","show_hosts":"TestData: Sandra Hochholzer, Martina Schweiger","title":"TestData:title","is_repetition":false,"playlist_id":6,"schedule_fallback_id":32,"show_fallback_id":112,"station_fallback_id":1,"rtr_category":"TestData: string","comment":"TestData: Kommentar","show_languages":"TestData: Sprachen","show_type":"TestData: Typ","show_categories":"TestData: Kategorie","show_topics":"TestData: Topic","show_musicfocus":"TestData: Fokus"}]'
110
                self.logger.critical("Using hardcoded Response!")
111
112
113
            else:
                html_response = "{}"

114

115
116
117
118
        # convert to dict
        schedule = simplejson.loads(html_response)

        # check data
119
120
121
        if not schedule:
            self.logger.warn("Got no schedule via Playout API (Steering)!")
            return None
122

123
        #self.fetched_schedule_data = self.remove_unnecessary_data(schedule)
124
        return self.remove_unnecessary_data(schedule)
125
126
127
128
129
130

    # ------------------------------------------------------------------------------------------ #
    def __fetch_schedule_playlists__(self):
        # store fetched entries => do not have to fetch playlist_id more than once
        fetched_entries=[]

131
132
133
134
135
136
137
138
139
140
141
142
143
        try:
            self.logger.warning("only fetching normal playlists. no fallbacks")
            for schedule in self.fetched_schedule_data:

                # Enhance schedule with details of show (e.g. slug)
                schedule = self.__fetch_show_details__(schedule)
                # retrieve playlist and the fallbacks for every schedule
                # if a playlist (like station_fallback) is already fetched, it is not fetched again but reused
                schedule["playlist"]          = self.__fetch_schedule_playlist__(schedule, "playlist_id",          fetched_entries)
                #schedule["schedule_fallback"] = self.__fetch_schedule_playlist__(schedule, "schedule_fallback_id", fetched_entries)
                #schedule["show_fallback"]     = self.__fetch_schedule_playlist__(schedule, "show_fallback_id",     fetched_entries)
                #schedule["station_fallback"]  = self.__fetch_schedule_playlist__(schedule, "station_fallback_id",  fetched_entries)

144
                #self.logger.info(str(schedule))
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160

        except Exception as e:
            self.logger.error("Error: "+str(e))

    # ------------------------------------------------------------------------------------------ #
    def __fetch_show_details__(self, schedule):
        servicetype = "api_show_"

        json_response = self.__fetch_data__(servicetype, "${ID}", str(schedule["show_id"]))
        show_details = simplejson.loads(json_response)

        # Augment "schedules" with details of "show"
        schedule["show_slug"] = show_details["slug"]
        ### ... add more properties here, if needed ... ###

        return schedule
161
162
163

    # ------------------------------------------------------------------------------------------ #
    def __fetch_schedule_playlist__(self, schedule, id_name, fetched_schedule_entries):
164
165
166
167
168
        servicetype = "importer"

        # fetch playlists from TANK
        if not "show_slug" in schedule:
            raise ValueError("Missing 'show_slug' for schedule", schedule)
169

170
171
        slug = str(schedule["show_slug"])
        json_response = self.__fetch_data__(servicetype, "${SLUG}", slug)
172

173
174
        # if a playlist is already fetched, do not fetch it again
        for entry in fetched_schedule_entries:
175
176
            # FIXME schedule["playlist_id"] is always None, review if entry["id"] is valid
            if entry["id"] == schedule[id_name]:
177
178
179
                self.logger.debug("playlist #" + str(schedule[id_name]) + " already fetched")
                return entry

180
        if self.config.get("use_test_data"):
181
            # FIXME move hardcoded test-data to separate testing logic.
182
            self.logger.warn("Using test-data for fetch-schedule-playlist")
183
184
185
186
            json_response = self.create_test_data(id_name, schedule)

        # convert to list
        schedule_entries = simplejson.loads(json_response)
187
188
        if "results" in schedule_entries:
            schedule_entries = schedule_entries["results"][0]
189
190
191
192
193
194
195
196
197
198
199
200
201

            for entry in schedule_entries["entries"]:
                if entry["uri"].startswith("file"):
                    entry["filename"] = self.convert_to_filename(entry["uri"])

            fetched_schedule_entries.append(schedule_entries)

        return schedule_entries

    def convert_to_filename(self, uri):
        # convert to normal filename
        e = self.config.get("audiofolder") + "/" + uri[7:] + ".flac"
        if not os.path.isfile(e):
202
            self.logger.warning("File %s does not exist!" % e)
203
204
205
        return e

    # ------------------------------------------------------------------------------------------ #
206
207
    def __fetch_data__(self, type, placeholder=None, value=None):
        # Init html_response
208
        html_response = b''
209
        url = self.__build_url__(type, placeholder, value)
210

211
212
        # Send request to the API and read the data
        try:
213
            if type not in self.url_parameter:
214
215
216
217
                if self.url[type] == "":
                    return False
                request = urllib.request.Request(url)
            else:
218
                request = urllib.request.Request(url, self.url_parameter[type])
219
220
221

            response = urllib.request.urlopen(request)
            html_response = response.read()
222

223
224
225
226
227
228
        except (urllib.error.URLError, IOError, ValueError) as e:
            self.logger.error("Cannot connect to " + self.url[type] +
                " (type: " + type + ")! Reason: " + str(e.reason))
            #if not self.has_already_fetched:  # first fetch
            #    self.logger.critical("exiting fetch data thread..")
            #    sys.exit()
229
230
231

        self.has_already_fetched = True
        return html_response.decode("utf-8")
232
233
234



235
    def __build_url__(self, type, placeholder=None, value=None):
236
237
238
        """
        Builds an API request URL using passed placeholder and value.
        """
239
240
241
242
243
        url = self.url[type]
        if placeholder:
            url = url.replace(placeholder, value)
            # print("built URL: "+url)
        return url
244
245
246



247
    def remove_unnecessary_data(self, schedule):
248
249
250
251
        """
        Removes all schedules which are not relevant for 
        further processing.
        """
252
253
254
255
        count_before = len(schedule)
        schedule = self.remove_data_more_than_24h_in_the_future(schedule)
        schedule = self.remove_data_in_the_past(schedule)
        count_after = len(schedule)
256

257
        self.logger.info("Removed %d unnecessary schedules from response. Entries left: %d" % ((count_before - count_after), count_after))
258
        return schedule
259
260
261



262
    def remove_data_more_than_24h_in_the_future(self, schedules):
263
264
265
266
267
268
        """ 
        Removes entries 24h in the future and 12 hours in the past.
        Note: This might influence resuming (in case of a crash)  
        single schedules which are longer than 12 hours long.
        Think e.g. live broadcasts.
        """
269
270
271
272
        items = []
        now = SimpleUtil.timestamp()
        now_plus_24hours = now + (12*60*60)
        now_minus_12hours = now - (12*60*60)
273

274
275
276
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
277

278
279
            if start_time <= now_plus_24hours and start_time >= now_minus_12hours:
                items.append(s)
280

281
        return items
282
283


284
    def remove_data_in_the_past(self, schedules):
285
286
287
288
        """
        Removes all schedules from the past, except the one which is 
        currently playing.
        """
289
290
291
292
293
294
295
        items = []
        now = SimpleUtil.timestamp()
        for s in schedules:
            start_time = datetime.strptime(s["start"], "%Y-%m-%dT%H:%M:%S")
            start_time = SimpleUtil.timestamp(start_time)
            end_time = datetime.strptime(s["end"], "%Y-%m-%dT%H:%M:%S")
            end_time = SimpleUtil.timestamp(end_time)
296

297
            # Append all elements in the future
298
299
300
301
302
            if start_time >= now:
                items.append(s)
             # Append the one which is playing now
            elif start_time < now < end_time:
                items.append(s)
303

304
        return items
305
306


307
308
309
310
311
312
313
314
315
316
    # ------------------------------------------------------------------------------------------ #
    def create_test_data(self, id_name, schedule):
        import random
        rand_id = random.randint(1, 10000)

        while rand_id in self.used_random_playlist_ids:
            rand_id = random.randint(1, 10000)

        self.used_random_playlist_ids.append(rand_id)

317
        # FIXME move hardcoded test-data to separate testing logic.
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
        # HARDCODED Testdata
        if id_name != "playlist_id":
            # FALLBACK TESTDATA

            if rand_id % 3 == 0:  # playlist fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 2 == 0:  # stream fallback
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool fallback
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///liedermacherei"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] == 0 or schedule[id_name] is None:
            # this happens when playlist id is not filled out in pv
            # json_response = '{"playlist_id": 0}'

            if rand_id % 4 == 0:  # playlist with two files
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/02 - Only Child - Breakneck.flac"}]}'
            elif rand_id % 3 == 0:  # playlist with jingle and then linein
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://1"}]}'
            elif rand_id % 2 == 0:  # playlist with jingle and then http stream
                json_response = '{"playlist_id":' + str(
                    rand_id) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://chill.out.airtime.pro:8000/chill_a"}]}'
            else:  # pool playlist
                json_response = '{"playlist_id":' + str(rand_id) + ',"entries":[{"source":"pool:///hiphop"}]}'

            schedule[id_name] = rand_id

        elif schedule[id_name] % 4 == 0:  # playlist with two files
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"file:///var/audio/fallback/NightmaresOnWax/DJ-Kicks/01 - Type - Slow Process.flac"}]}'
        elif schedule[id_name] % 3 == 0:  # playlist with jingle and then http stream
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"linein://0"}]}'
        elif schedule[id_name] % 2 == 0:  # playlist with jingle and then linein
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"file:///var/audio/fallback/music.flac"},{"source":"http://stream.fro.at:80/fro-128.ogg"}]}'
        else:  # pool playlist
            json_response = '{"playlist_id":' + str(schedule[id_name]) + ',"entries":[{"source":"pool:///chillout"}]}'

        self.logger.info("Using 'randomized' playlist: " + json_response + " for " + id_name[:-3] + " for show " + schedule["show_name"] + " starting @ " + schedule["start"])

362
        return json_response