calendar.py 14.3 KB
Newer Older
David Trattnig's avatar
David Trattnig committed
1

2
#
David Trattnig's avatar
David Trattnig committed
3
# Aura Engine (https://gitlab.servus.at/aura/engine)
4
#
David Trattnig's avatar
David Trattnig committed
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# Copyright (C) 2017-2020 - The Aura Engine Team.

# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.

# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.


21

22

23
import threading
24
import queue
25
import logging
26

27
from datetime import datetime
28

David Trattnig's avatar
David Trattnig committed
29
# from modules.scheduling.types import PlaylistType
30
from modules.base.utils import SimpleUtil as SU
David Trattnig's avatar
David Trattnig committed
31
from modules.base.models import Schedule, Playlist, PlaylistEntry, PlaylistEntryMetaData
32
from modules.scheduling.calender_fetcher import CalendarFetcher
33
34
35


class AuraCalendarService(threading.Thread):
36
37
38
39
40
41
42
43
44
45
46
47
    """ 
    The `AuraCalendarService` retrieves all current schedules and related
    playlists including audio files from the configured API endpoints and
    stores it in the local database.

    To perform the API queries it utilizes the CalendarFetcher class.


    Attributes:
        #FIXME Review attributes not needed.

    """
48
    queue = None
49
    config = None
50
    logger = None
51
52
    fetched_schedule_data = None
    calendar_fetcher = None
David Trattnig's avatar
David Trattnig committed
53
    stop_event = None
54

55
56
57
58
59
60
61
62

    def __init__(self, config):
        """
        Initializes the class.

        Args:
            config (AuraConfig):    The configuration
        """
63
        threading.Thread.__init__(self)
David Trattnig's avatar
David Trattnig committed
64

65
        self.config = config
66
        self.logger = logging.getLogger("AuraEngine")
67
        self.queue = queue.Queue()
David Trattnig's avatar
David Trattnig committed
68
        self.stop_event = threading.Event()
69
70
        self.calendar_fetcher = CalendarFetcher(config)

71
72


73
    def get_queue(self):
David Trattnig's avatar
David Trattnig committed
74
75
76
        """ 
        Retrieves the queue of fetched data.
        """
77
78
        return self.queue

David Trattnig's avatar
David Trattnig committed
79
80


81
82
    def run(self):
        """
83
84
        Fetch calendar data and store it in the database. Also handles local deletion of remotely
        deleted schedules.
85
86
87

        Returns
            Schedule ([]):  An arrar of retrieved schedules passed via `self.queue`
88
        """
89
        result = []
90
        now_unix = SU.timestamp()
91
        scheduling_window_start = self.config.get("scheduling_window_start")
92

93
        try:
94
            fetched_schedule_data = self.calendar_fetcher.fetch()
95
            self.logger.debug("Schedule data fetched from API: " + str(fetched_schedule_data))
96

97
            # If nothing is fetched, return
98
            if not fetched_schedule_data:
99
100
                self.queue.put("fetching_aborted Nothing fetched")
                return
101
            
102
103
104
105
            # Check if existing schedules have been deleted
            local_schedules = Schedule.select_programme(datetime.now())
            for local_schedule in local_schedules:

106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
                # Only allow deletion of schedules which are deleted before the start of the scheduling window
                if local_schedule.start_unix > now_unix:
                    if (local_schedule.start_unix - scheduling_window_start) > now_unix:

                        # Filter the local schedule from the fetched ones
                        existing_schedule = list(filter(lambda new_schedule: \
                            new_schedule["schedule_id"] == local_schedule.schedule_id, fetched_schedule_data))
                        
                        if existing_schedule:
                            # self.logger.debug("Schedule #%s is still existing remotely!" % (local_schedule.schedule_id))
                            pass
                        else:
                            self.logger.info("Schedule #%s has been deleted remotely, hence also delete it locally [%s]" % \
                                (local_schedule.schedule_id, str(local_schedule)))
                            local_schedule.delete(commit=True)
                            self.logger.info("Deleted local schedule #%s from database" % local_schedule.schedule_id)
122

123
                    else:
124
125
126
                        msg = "Schedule #%s has been deleted remotely. Since the scheduling window has already started, it won't be deleted locally." % \
                            local_schedule.schedule_id
                        self.logger.error(SU.red(msg))
127

128
            # Process fetched schedules    
129
            for schedule in fetched_schedule_data:
David Trattnig's avatar
David Trattnig committed
130
131
132
133
134
135
136
137
138

                # Check schedule for validity
                if "start" not in schedule:
                    self.logger.warning("No 'start' of schedule given. Skipping the schedule: %s " % str(schedule))
                    continue
                if "end" not in schedule:
                    self.logger.warning("No 'end' of schedule given. Skipping the schedule: %s " % str(schedule))
                    continue

139
                # Store the schedule
140
141
                schedule_db = self.store_schedule(schedule)

142
                # Store playlists to play
David Trattnig's avatar
David Trattnig committed
143
                self.store_playlist(schedule_db, schedule_db.playlist_id, schedule["playlist"])
David Trattnig's avatar
David Trattnig committed
144
                if schedule_db.schedule_fallback_id:
David Trattnig's avatar
David Trattnig committed
145
                    self.store_playlist(schedule_db, schedule_db.schedule_fallback_id, schedule["schedule_fallback"])
David Trattnig's avatar
David Trattnig committed
146
                if schedule_db.show_fallback_id:
David Trattnig's avatar
David Trattnig committed
147
                    self.store_playlist(schedule_db, schedule_db.show_fallback_id, schedule["show_fallback"])
David Trattnig's avatar
David Trattnig committed
148
                if schedule_db.station_fallback_id:
David Trattnig's avatar
David Trattnig committed
149
150
151
152
153
154
155
156
157
158
                    self.store_playlist(schedule_db, schedule_db.station_fallback_id, schedule["station_fallback"])


                # self.store_playlist(schedule_db, schedule_db.playlist_id, schedule["playlist"], PlaylistType.DEFAULT.id)
                # if schedule_db.schedule_fallback_id:
                #     self.store_playlist(schedule_db, schedule_db.schedule_fallback_id, schedule["schedule_fallback"], PlaylistType.TIMESLOT.id)
                # if schedule_db.show_fallback_id:
                #     self.store_playlist(schedule_db, schedule_db.show_fallback_id, schedule["show_fallback"], PlaylistType.SHOW.id)
                # if schedule_db.station_fallback_id:
                #     self.store_playlist(schedule_db, schedule_db.station_fallback_id, schedule["station_fallback"], PlaylistType.STATION.id)
159

160
161
162
163
                result.append(schedule_db)

            # Release the mutex
            self.queue.put(result)
164
        except Exception as e:
165
            # Release the mutex
166
            self.logger.warning("Fetching aborted due to: %s" % str(e), e)
167
            self.queue.put("fetching_aborted " + str(e))
168
169
170

        # terminate the thread
        return
171

172
173


174
    def store_schedule(self, schedule):
175
176
177
178
179
180
        """
        Stores the given schedule to the database.

        Args:
            schedule (Schedule):    The schedule
        """
181
        schedule_db = Schedule.select_show_on_datetime(schedule["start"])
Gottfried Gaisbauer's avatar
Gottfried Gaisbauer committed
182
        havetoadd = False
183

184
        if not schedule_db:
185
            self.logger.debug("no schedule with given schedule id in database => create new")
186
            schedule_db = Schedule()
Gottfried Gaisbauer's avatar
Gottfried Gaisbauer committed
187
            havetoadd = True
188
189
190
191


        schedule_db.show_id = schedule["show_id"]
        schedule_db.schedule_id = schedule["schedule_id"]
Gottfried Gaisbauer's avatar
Gottfried Gaisbauer committed
192
193
        schedule_db.schedule_start = schedule["start"]
        schedule_db.schedule_end = schedule["end"]
194
195
196
        schedule_db.show_name = schedule["show_name"]
        schedule_db.show_hosts = schedule["show_hosts"]
        schedule_db.is_repetition = schedule["is_repetition"]
197
        schedule_db.funding_category = schedule["show_fundingcategory"]
Gottfried Gaisbauer's avatar
Gottfried Gaisbauer committed
198
199
200
201
202
        schedule_db.languages = schedule["show_languages"]
        schedule_db.type = schedule["show_type"]
        schedule_db.category = schedule["show_categories"]
        schedule_db.topic = schedule["show_topics"]
        schedule_db.musicfocus = schedule["show_musicfocus"]
203
204
205
206
207
208

        schedule_db.playlist_id = schedule["playlist_id"]
        schedule_db.schedule_fallback_id = schedule["schedule_fallback_id"]
        schedule_db.show_fallback_id = schedule["show_fallback_id"]
        schedule_db.station_fallback_id = schedule["station_fallback_id"]

Gottfried Gaisbauer's avatar
Gottfried Gaisbauer committed
209
        schedule_db.store(add=havetoadd, commit=True)
210
211
212

        return schedule_db

David Trattnig's avatar
David Trattnig committed
213
214


David Trattnig's avatar
David Trattnig committed
215
216
    # def store_playlist(self, schedule_db, playlist_id, fetched_playlist, fallbackplaylist_type=0):
    def store_playlist(self, schedule_db, playlist_id, fetched_playlist):
217
218
219
        """
        Stores the Playlist to the database.
        """
220
        if not playlist_id or not fetched_playlist:
David Trattnig's avatar
David Trattnig committed
221
222
            self.logger.debug(f"Playlist ID#{playlist_id} is not available!")
            # self.logger.debug("Playlist type %s with ID '%s' is not available!" % (fallbackplaylist_type, playlist_id))
223
224
            return
            
225
226
        playlist_db = Playlist.select_playlist_for_schedule(schedule_db.schedule_start, playlist_id)
        havetoadd = False
227

228
229
230
        if not playlist_db:
            playlist_db = Playlist()
            havetoadd = True
231

232
        self.logger.debug("Storing playlist %d for schedule (%s)" % (playlist_id, str(schedule_db)))
233
234
235
        playlist_db.playlist_id = playlist_id
        playlist_db.schedule_start = schedule_db.schedule_start
        playlist_db.show_name = schedule_db.show_name
David Trattnig's avatar
David Trattnig committed
236
        # playlist_db.fallback_type = fallbackplaylist_type
David Trattnig's avatar
David Trattnig committed
237
238
239
240
        if "entries" in fetched_playlist:
            playlist_db.entry_count = len(fetched_playlist["entries"])
        else:
            playlist_db.entry_count = 0
241

242
        playlist_db.store(havetoadd, commit=True)
243
      
David Trattnig's avatar
David Trattnig committed
244
        if playlist_db.entry_count > 0:
245
            self.store_playlist_entries(schedule_db, playlist_db, fetched_playlist)
246

247
248
249
        return playlist_db


David Trattnig's avatar
David Trattnig committed
250

251
    def store_playlist_entries(self, schedule_db, playlist_db, fetched_playlist):
252
253
254
        """
        Stores the playlist entries to the database.
        """
255
        entry_num = 0
256
257
        time_marker = playlist_db.start_unix

258
        self.expand_entry_duration(schedule_db, fetched_playlist)
259
260
        self.delete_orphaned_entries(playlist_db, fetched_playlist)  

261
        for entry in fetched_playlist["entries"]:
262
            entry_db = PlaylistEntry.select_playlistentry_for_playlist(playlist_db.artificial_id, entry_num)
263
            havetoadd = False
264
265
            if not entry_db:
                entry_db = PlaylistEntry()
266
267
                havetoadd = True

268
269
270
            entry_db.entry_start = datetime.fromtimestamp(time_marker)
            entry_db.artificial_playlist_id = playlist_db.artificial_id
            entry_db.entry_num = entry_num
271
            entry_db.duration = SU.nano_to_seconds(entry["duration"])
272
273
274
275
276
277
278

            if "uri" in entry:
                # FIXME Refactor mix of uri/filename/file/source
                entry_db.uri = entry["uri"]
                entry_db.source = entry["uri"]
            if "filename" in entry:
                entry_db.source = entry["filename"]
279

280
            entry_db.store(havetoadd, commit=True)
281

282
283
            if "file" in entry:
                self.store_playlist_entry_metadata(entry_db, entry["file"]["metadata"])
284
285

            entry_num = entry_num + 1
286
            time_marker += entry_db.duration
287

David Trattnig's avatar
David Trattnig committed
288
289


290
291
292
293
294
295
296
297
298
299
300
301
302
303
    def delete_orphaned_entries(self, playlist_db, fetched_playlist):
        """
        Deletes all playlist entries which are beyond the current playlist's `entry_count`.
        Such entries might be existing due to a remotely changed playlist, which now has
        less entries than before.
        """
        new_last_idx = len(fetched_playlist["entries"])
        existing_last_idx = PlaylistEntry.count_entries(playlist_db.artificial_id)-1

        if existing_last_idx < new_last_idx:
            return 

        for entry_num in range(new_last_idx, existing_last_idx+1, 1):
            PlaylistEntry.delete_entry(playlist_db.artificial_id, entry_num)            
304
            self.logger.info(SU.yellow("Deleted playlist entry %s:%s" % (playlist_db.artificial_id, entry_num)))
305
306
307
            entry_num += 1


308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
    def expand_entry_duration(self, schedule_db, fetched_playlist):
        """
        If some playlist entry doesn't have a duration assigned, its duration is expanded to the
        remaining duration of the playlist (= schedule duration minus playlist entries with duration).
        If there's more than one entry without duration, such entries are removed from the playlist.
        """
        total_seconds = (schedule_db.schedule_end - schedule_db.schedule_start).total_seconds()
        total_duration = SU.seconds_to_nano(total_seconds)
        actual_duration = 0        
        missing_duration = []
        idx = 0

        for entry in fetched_playlist["entries"]:
            if not "duration" in entry:
                missing_duration.append(idx)
            else:
                actual_duration += entry["duration"]
            idx += 1
                
        if len(missing_duration) == 1:
            fetched_playlist["entries"][missing_duration[0]]["duration"] = total_duration - actual_duration
            self.logger.info("Expanded duration of playlist entry #%s:%s" % (fetched_playlist["id"], missing_duration[0]))

        elif len(missing_duration) > 1:
            # This case should actually never happen, as TANK doesn't allow more than one entry w/o duration anymore
            for i in reversed(missing_duration[1:-1]):
                self.logger.error(SU.red("Deleted Playlist Entry without duration: %s" % \
                    str(fetched_playlist["entries"][i])))
                del fetched_playlist["entries"][i]


339

340
    def store_playlist_entry_metadata(self, entry_db, metadata):
341
342
343
        """
        Stores the meta-data for a PlaylistEntry.
        """
344
        metadata_db = PlaylistEntryMetaData.select_metadata_for_entry(entry_db.artificial_id)
345
        havetoadd = False
346
347
        if not metadata_db:
            metadata_db = PlaylistEntryMetaData()
348
349
            havetoadd = True

350
        metadata_db.artificial_entry_id = entry_db.artificial_id
351

352
        if "artist" in metadata:
353
            metadata_db.artist = metadata["artist"]
354
355
        else:
            metadata_db.artist = ""
356
        
357
        if "album" in metadata:
358
            metadata_db.album = metadata["album"]
359
360
        else:
            metadata_db.album = ""
361

362
363
        if "title" in metadata:
            metadata_db.title = metadata["title"]
364
        else:
365
            metadata_db.title = ""
366

367
        metadata_db.store(havetoadd, commit=True)
Gottfried Gaisbauer's avatar
Gottfried Gaisbauer committed
368

David Trattnig's avatar
David Trattnig committed
369

David Trattnig's avatar
David Trattnig committed
370

371
    def stop(self):
David Trattnig's avatar
David Trattnig committed
372
        self.stop_event.set()