Commit 0dc3aa77 authored by David Trattnig's avatar David Trattnig
Browse files

PostgreSQL support. #73

parent c76e2915
...@@ -66,6 +66,13 @@ api_engine_store_clock="http://localhost:8008/api/v1/clock" ...@@ -66,6 +66,13 @@ api_engine_store_clock="http://localhost:8008/api/v1/clock"
api_engine_store_health="http://localhost:8008/api/v1/source/health/${ENGINE_NUMBER}" api_engine_store_health="http://localhost:8008/api/v1/source/health/${ENGINE_NUMBER}"
[scheduler] [scheduler]
# Database settings: Use 'postgresql' or 'mysql'
db_type="postgresql"
db_name="aura_engine"
db_user="aura_engine"
db_pass="---SECRET--PASSWORD---"
db_host="localhost"
db_charset="utf8"
# Base path as seen by "engine-core", not accessed by "engine"; this is required to construct the absolute audio file path (check "Audio Store" in the docs) # Base path as seen by "engine-core", not accessed by "engine"; this is required to construct the absolute audio file path (check "Audio Store" in the docs)
# Either provide an absolute base path or a relative one starting in the `engine-core/src` directory. In case of `engine-core` running in docker use `/var/audio/source` # Either provide an absolute base path or a relative one starting in the `engine-core/src` directory. In case of `engine-core` running in docker use `/var/audio/source`
audio_source_folder="audio/source" audio_source_folder="audio/source"
...@@ -76,12 +83,6 @@ audio_playlist_folder="audio/playlist" ...@@ -76,12 +83,6 @@ audio_playlist_folder="audio/playlist"
engine_latency_offset=0.5 engine_latency_offset=0.5
# How often should the calendar be fetched in seconds. This determines the time of the last changes applied, before a specific show aired # How often should the calendar be fetched in seconds. This determines the time of the last changes applied, before a specific show aired
fetching_frequency=30 fetching_frequency=30
# The schedule is fetched on every "fetching_frequency" cycle and stored in a local database
db_user="aura"
db_name="aura_engine"
db_pass="---SECRET--PASSWORD---"
db_host="localhost"
db_charset="utf8"
# The scheduling window defines when the entries of each timeslot are queued for play-out. The windows start at (timeslot.start - window_start) seconds # The scheduling window defines when the entries of each timeslot are queued for play-out. The windows start at (timeslot.start - window_start) seconds
# and ends at (timeslot.end - window.end) seconds. Its also worth noting, that timeslots can only be deleted before the start of the window. # and ends at (timeslot.end - window.end) seconds. Its also worth noting, that timeslots can only be deleted before the start of the window.
scheduling_window_start=60 scheduling_window_start=60
......
...@@ -66,6 +66,13 @@ api_engine_store_clock="http://127.0.0.1:8008/api/v1/clock" ...@@ -66,6 +66,13 @@ api_engine_store_clock="http://127.0.0.1:8008/api/v1/clock"
api_engine_store_health="http://127.0.0.1:8008/api/v1/source/health/${ENGINE_NUMBER}" api_engine_store_health="http://127.0.0.1:8008/api/v1/source/health/${ENGINE_NUMBER}"
[scheduler] [scheduler]
# Database settings: Use 'postgresql' or 'mysql'
db_type="postgresql"
db_name="aura_engine"
db_user="aura_engine"
db_pass="---SECRET--PASSWORD---"
db_host="127.0.0.1"
db_charset="utf8"
# Base path as seen by "engine-core", not accessed by "engine"; this is required to construct the absolute audio file path (check "Audio Store" in the docs) # Base path as seen by "engine-core", not accessed by "engine"; this is required to construct the absolute audio file path (check "Audio Store" in the docs)
# Either provide an absolute base path or a relative one starting in the `engine-core/src` directory. In case of `engine-core` running in docker use `/var/audio/source` # Either provide an absolute base path or a relative one starting in the `engine-core/src` directory. In case of `engine-core` running in docker use `/var/audio/source`
audio_source_folder="/var/audio/source" audio_source_folder="/var/audio/source"
...@@ -76,12 +83,6 @@ audio_playlist_folder="/var/audio/playlist" ...@@ -76,12 +83,6 @@ audio_playlist_folder="/var/audio/playlist"
engine_latency_offset=0.5 engine_latency_offset=0.5
# How often should the calendar be fetched in seconds. This determines the time of the last changes applied, before a specific show aired # How often should the calendar be fetched in seconds. This determines the time of the last changes applied, before a specific show aired
fetching_frequency=300 fetching_frequency=300
# The schedule is fetched on every "fetching_frequency" cycle and stored in a local database
db_user="aura"
db_name="aura_engine"
db_pass="---SECRET--PASSWORD---"
db_host="localhost"
db_charset="utf8"
# The scheduling window defines when the entries of each timeslot are queued for play-out. The windows start at (timeslot.start - window_start) seconds # The scheduling window defines when the entries of each timeslot are queued for play-out. The windows start at (timeslot.start - window_start) seconds
# and ends at (timeslot.end - window.end) seconds. Its also worth noting, that timeslots can only be deleted before the start of the window. # and ends at (timeslot.end - window.end) seconds. Its also worth noting, that timeslots can only be deleted before the start of the window.
scheduling_window_start=60 scheduling_window_start=60
......
...@@ -66,6 +66,13 @@ api_engine_store_clock="http://localhost:8008/api/v1/clock" ...@@ -66,6 +66,13 @@ api_engine_store_clock="http://localhost:8008/api/v1/clock"
api_engine_store_health="http://localhost:8008/api/v1/source/health/${ENGINE_NUMBER}" api_engine_store_health="http://localhost:8008/api/v1/source/health/${ENGINE_NUMBER}"
[scheduler] [scheduler]
# Database settings: Use 'postgresql' or 'mysql'
db_type="postgresql"
db_name="aura_engine"
db_user="aura_engine"
db_pass="---SECRET--PASSWORD---"
db_host="localhost"
db_charset="utf8"
# Base path as seen by "engine-core", not accessed by "engine"; this is required to construct the absolute audio file path (check "Audio Store" in the docs) # Base path as seen by "engine-core", not accessed by "engine"; this is required to construct the absolute audio file path (check "Audio Store" in the docs)
# Either provide an absolute base path or a relative one starting in the `engine-core/src` directory. In case of `engine-core` running in docker use `/var/audio/source` # Either provide an absolute base path or a relative one starting in the `engine-core/src` directory. In case of `engine-core` running in docker use `/var/audio/source`
audio_source_folder="audio/source" audio_source_folder="audio/source"
...@@ -76,12 +83,6 @@ audio_playlist_folder="audio/playlist" ...@@ -76,12 +83,6 @@ audio_playlist_folder="audio/playlist"
engine_latency_offset=0.5 engine_latency_offset=0.5
# How often should the calendar be fetched in seconds. This determines the time of the last changes applied, before a specific show aired # How often should the calendar be fetched in seconds. This determines the time of the last changes applied, before a specific show aired
fetching_frequency=300 fetching_frequency=300
# The schedule is fetched on every "fetching_frequency" cycle and stored in a local database
db_user="aura"
db_name="aura_engine"
db_pass="---SECRET--PASSWORD---"
db_host="localhost"
db_charset="utf8"
# The scheduling window defines when the entries of each timeslot are queued for play-out. The windows start at (timeslot.start - window_start) seconds # The scheduling window defines when the entries of each timeslot are queued for play-out. The windows start at (timeslot.start - window_start) seconds
# and ends at (timeslot.end - window.end) seconds. Its also worth noting, that timeslots can only be deleted before the start of the window. # and ends at (timeslot.end - window.end) seconds. Its also worth noting, that timeslots can only be deleted before the start of the window.
scheduling_window_start=60 scheduling_window_start=60
......
mysqlclient==1.3.12
\ No newline at end of file
\c postgres
create database aura_engine;
create user aura_engine with encrypted password '1234';
grant all privileges on database aura_engine to aura_engine;
psycopg2-binary==2.8.6
\ No newline at end of file
...@@ -4,8 +4,8 @@ ...@@ -4,8 +4,8 @@
- [Install for Development](#install-for-development) - [Install for Development](#install-for-development)
- [Prerequisites](#prerequisites) - [Prerequisites](#prerequisites)
- [Setting up the database](#setting-up-the-database)
- [Preparation](#preparation) - [Preparation](#preparation)
- [Setting up the database](#setting-up-the-database)
- [Configuration](#configuration) - [Configuration](#configuration)
- [Running Engine](#running-engine) - [Running Engine](#running-engine)
- [Daemonized Engine](#daemonized-engine) - [Daemonized Engine](#daemonized-engine)
...@@ -23,30 +23,9 @@ Aura Engine runs on any modern Debian-based OS. It requires at least ...@@ -23,30 +23,9 @@ Aura Engine runs on any modern Debian-based OS. It requires at least
- [`Python 3.8+`](https://www.python.org/downloads/release/python-380/) - [`Python 3.8+`](https://www.python.org/downloads/release/python-380/)
- [`pip`](https://pip.pypa.io/en/stable/) - [`pip`](https://pip.pypa.io/en/stable/)
- [`git`](https://git-scm.com/) - [`git`](https://git-scm.com/)
- ['virtualenv'](https://pypi.org/project/virtualenv/) (development only) - ['virtualenv'](https://pypi.org/project/virtualenv/)
- DBMS server/client libraries for MariaDB or PostgreSQL (see below) - [PostgreSQL 13+](https://www.postgresql.org/) or [MariaDB 10+](https://mariadb.org/)
# Setting up the database
Depending on the DBMS you are planning to use you'll need to have the relevant server/client libaries to be present.
**MariaDB**
```shell
sudo apt install \
python3.8-dev \
default-libmysqlclient-dev \
mariadb-server \
libmariadbclient-dev
```
The following installation script sets up the initial databases and users.
```bash
bash scripts/setup-db.sh
```
As soon as this is completed, carefully check if any error occured. In case your database has been setup automatically, note the relevant credentials for later use in your `engine.ini` configuration.
**Setting up the project structure** **Setting up the project structure**
...@@ -94,6 +73,28 @@ cp config/sample.development.engine.ini config/engine.ini ...@@ -94,6 +73,28 @@ cp config/sample.development.engine.ini config/engine.ini
cp config/sample.production.engine.ini config/engine.ini cp config/sample.production.engine.ini config/engine.ini
``` ```
### Setting up the database
The primary database supported by AURA is PostgreSQL.
```bash
# Additional Python packages for PostgreSQL
pip3 install -r contrib/postgresql-requirements.txt
# Create database and user (change password in script)
sudo -u postgres psql -f contrib/postgresql-create-database.sql
```
Alternatively you can also use MariaDB:
```bash
# Additional Python packages for MariaDB
pip3 install -r contrib/mariadb-requirements.txt
# Create database and user (change password in script)
sudo mysql -u root -p < contrib/mariadb-database.sql
```
You might want to change the password for the database user created by the relevant script.
## Configuration ## Configuration
In your development environment edit following file to configure the engine: In your development environment edit following file to configure the engine:
......
#!/bin/bash
# Check if databases are already set-up
if test -f "$LOCKFILE_DB"; then
echo "Aura Engine Databases are already existing! Skipping..."
else
# Create random password
PASS_ENGINE="$(openssl rand -base64 24)"
# Create databases and users
echo "--- SETTING UP DATABASE AND USERS ---"
echo "Please enter the MySQL/MariaDB root password!"
stty -echo
printf "Password: "
read rootpasswd
stty echo
printf "\n"
echo "---"
echo "Creating database for Aura Engine..."
mysql -uroot -p${rootpasswd} -e "CREATE DATABASE aura_engine CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci;"
mysql -uroot -p${rootpasswd} -e "CREATE USER 'aura'@'localhost' IDENTIFIED BY '${PASS_ENGINE}';"
mysql -uroot -p${rootpasswd} -e "GRANT ALL PRIVILEGES ON aura_engine.* TO 'aura'@'localhost';"
mysql -uroot -p${rootpasswd} -e "FLUSH PRIVILEGES;"
echo "Done."
echo
echo
echo "Please note your database credentials for the next configuration steps:"
echo "-----------------------------------------------------------------------"
echo " Database: 'aura_engine'"
echo " User: 'aura'"
echo " Password: '${PASS_ENGINE}'"
echo "-----------------------------------------------------------------------"
echo
fi
\ No newline at end of file
#!/bin/bash
#
# Setup Database
#
# Set LOCK file location
LOCKFILE_DB=.engine.install-db.lock
# Check if databases are already set-up
if test -f "$LOCKFILE_DB"; then
echo "Aura Engine Databases are already existing! Skipping..."
else
echo "Setting up database ..."
echo
echo "Which database system do you want to use? (Press '1' or '2')"
echo " [1] MariaDB"
echo " [2] Other / Manually"
echo
while true; do
read -rsn1 input
if [ "$input" = "1" ]; then
echo "Creating DB for MariaDB ..."
bash scripts/setup-db-mariadb.sh
break
fi
if [ "$input" = "2" ]; then
echo "Manual database setup selected."
break
fi
done
# Create lockfile to avoid accidential re-creation of the database
touch $LOCKFILE_DB
fi
...@@ -103,18 +103,22 @@ class AuraConfig: ...@@ -103,18 +103,22 @@ class AuraConfig:
return self.__dict__[key] return self.__dict__[key]
def get_database_uri(self): def get_database_uri(self):
""" """
Retrieves the database connection string. Retrieves the database connection string.
""" """
db_name = self.get("db_name") db_name = str(self.get("db_name"))
db_user = self.get("db_user") db_user = str(self.get("db_user"))
db_pass = str(self.get("db_pass")) db_pass = str(self.get("db_pass"))
db_host = self.get("db_host") db_host = str(self.get("db_host"))
db_type = str(self.get("db_type"))
db_charset = self.get("db_charset", "utf8") db_charset = self.get("db_charset", "utf8")
return "mysql://" + db_user + ":" + db_pass + "@" + db_host + "/" + db_name + "?charset=" + db_charset if db_type == "mysql":
return "mysql://" + db_user + ":" + db_pass + "@" + db_host + "/" + db_name + "?charset=" + db_charset
elif db_type == "postgresql":
return f"postgresql+psycopg2://{db_user}:{db_pass}@{db_host}/{db_name}?client_encoding={db_charset}"
else:
return f"Error: invalid database type '{db_type}'"
def load_config(self): def load_config(self):
......
...@@ -109,9 +109,16 @@ class AuraDatabaseModel(): ...@@ -109,9 +109,16 @@ class AuraDatabaseModel():
try: try:
Playlist.is_empty() Playlist.is_empty()
except sa.exc.ProgrammingError as e: except sa.exc.ProgrammingError as e:
errcode = e.orig.args[0] is_available = True
if errcode == 1146: # Error for no such table # PostgreSQL table not available
if e.code == "f405":
is_available = False
# MariaDB table not available
elif e.orig.args[0] == 1146:
is_available = False
if not is_available:
model = AuraDatabaseModel() model = AuraDatabaseModel()
model.recreate_db() model.recreate_db()
else: else:
...@@ -122,7 +129,7 @@ class AuraDatabaseModel(): ...@@ -122,7 +129,7 @@ class AuraDatabaseModel():
def recreate_db(systemexit = False): def recreate_db(systemexit = False):
""" """
Deletes all tables and re-creates the database. Deletes all tables and re-creates the database.
""" """
Base.metadata.drop_all() Base.metadata.drop_all()
Base.metadata.create_all() Base.metadata.create_all()
DB.session.commit() DB.session.commit()
...@@ -166,11 +173,11 @@ class Timeslot(DB.Model, AuraDatabaseModel): ...@@ -166,11 +173,11 @@ class Timeslot(DB.Model, AuraDatabaseModel):
default_show_playlist = relationship("Playlist", default_show_playlist = relationship("Playlist",
primaryjoin="and_(Timeslot.timeslot_start==Playlist.timeslot_start, \ primaryjoin="and_(Timeslot.timeslot_start==Playlist.timeslot_start, \
Timeslot.default_show_playlist_id==Playlist.playlist_id, Timeslot.show_name==Playlist.show_name)", Timeslot.default_show_playlist_id==Playlist.playlist_id, Timeslot.show_name==Playlist.show_name)",
uselist=False, back_populates="timeslot") uselist=False, back_populates="timeslot")
schedule_fallback = relationship("Playlist", schedule_fallback = relationship("Playlist",
primaryjoin="and_(Timeslot.timeslot_start==Playlist.timeslot_start, \ primaryjoin="and_(Timeslot.timeslot_start==Playlist.timeslot_start, \
Timeslot.schedule_fallback_id==Playlist.playlist_id, Timeslot.show_name==Playlist.show_name)", Timeslot.schedule_fallback_id==Playlist.playlist_id, Timeslot.show_name==Playlist.show_name)",
uselist=False, back_populates="timeslot") uselist=False, back_populates="timeslot")
show_fallback = relationship("Playlist", show_fallback = relationship("Playlist",
primaryjoin="and_(Timeslot.timeslot_start==Playlist.timeslot_start, \ primaryjoin="and_(Timeslot.timeslot_start==Playlist.timeslot_start, \
Timeslot.show_fallback_id==Playlist.playlist_id, Timeslot.show_name==Playlist.show_name)", Timeslot.show_fallback_id==Playlist.playlist_id, Timeslot.show_name==Playlist.show_name)",
...@@ -203,7 +210,7 @@ class Timeslot(DB.Model, AuraDatabaseModel): ...@@ -203,7 +210,7 @@ class Timeslot(DB.Model, AuraDatabaseModel):
topic = Column(String(256)) topic = Column(String(256))
musicfocus = Column(String(256)) musicfocus = Column(String(256))
is_repetition = Column(Boolean()) is_repetition = Column(Boolean())
# Transients # Transients
active_entry = None active_entry = None
...@@ -233,7 +240,7 @@ class Timeslot(DB.Model, AuraDatabaseModel): ...@@ -233,7 +240,7 @@ class Timeslot(DB.Model, AuraDatabaseModel):
""" """
timeslots = DB.session.query(Timeslot).\ timeslots = DB.session.query(Timeslot).\
filter(Timeslot.timeslot_start >= date_from).\ filter(Timeslot.timeslot_start >= date_from).\
order_by(Timeslot.timeslot_start).all() order_by(Timeslot.timeslot_start).all()
return timeslots return timeslots
...@@ -278,7 +285,7 @@ class Timeslot(DB.Model, AuraDatabaseModel): ...@@ -278,7 +285,7 @@ class Timeslot(DB.Model, AuraDatabaseModel):
playlist = self.playlist playlist = self.playlist
return { return {
"timeslot_id": self.timeslot_id, "timeslot_id": self.timeslot_id,
"timeslot_start": self.timeslot_start.isoformat(), "timeslot_start": self.timeslot_start.isoformat(),
"timeslot_end": self.timeslot_end.isoformat(), "timeslot_end": self.timeslot_end.isoformat(),
...@@ -291,7 +298,7 @@ class Timeslot(DB.Model, AuraDatabaseModel): ...@@ -291,7 +298,7 @@ class Timeslot(DB.Model, AuraDatabaseModel):
"comment": self.comment, "comment": self.comment,
"playlist_id": self.playlist_id, "playlist_id": self.playlist_id,
"schedule_default_id": self.schedule_default_id, "schedule_default_id": self.schedule_default_id,
"show_default_id": self.show_default_id, "show_default_id": self.show_default_id,
"schedule_fallback_id": self.schedule_fallback_id, "schedule_fallback_id": self.schedule_fallback_id,
"show_fallback_id": self.show_fallback_id, "show_fallback_id": self.show_fallback_id,
"station_fallback_id": self.station_fallback_id, "station_fallback_id": self.station_fallback_id,
...@@ -385,12 +392,12 @@ class Playlist(DB.Model, AuraDatabaseModel): ...@@ -385,12 +392,12 @@ class Playlist(DB.Model, AuraDatabaseModel):
Args: Args:
playlist_id (Integer): The ID of the playlist playlist_id (Integer): The ID of the playlist
Returns: Returns:
(Array<Playlist>): An array holding the playlists (Array<Playlist>): An array holding the playlists
""" """
return DB.session.query(Playlist).filter(Playlist.playlist_id == playlist_id).order_by(Playlist.timeslot_start).all() return DB.session.query(Playlist).filter(Playlist.playlist_id == playlist_id).order_by(Playlist.timeslot_start).all()
@staticmethod @staticmethod
def is_empty(): def is_empty():
...@@ -488,7 +495,7 @@ class PlaylistEntry(DB.Model, AuraDatabaseModel): ...@@ -488,7 +495,7 @@ class PlaylistEntry(DB.Model, AuraDatabaseModel):
# Transients # Transients
entry_start_actual = None # Assigned when the entry is actually played entry_start_actual = None # Assigned when the entry is actually played
channel = None # Assigned when entry is actually played channel = None # Assigned when entry is actually played
queue_state = None # Assigned when entry is about to be queued queue_state = None # Assigned when entry is about to be queued
status = None # Assigned when state changes status = None # Assigned when state changes
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment