Merge branch 'feature-dockerized' into 'develop'

Adaptation for an AuRa wide docker setup

See merge request !13

.env.example

+# This has to be set to any good enough random string. E.g. something that
+# `pwgen -s 32 1` would print out. If you want to know more about this go to
+# https://docs.djangoproject.com/en/3.2/ref/settings/#secret-key
+# (mandatory setting)
+SECRET_KEY=put-something-awesomely-random-here
+
+# A comma-separated list of hostnames/IPs Django should listen to. For a
+# production setup this will be something like aura.example.org, for a dev
+# setup you might just use the default settings.
+# (default: 127.0.0.1, localhost)
+#ALLOWED_HOSTS=
+
+# A comma-separated list of URIs where the webclients live that should be able
+# to access the steering API. In particular the dashboard. Might not be needed
+# in a production setup if steering and dashboard share the same domain. In
+# a dev setup the defaults might be just fine.
+# (default: http://127.0.0.1:8080, http://localhost:8080)
+#CORS_ORIGIN_WHITELIST=
+
+# The database settings.
+# if you use a dev environment where django is not running inside a docker
+# container, but you use the postgres container for the db and map its port,
+# then use localhost as the database hostname
+# (default host: steering-postgres ; or if RUN_IN_DOCKER is False: localhost)
+# (default port: 5432)
+# (default name: steering)
+# (default user: steering)
+# (pass is a mandatory setting)
+#DBHOST=
+#DBPORT=
+#DBNAME=
+#DBUSER=
+DBPASS=change-to-something-secure
+
+# The timezone of this server. For a list of all available tz database names see
+# https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
+# (default: Europe/Vienna)
+#TIME_ZONE=
+
+# The language code for the localization of this server. For a list of available
+# codes see http://www.i18nguy.com/unicode/language-identifiers.html
+# (default: de)
+#LANGUAGE_CODE=
+
+# If steering is run inside a docker container. This will be de default for a
+# production deployment. In a dev scenario you might still want to have the
+# database in its container, but run the steering dev server directly on your
+# host. In this case make this False.
+# (default: True)
+#RUN_IN_DOCKER=
+
+# This should be turned on only for your development environment unless you
+# know exactly what you are doing and what the consequences are.
+# (default: False)
+#DEBUG=

.gitignore

@@ -13,4 +13,5 @@ pv/cache/*
 pv/mysql.cnf
 pv/postgresql.cnf
 venv/
-python/
\ No newline at end of file
+python/
+.env

Dockerfile

-FROM python:3.6
+FROM python:3.8 AS base
 
-COPY requirements.txt /tmp/
+ENV PYTHONUNBUFFERED=1
 
-RUN pip install --no-cache-dir -r /tmp/requirements.txt
+WORKDIR /aura
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
 
-#RUN python /tmp/manage.py migrate
-#ENTRYPOINT ["/tmp/entrypoint.sh"]
+EXPOSE 8000
 
-# Superuser parameters are --username USERNAME and --email EMAIL but there is no password
-# RUN python /tmp/manage.py createsuperuser
+
+FROM base AS dev
+
+#VOLUME .
+CMD ["python", "manage.py", "runserver", "0.0.0.0:8000" ]
+
+
+FROM base AS prod
+
+COPY . .
+CMD ["sh", "-c", "gunicorn -b 0.0.0.0:8000 -w $(nproc) pv.wsgi"]

README.rst

@@ -5,28 +5,32 @@ AURA Steering: Program Scheduler
 Installation
 ------------
 
-If you have a working Docker environment, do::
+If you want to have quick dev server and already have the *steering-postgres*
+container of the *aura-web* repository running, follow the steps outlined in the
+*Setting up the configuration* section and do::
 
-    $ docker build -t pv_container .
-    $ docker run -p 8000:8000 -d pv_container:latest
+    $ docker build -t steering-django --target dev .
+    $ docker run --rm -v $(pwd):/aura --network auraweb_auranet steering-django:latest python manage.py migrate
+    $ docker run -it --rm -v $(pwd):/aura --network auraweb_auranet steering-django:latest python manage.py createsuperuser
+    $ docker run --rm -p 8000:8000 -v $(pwd):/aura --network auraweb_auranet steering-django:latest
 
-and log into it at http://127.0.0.1:8000/admin/ with username "admin" and password "admin". Full setup without Docker is done as described below.
+and log into it at http://127.0.0.1:8000/admin/ with the credentials you have
+set in the ``createsuperuser`` step. Once this is done, every other time you
+want to start a *steering-django* container, you will only have to use the
+last command.
+
+**Full setup without Docker** is done as described below.
 
 To get setup you must have the following installed:
 
-* PostgresSQL or MySQL client development libraries
+* PostgresSQL (except you are using the aura-web docker container *steering-postgres* for it)
 * JPEG library development files
 * Python 3.8 or later including development files
 
-In Debian or Ubuntu (or derivatives) you should be able to achieve this with this command:
-
-* Using PostgreSQL::
+In Debian or Ubuntu (or derivatives) you should be able to achieve this with this command::
 
     $ sudo apt-get install postgresql postgresql-contrib libjpeg-dev python3 python3-dev
 
-* Using MySQL::
-
-    $ sudo apt-get install libmysqlclient-dev libjpeg-dev python3 python3-dev
 
 Setting up the environment
 --------------------------
@@ -44,57 +48,21 @@ Change into the base directory of this software and install the project dependen
 Setting up the configuration
 ----------------------------
 
-By default the project is set up to run on a SQLite database.
-
-Create a file ``pv/local_settings.py`` and add at least the following line::
-
-    SECRET_KEY = 'secret key'
-
-(obviously replacing "secret key" with a key of your choice).
-
-Setting up PostgreSQL
----------------------
-
-We recommend using PostgreSQL in order to be able to use the collation utf8mb64_unicode_ci and thus being able to display all languages.
-
-To use PostgreSQL, add the following to your ``local_settings.py`` (before migrating) and add your credentials::
-
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.postgresql',
-            'NAME': '',
-            'USER': '',
-            'PASSWORD': '',
-            'HOST': 'localhost',
-            'PORT': '5432'
-        }
-    }
+Copy the ``.env.example`` file to ``.env`` and change the values accordingly.
+You have to at least provide the ``SECRET_KEY`` and the ``DBPASS`` values
+for Django to start. The file provides extensive comments on all the settings.
+For a developments environment the defaults should be just fine if you use the
+*steering-postgres* docker container from the *aura-web* repository. If you
+want to create your own database on you local machine, you will have to use
+*steering* as the database and user name, or adopt the ``DB*`` values
+accordingly.
 
-Setting up MySQL
-----------------
+Also be aware that there is a ``RUN_IN_DOCKER`` setting that is ``True`` by
+default. This should be fine for a production environment where Django is
+run inside a container. If you follow these steps here to create your own
+development environment you have to set this setting to ``False``. You also
+might want to set ``DEBUG`` to ``True`` for your development environment.
 
-**Note:** When adding your database, make sure you **don't** use the collation utf8mb4_unicode_ci or you will get a key length error during migration. (use e.g. utf8_general_ci instead).
-
-To use MySQL, add the following to your ``local_settings.py`` (before migrating)::
-
-    DATABASES = {
-        'default': {
-            'ENGINE': 'django.db.backends.mysql',
-            'OPTIONS': {
-                'read_default_file': os.path.join(PROJECT_DIR, 'mysql.cnf'),
-            },
-        }
-    }
-
-Create a file ``pv/mysql.cnf`` and give your MySQL credentials::
-
-    [client]
-    database =
-    host = localhost
-    port = 3306
-    user =
-    password =
-    default-character-set = utf8
 
 Setting up the database
 -----------------------
@@ -119,6 +87,30 @@ In development you should run::
     (python)$ python manage.py runserver
 
 
-After this you can open http://127.0.0.1:8000/admin in your browser and log in with username "admin" and password "admin".
+After this you can open http://127.0.0.1:8000/admin in your browser and log in
+with the credentials you have chosen in the ``createsuperuser`` command.
+
+If you are using some placeholder credentials, make sure to change your password
+by visiting http://127.0.0.1:8000/admin/auth/user/1/password/
+
+Configuring OpenID Connect clients
+----------------------------------
+
+To make AuRa usable, you have to set up OpenID Connect (OIDC) clients for the
+*dashboard* and *tank*, so they can make authenticated requests on behalf of
+the user against the *steering* API.
+
+To do so, you can either visit the Django admin interface and create an RSA key
+as well as two clients, or do so programmatically by running::
+
+    (python)$ python manage.py creatersakey
+    (python)$ python manage.py create_oidc_client dashboard public -r "id_token token" -u https://aura-test.o94.at/oidc_callback.html -u https://aura-test.o94.at/oidc_callback_silentRenew.html -p https://aura-test.o94.at/
+    (python)$ python manage.py create_oidc_client tank confidential -r "code" -u https://aura-test.o94.at/tank/auth/oidc/callback
+
+In these examples you will have to *https://aura-test.o94.at* and
+*https://aura-test.o94.at/tank_with* with wherever *dashboard* and *tank* are
+running in your setup. In a local development environment this might be
+something like *http://localhost:8080* and *http://localhost:4000* respectively.
 
-Make sure to change your password by visiting http://127.0.0.1:8000/admin/auth/user/1/password/
+The client id and in case of the tank also the client secret are then needed for
+the configuration of those components.

program/management/commands/create_oidc_client.py

+from django.core.management.base import BaseCommand, CommandError
+import json, sys, random, string
+from oidc_provider.models import Client, ResponseType
+
+
+class Command(BaseCommand):
+    help = 'Sets up an OIDC client / relaying party. For details check out the' + \
+        'section on Relying Parties at https://django-oidc-provider.readthedocs.io'
+
+    def add_arguments(self, parser):
+        parser.add_argument('name', type=str,
+            help='A label that you associate with this client')
+        parser.add_argument('client_type', type=str, choices=['public', 'confidential'],
+            help='The type of client can be either public or confidential')
+        parser.add_argument('--no-require-consent', dest='require_consent', action='store_false',
+            help='By default user consent is required. Use this to skip user consent.')
+        parser.add_argument('--no-reuse-consent', dest='reuse_consent', action='store_false',
+            help='By default user consent will be reused. Use this if the user should provide consent on every login.')
+        parser.set_defaults(require_consent=True, reuse_consent=True)
+        parser.add_argument('-u', '--redirect-uri', type=str, action='append',
+            help='Redirect URI after successful authentication. Can be used more than once.')
+        parser.add_argument('-p', '--post-logout-redirect', type=str, action='append',
+            help='Post logout redirect URI. Can be used more than once.')
+        parser.add_argument('-s', '--scope', type=str, action='append',
+            help='Authorized scope values for this client. Can be used more than once.')
+        parser.add_argument('-r', dest='response_types', action='append',
+            choices=['code', 'id_token', 'id_token token', 'code token', 'code id_token', 'code id_token token'],
+            help='The type of response the client will get.')
+        parser.add_argument('-i', '--id-only', dest='id_only', action='store_true',
+            help='Do not print anything else then the ID of the newly created client '+\
+                 '(and the client secret in case of confidential clients).')
+        parser.set_defaults(id_only=False)
+
+
+    def handle(self, *args, **options):
+        # generate a new client ID and secret
+        client_id = False
+        counter = 0
+        while not client_id:
+            client_id = random.randint(100000, 999999)
+            counter += 1
+            if counter > 10000:
+                raise CommandError('Could not find a free client_id. Already'+\
+                    ' tried 10000 times. There seems to be something seriously'+\
+                    ' wrong with your setup. Please inspect manually.')
+            try:
+                Client.objects.get(client_id=client_id)
+            except Client.DoesNotExist:
+                pass
+            else:
+                client_id = False
+
+        client_secret = ''
+        if options['client_type'] == 'confidential':
+            client_secret = ''.join(random.SystemRandom().choice(string.ascii_letters + string.digits) for _ in range(32))
+
+        # initialize lists if no option was provided
+        if options['redirect_uri'] is None:
+            options['redirect_uri'] = []
+        if options['post_logout_redirect'] is None:
+            options['post_logout_redirect'] = []
+        if options['scope'] is None:
+            options['scope'] = []
+
+        if not options["id_only"]:
+            self.stdout.write(f'Creating client with name {options["name"]}')
+        try:
+            c = Client(
+                client_id=client_id,
+                client_secret=client_secret,
+                name=options['name'], client_type=options['client_type'],
+                redirect_uris=options['redirect_uri'],
+                require_consent=options['require_consent'],
+                reuse_consent=options['reuse_consent'],
+                post_logout_redirect_uris=options['post_logout_redirect'],
+                scope=options['scope'],
+            )
+            c.save()
+        except:
+            raise CommandError('Could not create an OpenID connect client' +\
+                f' due to the following error: {sys.exc_info()}')
+
+
+        if options['response_types']:
+            try:
+                for r_value in options['response_types']:
+                    r = ResponseType.objects.get(value=r_value)
+                    c.response_types.add(r)
+            except:
+                raise CommandError('Client was stored, but could not set response_types'+\
+                    f' due to the following error: {sys.exc_info()}')
+
+        if options["id_only"]:
+            if options['client_type'] == 'confidential':
+                self.stdout.write(f'{c.client_id} {c.client_secret}')
+            else:
+                self.stdout.write(f'{c.client_id}')
+        else:
+            self.stdout.write(f'Successfully created new OIDC client, with ID: {c.client_id}')
+            if options['client_type'] == 'confidential':
+                self.stdout.write(f'The secret for this confidential client is: {c.client_secret}')

pv/local_settings.py.sample

-
-import os
-
-from corsheaders.defaults import default_headers
-
-CONFIG_DIR = '/etc/aura/'
-DB_CONFIG = 'steering.mysql.cnf'
-
-SECRET_KEY = '---some-secred-key---'
-
-DATABASES = {
-    'default': {
-        'ENGINE': 'django.db.backends.mysql',
-        'OPTIONS': {
-            'read_default_file': os.path.join(CONFIG_DIR, DB_CONFIG),
-        },
-    }
-}
-
-CORS_ALLOW_CREDENTIALS = True
-CORS_ORIGIN_WHITELIST = (
-    'http://localhost:8080'
-    # 'https://aura-test.o94.at',
-    # 'https://aura-test.o94.at:443',
-)
-CORS_ALLOW_HEADERS = list(default_headers) + [
-    'content-disposition',
-]
-
-# Comment out the following for temporary debugging, if you want to use the
-# native DRF web forms
-"""
-REST_FRAMEWORK = {
-    # Use Django's standard `django.contrib.auth` permissions,
-    # or allow read-only access for unauthenticated users.
-    'DEFAULT_PERMISSION_CLASSES': [
-        'rest_framework.permissions.DjangoModelPermissionsOrAnonReadOnly'
-    ],
-}
-"""

pv/settings.py

 # Django settings for pv project.
 
 import os.path
+import environ
+from corsheaders.defaults import default_headers
 
 # Paths
 
@@ -18,35 +20,45 @@ STATIC_URL = '/static/'
 
 ROOT_URLCONF = 'pv.urls'
 
-DEBUG = True
+env = environ.Env()
+env.read_env(env_file=PROJECT_DIR+'/../.env')
+
+DOCKER = env.bool('RUN_IN_DOCKER', default=True)
+DEBUG = env.bool('DEBUG', default=False)
 SITE_ID = 1
 ADMINS = ()
 MANAGERS = ADMINS
 
 # Must be set if DEBUG is False
-ALLOWED_HOSTS = ['127.0.0.1', 'localhost']
+ALLOWED_HOSTS = env.list('HOSTNAMES', default=['127.0.0.1', 'localhost'])
 
 # Whitelist IPs that access the API
-CORS_ORIGIN_WHITELIST = (
-    'http://localhost',
-    'http://localhost:8080'
-)
+CORS_ORIGIN_WHITELIST = env.list('CORS_ORIGIN_WHITELIST', default=(
+    'http://localhost:8080',
+    'http://127.0.0.1:8080'
+))
+CORS_ALLOW_CREDENTIALS = True
+CORS_ALLOW_HEADERS = list(default_headers) + [
+    'content-disposition',
+]
 
 # Define which database backend to use for our apps
 DATABASES = {
     # SQLITE
-    'default': {
-        'ENGINE': 'django.db.backends.sqlite3',
-        'NAME': os.path.join(PROJECT_DIR, 'dev_data.sqlite'),
-    },
+    #'default': {
+    #    'ENGINE': 'django.db.backends.sqlite3',
+    #    'NAME': os.path.join(PROJECT_DIR, 'dev_data.sqlite'),
+    #},
 
     # PostgreSQL
-    # 'default': {
-    #        'ENGINE': 'django.db.backends.postgresql',
-    #        'OPTIONS': {
-    #            'read_default_file': os.path.join(PROJECT_DIR, 'postgresql.cnf'),
-    #        },
-    #    },
+    'default': {
+        'ENGINE': 'django.db.backends.postgresql',
+        'NAME': env.str('DBNAME', default='steering'),
+        'USER': env.str('DBUSER', default='steering'),
+        'PASSWORD': env.str('DBPASS'),
+        'HOST': env.str('DBHOST', default='steering-postgres'),
+        'PORT': env.str('DBPORT', '5432'),
+    },
 
     # MySQL
     #    'default': {
@@ -55,18 +67,19 @@ DATABASES = {
     #            'read_default_file': os.path.join(PROJECT_DIR, 'mysql.cnf'),
     #        },
     #   },
-
 }
+if not DOCKER:
+    DATABASES['default']['HOST'] = env.str('DBHOST', default='localhost')
 
 CACHE_BACKEND = 'locmem://'
 
 # LOCALIZATION
-TIME_ZONE = 'Europe/Vienna'
-LANGUAGE_CODE = 'de'
+TIME_ZONE = env.str('TIME_ZONE', default='Europe/Vienna')
+LANGUAGE_CODE = env.str('LANGUAGE_CODE', default='de')
 USE_I18N = True
 USE_L10N = True
 
-SECRET_KEY = ''
+SECRET_KEY = env.str('SECRET_KEY')
 
 TEMPLATES = [
     {

requirements.txt

@@ -3,8 +3,15 @@ django-cors-headers==3.2.1
 django-oidc-provider==0.7.0
 django-tinymce==2.8.0
 django-versatileimagefield==1.11
+django-environ==0.4.5
 djangorestframework==3.11.0
 drf-nested-routers==0.91
 Pillow==4.3.0
 python-dateutil==2.8.1
 PyYAML==3.13
+
+# needed for the database (container)
+psycopg2_binary==2.8.6
+
+# needed for production server
+gunicorn==20.0.4