Merge branch 'aura-sync-fixes' into 'master'

add fixes that emerged from using oidc-client-stubs in aura-sync

See merge request !2

README.md

@@ -58,6 +58,85 @@ Make sure to have the config right, that can be imported from _config.py_
 in the `parameters` dictionary, depending on what flow you choose (e.g.
 `parameters["code"] = "it_token token"` for an implicit flow).
 
+#### Usage in other Python projects
+
+The Python part of this project has been used in an API-based database migration
+tool for AURA at Radio Orange, Vienna. Therefore some fixes have been made and
+functionality has been added. Before installation the oidc client was tested by
+running the `main.py` test script which revealed a number of bugs. Aside from
+these for a first installation it seemed to be necessary to install a webserver
+(flask) to be able to create a callback address, needed by the client definition
+in steering. However, it turned out this server (and the callback page) could be
+deleted later without a problem, so this note remains here as a hint to a hack
+which once had been useful but cannot be confirmed as a dependency.
+
+For Python projects it is recommended to rather copy the client's project files
+over to the other project than trying to include them e.g. as submodule. Ideally
+create a directory somewhere where your own files can see the client's files
+and copy over the contents of the `python` directory. You will have to create a
+client in steering or on the commandline. Copy `config.sample.py` to `config.py`
+and edit according to your steering client.
+
+By importing `authorization` from `steering` you get a handful of useful methods
+that allow you to automate authorization respectively to make it implicit in
+API calls.
+
+An example authorization handled in a class `BaseRestConnector`:
+
+```python
+import json
+from oidc.config import config as oidc_config
+from oidc.steering import authorization
+from oidc.tank import session
+
+class BaseRESTConnector():
+    """
+    Do OIDC authorization before instancing any RESTConnectors.
+
+    Allows us to make OIDC authentication implicit in all RESTConnectors.
+    """
+    oidc_config["verbosity"] = 0
+    params = { "response_type": "code" }
+    oidc = authorization.get_auth_code(oidc_config, params)
+    oidc = authorization.get_token_from_code(oidc_config, oidc["code"])
+    # print("Bearer " + oidc["access_token"])
+    tank_session = session.get_session_token(oidc_config, oidc["access_token"]).text
+    tank_token = json.loads(tank_session)["token"]
+```
+
+Later in your requests, if you receive a `401`:
+
+```python
+BaseRESTConnector.oidc = authorization.refresh_access_token(oidc_config, BaseRESTConnector.oidc["refresh_token"])
+```
+
+The same thing may occur when communicating with tank as we established the
+connection via steering oidc client. However, `401` can also mean the (tank)
+session became invalid in which case you should be able to recreate it like
+this:
+
+```python
+tank_session_request = session.get_session_token(oidc_config, BaseRESTConnector.oidc["access_token"])
+if tank_session_request.status_code == 200:
+    BaseRESTConnector.tank_token = json.loads(tank_session_request.text)["token"]
+```
+
+`get_session_token()` implements a full session creation in tank, using `oidc`
+as backend (`tank/session.py`).
+
+##### Fixes:
+- The authorisation process has to operate on html forms which are filtered by
+regex patterns. Attribute values all seem to use double quotes for their values.
+Regex patterns have been adapted accordingly.
+- Python doesn't seem to like URL-encoded strings. URLs are now being prepared
+before they are being processed.
+- Use raise instead of sys.exit in error cases (not absolutely sure this has
+been done correctly)
+
+##### added:
+
+- `refresh_access_token()` (in `authorization`)
+
 ### Javascript
 
 For the JS client the approach is a bit different, as the most probable

python/steering/authorization.py

@@ -32,10 +32,41 @@ def get_token_from_code (cfg, code):
         if cfg["verbosity"] >= 1:
             print("Calling OIDC token endpoint to exchange access code for token")
         response = requests.post(url, headers=headers, data=payload, allow_redirects=False)
-    except Error as e:
-        e = sys.exc_info()
-        print(e[0].__name__, ':', e[1])
-        sys.exit(1)
+    # except Error as e:
+    #     e = sys.exc_info()
+    #     print(e[0].__name__, ':', e[1])
+    #     sys.exit(1)
+    except Exception as e:
+        print("Error while requesting token from code", e)
+        raise e
+    if cfg["verbosity"] >= 2:
+        print(response.json())
+    return response.json()
+
+
+def refresh_access_token (cfg, refresh_token):
+    """
+    Refresh the access_code if access_code has expired
+    """
+    url = cfg["base_url"] + cfg["token_endpoint"]
+    headers = {"User-Agent": cfg["user_agent"]}
+    payload = {
+        "client_id": cfg["client_id"],
+        "client_secret": cfg["client_secret"],
+        "grant_type": "refresh_token",
+        "refresh_token": refresh_token
+    }
+    try:
+        if cfg["verbosity"] >= 1:
+            print("Calling OIDC token endpoint to refresh access_token")
+        response = requests.post(url, headers=headers, data=payload, allow_redirects=False)
+        if response.status_code == 200:
+            print("*** New access token created successfully ***")
+        else:
+            print(f"Something went wrong while requesting new access token: {response.status_code} | {response.text}")
+    except Exception as e:
+        print("Error while refreshing access_token:", e)
+        raise e
     if cfg["verbosity"] >= 2:
         print(response.json())
     return response.json()

python/steering/flow_stages.py

+import html
 import requests
 import sys
 import re
 
+from urllib.parse import unquote
 
 def initiate_flow (cfg, parameters):
     """
@@ -45,10 +47,14 @@ def handle_login_form (cfg, parameters):
     url = cfg["base_url"] + parameters["location"]
     try:
         response = requests.get(url, headers=headers, allow_redirects=False)
-    except Error as e:
-        e = sys.exc_info()
-        print(e[0].__name__, ':', e[1])
-        sys.exit(1)
+    # except Error as e:
+    #     e = sys.exc_info()
+    #     print(e[0].__name__, ':', e[1])
+    #     sys.exit(1)
+    except Exception as e:
+        print("Error while requesting the login form", e)
+        raise e
+
 
     # save cookies (including csrf token and session id), and extract form data
     jar = response.cookies
@@ -58,7 +64,7 @@ def handle_login_form (cfg, parameters):
     # arguments of the csrfmiddlewaretoken input tag
     m = re.search("<input type=['\"]hidden['\"] name=['\"]csrfmiddlewaretoken['\"] value=['\"]([^'\"]*)['\"]", response.text)
     csrf_mw_token = m.groups()[0]
-    m = re.search('<input type="hidden" name="next" value="([^"]*)"', response.text)
+    m = re.search("<input type=['\"]hidden['\"] name=['\"]next['\"] value=['\"]([^'\"]*)['\"]", response.text)
     next_field = m.groups()[0]
 
     # submit login data
@@ -78,10 +84,14 @@ def handle_login_form (cfg, parameters):
         if cfg["verbosity"] >= 1:
             print("## Stage 3: submitting login data")
         response = requests.post(url, headers=headers, cookies=jar, data=payload, allow_redirects=False)
-    except Error as e:
-        e = sys.exc_info()
-        print(e[0].__name__, ':', e[1])
-        sys.exit(1)
+    # except Error as e:
+    #     e = sys.exc_info()
+    #     print(e[0].__name__, ':', e[1])
+    #     sys.exit(1)
+    except Exception as e:
+        print("Error while submitting login data", e)
+        raise e
+
     if cfg["verbosity"] >= 2:
         print("CSRF cookie:", jar.get("csrftoken"))
         print("session cookie:", jar.get("sessionid"))
@@ -95,15 +105,21 @@ def handle_login_form (cfg, parameters):
 
     # attempt to retrieve final callback redirect
     jar = response.cookies
+    next_field = unquote(next_field)
+    next_field = html.unescape(next_field)
     url = cfg["base_url"] + next_field
     try:
         if cfg["verbosity"] >= 1:
             print("## Stage 4: get token in final callback location")
         response = requests.get(url, headers=headers, cookies=jar, allow_redirects=False)
-    except Error as e:
-        e = sys.exc_info()
-        print(e[0].__name__, ':', e[1])
-        sys.exit(1)
+    # except Error as e:
+    #     e = sys.exc_info()
+    #     print(e[0].__name__, ':', e[1])
+    #     sys.exit(1)
+    except Exception as e:
+        print("Error while trying to retrieve final callback redirect", e)
+        raise e
+
 
     # if explicit consent is required, we will not be redirected but get a
     # consent form which we have to submit, before the final redirect can happen
@@ -113,7 +129,7 @@ def handle_login_form (cfg, parameters):
         # extract form data (cookies from last request have to be reused)
         m = re.search('<form method="post" action="([^"]*)"', response.text)
         submit_url = m.groups()[0]
-        m = re.search("<input type='hidden' name='csrfmiddlewaretoken' value='([^']*)'", response.text)
+        m = re.search('<input type="hidden" name="csrfmiddlewaretoken" value="([^"]*)"', response.text)
         csrf_mw_token = m.groups()[0]
         if cfg["verbosity"] >= 2:
             print("CSRF cookie:", jar.get("csrftoken"))
@@ -136,10 +152,14 @@ def handle_login_form (cfg, parameters):
             if cfg["verbosity"] >= 1:
                 print("submitting consent form")
             response = requests.post(url, headers=headers, cookies=jar, data=payload, allow_redirects=False)
-        except Error as e:
-            e = sys.exc_info()
-            print(e[0].__name__, ':', e[1])
-            sys.exit(1)
+        # except Error as e:
+        #     e = sys.exc_info()
+        #     print(e[0].__name__, ':', e[1])
+        #     sys.exit(1)
+        except Exception as e:
+            print("Error while submitting consent form", e)
+            raise e
+
 
     # return callback location
     return response.headers["Location"]
@@ -150,7 +170,7 @@ def get_token_from_callback (cfg, url):
     Extract any relevant information from a callback redirect URL
     """
     if cfg["verbosity"] >= 1:
-        print("Stage 5: processing the callback redirect URL")
+        print("## Stage 5: processing the callback redirect URL")
     if cfg["verbosity"] >= 2:
         print("callback URL:", url)
     oidc = {}