feat/sql_alchemy (#51)

Author: JarbasAl

.github/workflows/build_tests.yml

@@ -47,5 +47,4 @@ jobs:
           python setup.py bdist_wheel
       - name: Install package
         run: |
-          pip install .
-      - uses: pypa/gh-action-pip-audit@v1.0.0
+          pip install .

.github/workflows/unit_tests.yml

@@ -1,12 +1,12 @@
 FROM ubuntu:latest
 
 RUN apt-get update && \
-  apt-get install -y git python3 python3-dev python3-pip curl build-essential libffi-dev python3-numpy rustc
+  apt-get install -y git python3 python3-dev python3-pip curl build-essential libffi-dev python3-numpy rustc flac libmysqlclient-dev
 
 RUN pip3 install SpeechRecognition==3.8.1
 
 COPY . /tmp/ovos-backend
-RUN pip3 install /tmp/ovos-backend
+RUN pip3 install /tmp/ovos-backend[mysql]
 
 RUN pip3 install git+https://github.com/OpenVoiceOS/ovos-backend-manager
 

Dockerfile

@@ -10,6 +10,10 @@ This is NOT meant to provision third party devices, but rather to run on the myc
 
 Documentation can be found at https://openvoiceos.github.io/community-docs/personal_backend
 
+NOTES: 
+- this backend moved to SQL databases on release 0.1.6a10, json databases from older version are not compatible
+- at the time of writing, backend manager does not yet work with this backend version
+
 
 ## Install
 

README.md

@@ -13,13 +13,16 @@
 
 from flask import Flask
 from ovos_local_backend.configuration import CONFIGURATION
+from ovos_local_backend.database import connect_db
 
 API_VERSION = CONFIGURATION["api_version"]
 
 
 def create_app():
     app = Flask(__name__)
 
+    app, db = connect_db(app)
+
     from ovos_local_backend.utils import nice_json
     from ovos_local_backend.backend.decorators import noindex
     from ovos_local_backend.backend.auth import get_auth_routes
@@ -28,12 +31,15 @@ def create_app():
     from ovos_local_backend.backend.precise import get_precise_routes
     from ovos_local_backend.backend.external_apis import get_services_routes
     from ovos_local_backend.backend.admin import get_admin_routes
+    from ovos_local_backend.backend.crud import get_database_crud
+
     app = get_auth_routes(app)
     app = get_device_routes(app)
     app = get_stt_routes(app)
     app = get_precise_routes(app)
     app = get_services_routes(app)
     app = get_admin_routes(app)
+    app = get_database_crud(app)
 
     @app.route("/", methods=['GET'])
     @noindex

ovos_local_backend/backend/__init__.py

@@ -11,16 +11,18 @@
 # limitations under the License.
 #
 import time
+
+import base64
+from flask import request
 from ovos_local_backend.backend import API_VERSION
-from ovos_local_backend.utils import nice_json
-from ovos_local_backend.database.settings import DeviceDatabase
-from ovos_local_backend.configuration import CONFIGURATION
 from ovos_local_backend.backend.decorators import noindex, requires_admin
-from flask import request
+import ovos_local_backend.database as db
 from ovos_local_backend.utils import generate_code
+from ovos_local_backend.utils import nice_json
 from ovos_local_backend.utils.geolocate import get_request_location
 
 
+
 def get_admin_routes(app):
     @app.route("/" + API_VERSION + "/admin/<uuid>/pair", methods=['GET'])
     @requires_admin
@@ -30,8 +32,7 @@ def pair_device(uuid):
         token = f"{code}:{uuid}"
         # add device to db
         location = get_request_location()
-        with DeviceDatabase() as db:
-            db.add_device(uuid, token, location=location)
+        db.add_device(uuid, token, location=location)
 
         device = {"uuid": uuid,
                   "expires_at": time.time() + 99999999999999,
@@ -43,72 +44,21 @@ def pair_device(uuid):
     @requires_admin
     @noindex
     def set_device(uuid):
-        data = request.json
-        with DeviceDatabase() as db:
-            device = db.get_device(uuid)
-            if not device:
-                return {"error": "unknown device"}
-            if "name" in data:
-                device.name = data["name"]
-            if "opt_in" in data:
-                device.opt_in = data["opt_in"]
-            if "device_location" in data:
-                device.device_location = data["device_location"]
-            if "email" in data:
-                device.email = data["email"]
-            if "isolated_skills" in data:
-                device.isolated_skills = data["isolated_skills"]
-            if "lang" in data:
-                device.lang = data["lang"]
-            db.update_device(device)
-            return nice_json(device.serialize())
+        device_data = db.update_device(uuid, **request.json)
+        return nice_json(device_data)
 
     @app.route("/" + API_VERSION + "/admin/<uuid>/location", methods=['PUT'])
     @requires_admin
     @noindex
     def set_location(uuid):
-        with DeviceDatabase() as db:
-            device = db.get_device(uuid)
-            if not device:
-                return {"error": "unknown device"}
-            device.location = request.json
-            db.update_device(device)
-            return nice_json(device.serialize())
+        device_data = db.update_device(uuid, location=request.json)
+        return nice_json(device_data)
 
     @app.route("/" + API_VERSION + "/admin/<uuid>/prefs", methods=['PUT'])
     @requires_admin
     @noindex
     def set_prefs(uuid):
-        data = request.json
-        with DeviceDatabase() as db:
-            device = db.get_device(uuid)
-            if not device:
-                return {"error": "unknown device"}
-            if "time_format" in data:
-                device.time_format = data["time_format"]
-            if "date_format" in data:
-                device.date_format = data["date_format"]
-            if "system_unit" in data:
-                device.system_unit = data["system_unit"]
-            if "lang" in data:
-                device.lang = data["lang"]
-            if "tts_module" in data:
-                device.default_tts = data["tts_module"]
-                if "tts_config" in data:
-                    device.default_tts_cfg = data["tts_config"]
-                elif data["tts_module"] in CONFIGURATION["tts_configs"]:
-                    device.default_tts_cfg = CONFIGURATION["tts_configs"][data["tts_module"]]
-                else:
-                    device.default_tts_cfg = {}
-            if "wake_word" in data:
-                device.default_ww = data["wake_word"]
-                if "ww_config" in data:
-                    device.default_ww_cfg = data["ww_config"]
-                elif data["ww_module"] in CONFIGURATION["ww_configs"]:
-                    device.default_ww_cfg = CONFIGURATION["ww_configs"][data["ww_module"]]
-                else:
-                    device.default_ww_cfg = {}
-            db.update_device(device)
-            return nice_json(device.serialize())
+        device_data = db.update_device(uuid, **request.json)
+        return nice_json(device_data)
 
     return app

ovos_local_backend/backend/admin.py

@@ -20,7 +20,8 @@
 
 from ovos_local_backend.backend import API_VERSION
 from ovos_local_backend.backend.decorators import noindex, requires_auth
-from ovos_backend_client.database import OAuthTokenDatabase, OAuthApplicationDatabase
+from ovos_local_backend.database import add_oauth_application, add_oauth_token, get_oauth_application, get_oauth_token
+
 from ovos_local_backend.utils import nice_json
 
 os.environ['OAUTHLIB_INSECURE_TRANSPORT'] = '1'
@@ -50,36 +51,40 @@ def oauth_url(oauth_id):
         """ send auth url to user to confirm authorization,
         once user opens it callback is triggered
         """
+        auth = request.headers.get('Authorization', '').replace("Bearer ", "")
+        uid = auth.split(":")[-1]
+        token_id = f"{uid}|{oauth_id}"
+
         params = dict(request.args)
-        params["callback_endpoint"] = request.base_url + f"/{API_VERSION}/auth/callback/{oauth_id}"
+        params["callback_endpoint"] = request.base_url + f"/{API_VERSION}/auth/callback/{token_id}"
 
         client = WebApplicationClient(params["client_id"])
         request_uri = client.prepare_request_uri(
             params["auth_endpoint"],
             redirect_uri=params["callback_endpoint"],
             scope=params["scope"],
         )
-        with OAuthApplicationDatabase() as db:
-            db.add_application(oauth_id,
-                               params["client_id"],
-                               params["client_secret"],
-                               params["auth_endpoint"],
-                               params["token_endpoint"],
-                               params["refresh_endpoint"],
-                               params["callback_endpoint"],
-                               params["scope"])
+
+        add_oauth_application(token_id=token_id,
+                               client_id=params["client_id"],
+                               client_secret=params["client_secret"],
+                               auth_endpoint=params["auth_endpoint"],
+                               token_endpoint=params["token_endpoint"],
+                               refresh_endpoint=params["refresh_endpoint"],
+                               callback_endpoint=params["callback_endpoint"],
+                               scope=params["scope"])
 
         return request_uri, 200
 
-    @app.route(f"/{API_VERSION}/auth/callback/<oauth_id>", methods=['GET'])
+    @app.route(f"/{API_VERSION}/auth/callback/<token_id>", methods=['GET'])
     @noindex
-    def oauth_callback(oauth_id):
+    def oauth_callback(token_id):
         """ user completed oauth, save token to db
         """
         params = dict(request.args)
         code = params["code"]
 
-        data = OAuthApplicationDatabase()[oauth_id]
+        data = get_oauth_application(token_id)
         client_id = data["client_id"]
         client_secret = data["client_secret"]
         token_endpoint = data["token_endpoint"]
@@ -99,17 +104,16 @@ def oauth_callback(oauth_id):
             auth=(client_id, client_secret),
         ).json()
 
-        with OAuthTokenDatabase() as db:
-            db.add_token(oauth_id, token_response)
-
+        add_oauth_token(token_id, token_response)
         return nice_json(params)
 
     @app.route(f"/{API_VERSION}/device/<uuid>/token/<oauth_id>", methods=['GET'])
     @requires_auth
     @noindex
     def oauth_token(uuid, oauth_id):
         """a device is requesting a token for a previously approved OAuth app"""
-        data = OAuthTokenDatabase().get(oauth_id) or {}
+        token_id = f"@{uuid}|{oauth_id}"
+        data = get_oauth_token(token_id)
         return nice_json(data)
 
     return app