Use TimescaleDB as database for optimized time series queries

.dockerignore

@@ -0,0 +1,6 @@
+db-data/
+docker-compose.yml
+Dockerfile
+.env
+.git/
+.gitignore

.env

@@ -1,12 +1,11 @@
-ENGINE_TYPE=postgresql
-DB_HOST=db
+DB_HOST=timescaledb
 DB_PORT=5432
 DB_NAME=postgres
 DB_USER=postgres
 DB_PASSWORD=postgres
 
 SECRET_KEY=very-secret-key
-ALLOWED_HOSTS=localhost,example.com,interface
+ALLOWED_HOSTS=localhost,example.com,interface,192.168.0.19
 CORS_ORIGIN_ALLOW_ALL=False
 CORS_ORIGIN_WHITELIST=http://example.com,http://localhost,http://interface:8000
 CHART_TYPE=uplot

.github/workflows/ruff-format-check.yml

@@ -0,0 +1,31 @@
+name: Ruff Format Check
+
+on:
+  push:
+    paths:
+      - '**.py'
+  pull_request:
+    paths:
+      - '**.py'
+
+jobs:
+  ruff-format:
+    name: Check code formatting with Ruff
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12'
+
+      - name: Install Ruff
+        run: |
+          pip install ruff
+
+      - name: Run ruff format check
+        run: |
+          ruff format --check .

.github/workflows/test.yml

@@ -16,6 +16,21 @@ jobs:
       matrix:
         python-version: ['3.10', '3.11', '3.12']
 
+    services:
+      timescaledb:
+        image: timescale/timescaledb:latest-pg16
+        env:
+          POSTGRES_PASSWORD: postgres
+        # Set health checks to wait until postgres has started
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
+        ports:
+          # Maps tcp port 5432 on service container to the host
+          - 5432:5432
+
     steps:
     - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
@@ -30,3 +45,9 @@ jobs:
       run: |
         python manage.py collectstatic --noinput
         python manage.py test
+      env:
+        DB_NAME: postgres
+        DB_USER: postgres
+        DB_PASSWORD: postgres
+        DB_HOST: localhost
+        DB_PORT: 5432

Dockerfile

@@ -5,7 +5,7 @@ WORKDIR /app
 
 COPY requirements.txt .
 
-RUN apk add --no-cache --virtual .build-deps gcc musl-dev mariadb-connector-c-dev libpq-dev
+RUN apk add --no-cache --virtual .build-deps gcc musl-dev libpq-dev
 
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir gunicorn && \
@@ -20,10 +20,11 @@ COPY --from=builder /usr/local/lib/python3.12/site-packages /usr/local/lib/pytho
 COPY --from=builder /usr/local/bin/gunicorn /usr/local/bin/gunicorn
 COPY --from=builder /app /app
 
-RUN apk add --no-cache mariadb-connector-c libpq
+RUN apk add --no-cache libpq
 
 COPY . .
 
-CMD ["sh", "-c", "python manage.py makemigrations && python manage.py migrate && python manage.py collectstatic --noinput && gunicorn --bind=0.0.0.0:8000 --timeout 300 --workers=3 --threads=3 --max-requests 5 --max-requests-jitter 2 pim.wsgi:application"]
+
+CMD ["sh", "-c", "python manage.py migrate && python manage.py collectstatic --noinput && gunicorn --bind=0.0.0.0:8000 --timeout 300 --workers=3 --threads=3 --max-requests 20 --max-requests-jitter 5 pim.wsgi:application"]
 
 EXPOSE 8000/tcp

README.md

@@ -2,6 +2,9 @@
 
 The Django application is designed to allow users to view power usage charts for MyStrom and Shelly3EM devices. The application collects the current power usage data every 60 seconds, which is then used to calculate and present power usage charts to the user.
 
+This project uses [TimeScaleDB](https://www.timescale.com/) as database. 
+The database is optimized for time-series data and this project uses queries which are only compatible with TimeScaleDB.
+
 There are two main views: the Device view and the Result view. Here's what each view looks like:
 
 #### Device View
@@ -37,7 +40,6 @@ pip install -r requirements.txt
 
 Here's a breakdown of all the environment variables that are being used in the Django application:
 
-- `ENGINE_TYPE`: Specifies the type of database engine to use, either `mysql` or `postgresql`.
 - `DB_NAME`: Specifies the name of the database to use.
 - `DB_USER`: Specifies the username to use when connecting to the database.
 - `DB_PASSWORD`: Specifies the password to use when connecting to the database.
@@ -87,7 +89,6 @@ Static files are served with WhiteNoise.
 docker run \
    --name mystrom-interface \
    -p 8000:8000 \
-   -e ENGINE_TYPE={mysql/postgresql}
    -e DB_NAME=db-name \
    -e DB_USER=username \
    -e DB_PASSWORD=password \
@@ -107,7 +108,3 @@ Make sure to replace the content in `{...}` with your variable of your choice.
 ```sh
 docker compose up
 ```
-
-## Configurations
-### Important things to know
-Database configuraton in `pim/settings.py` under `DATABASES`

docker-compose.yml

@@ -1,19 +1,42 @@
+
 services:
   interface:
     container_name: interface
     image: ghcr.io/maexled/mystrom-django-interface:master
-#   build: .
+    # build: .
     privileged: true
     restart: always
     ports:
       - "80:8000"
     env_file:
       - .env
+    depends_on:
+      - timescaledb
+
+  timescaledb:
+    image: timescale/timescaledb:latest-pg16
+    environment:
+      POSTGRES_DB: postgres
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: postgres
+    volumes:
+      - ./db-data:/var/lib/postgresql/data
+
+  pgadmin:
+    image: dpage/pgadmin4
+    environment:
+      PGADMIN_DEFAULT_EMAIL: admin@admin.com
+      PGADMIN_DEFAULT_PASSWORD: admin
+    ports:
+      - "8080:80"
+    depends_on:
+      - timescaledb
 
   requester:
     container_name: interface-requester
     image: curlimages/curl:7.80.0
-    command: ["sh", "-c", "while true; do sleep 60; curl -X POST http://interface:8000/shelly-api/devices/request-and-save-results >/dev/null 2>&1 & curl -X POST http://interface:8000/api/devices/request-and-save-results >/dev/null 2>&1; done"]    
+    command: ["sh", "-c", "while true; do sleep 60; curl -fsS -X POST http://interface:8000/shelly-api/devices/request-and-save-results && curl -fsS -X POST http://interface:8000/api/devices/request-and-save-results; done"]
     depends_on:
       - interface
+
 

interface/admin.py

@@ -1,3 +1 @@
-from django.contrib import admin
-
 # Register your models here.

interface/apps.py

@@ -2,5 +2,5 @@
 
 
 class MystromConfig(AppConfig):
-    default_auto_field = 'django.db.models.BigAutoField'
-    name = 'interface'
+    default_auto_field = "django.db.models.BigAutoField"
+    name = "interface"

interface/forms.py

@@ -1,32 +1,24 @@
 from django import forms
 from mystrom_rest.models import MystromDevice
 from shelly3em_rest.models import Shelly3EMDevice
-
+
+
 # creating a form
 class MystromDeviceForm(forms.ModelForm):
-
     # create meta class
     class Meta:
         # specify model to be used
         model = MystromDevice
- 
+
         # specify fields to be used
-        fields = [
-            "name",
-            "ip",
-            "active"
-        ]
+        fields = ["name", "ip", "active"]
+
 
 class Shelly3EMDeviceForm(forms.ModelForm):
-
     # create meta class
     class Meta:
         # specify model to be used
         model = Shelly3EMDevice
- 
+
         # specify fields to be used
-        fields = [
-            "name",
-            "ip",
-            "active"
-        ]
+        fields = ["name", "ip", "active"]

interface/templatetags/classname.py

@@ -2,6 +2,7 @@
 
 register = template.Library()
 
+
 @register.filter
 def classname(obj):
-    return obj.__class__.__name__
+    return obj.__class__.__name__

interface/tests.py

@@ -1,82 +1,87 @@
 from django.urls import reverse
 from django.test import TestCase
-from mystrom_rest.models import MystromDevice, MystromResult
+from mystrom_rest.models import MystromDevice
 from bs4 import BeautifulSoup
 
+
 class MystromDevicesTestCase(TestCase):
     def setUp(self):
-        MystromDevice.objects.create(name="Device 1", ip="192.168.0.205")
-        MystromDevice.objects.create(name="Device 2", ip="127.0.0.1")
+        self.device1 = MystromDevice.objects.create(name="Device 1", ip="192.168.0.205")
+        self.device2 = MystromDevice.objects.create(name="Device 2", ip="127.0.0.1")
 
     def test_create_device(self):
-        url = reverse('mystrom_devices')
-        response = self.client.post(url, {'name': 'test', 'ip': '127.0.0.1'})
+        url = reverse("mystrom_devices")
+        response = self.client.post(url, {"name": "test", "ip": "127.0.0.1"})
         self.assertContains(response, "test")
         self.assertContains(response, "127.0.0.1")
         self.assertContains(response, "<td>test</td>")
         self.assertContains(response, "<td>127.0.0.1</td>")
         self.assertEqual(len(MystromDevice.objects.all()), 3)
-    
+
     def test_create_device_fail_invalid_ip(self):
-        url = reverse('mystrom_devices')
-        response = self.client.post(url, {'name': 'test', 'ip': 'Not an ip'})
-        self.assertContains(response, "<div class=\"invalid-feedback\">Not valid IP Address</div>")
+        url = reverse("mystrom_devices")
+        response = self.client.post(url, {"name": "test", "ip": "Not an ip"})
+        self.assertContains(
+            response, '<div class="invalid-feedback">Not valid IP Address</div>'
+        )
         self.assertEqual(len(MystromDevice.objects.all()), 2)
 
     def test_delete_devices(self):
-        url = reverse('mystrom_devices')
+        url = reverse("mystrom_devices")
         response = self.client.delete(url)
         self.assertNotContains(response, "<tr>")
-        self.assertEqual(len(MystromDevice.objects.all()), 0)    
-        
+        self.assertEqual(len(MystromDevice.objects.all()), 0)
+
     def test_get_create_device_form(self):
-        url = reverse('mystrom_devices')
+        url = reverse("mystrom_devices")
         response = self.client.get(url)
         self.assertContains(response, "Create")
         self.assertContains(response, "<form")
         self.assertContains(response, "</form>")
 
     def test_update_device(self):
-        url = reverse('mystrom_device', args=(1,))
-        device = MystromDevice.objects.get(id=1)
+        url = reverse("mystrom_device", args=(self.device1.id,))
+        device = MystromDevice.objects.get(id=self.device1.id)
 
-        self.assertEqual(device.name, "Device 1")
-        response = self.client.post(url, {'name': 'Not Device 1', 'ip': '192.168.0.205'})
+        self.assertEqual(device.name, self.device1.name)
+        response = self.client.post(
+            url, {"name": "Not Device 1", "ip": "192.168.0.205"}
+        )
         self.assertContains(response, "Not Device 1")
 
-        device = MystromDevice.objects.get(id=1)
+        device = MystromDevice.objects.get(id=self.device1.id)
         self.assertEqual(device.name, "Not Device 1")
 
     def test_update_device_fail_invalid_ip(self):
-        url = reverse('mystrom_device', args=(1,))
-        response = self.client.post(url, {'name': 'Not Device 1', 'ip': 'Not an ip'})
-        self.assertContains(response, "<div class=\"invalid-feedback\">Not valid IP Address</div>")
+        url = reverse("mystrom_device", args=(self.device1.id,))
+        response = self.client.post(url, {"name": "Not Device 1", "ip": "Not an ip"})
+        self.assertContains(
+            response, '<div class="invalid-feedback">Not valid IP Address</div>'
+        )
         self.assertEqual(len(MystromDevice.objects.all()), 2)
 
     def test_delete_device(self):
-        url = reverse('mystrom_device', args=(1,))
+        url = reverse("mystrom_device", args=(self.device1.id,))
         response = self.client.delete(url)
         self.assertContains(response, "<td>Device 2</td>")
         self.assertNotContains(response, "<td>Device 1</td>")
         self.assertEqual(len(MystromDevice.objects.all()), 1)
 
     def test_get_update_device_form(self):
-        url = reverse('mystrom_device', args=(1,))
+        url = reverse("mystrom_device", args=(self.device1.id,))
         response = self.client.get(url)
         self.assertContains(response, "Edit device")
         self.assertContains(response, "<form")
         self.assertContains(response, "</form>")
 
+
 class MystromResultsTestCase(TestCase):
     def setUp(self):
         self.device = MystromDevice.objects.create(name="Device 1", ip="192.168.0.205")
 
     def test_get_results_page(self):
-        url = reverse('results')
+        url = reverse("results")
         response = self.client.get(url)
-        soup = BeautifulSoup(response.content, 'html.parser')
-        button = soup.find(id='mystrom-' + str(self.device.id))
+        soup = BeautifulSoup(response.content, "html.parser")
+        button = soup.find(id="mystrom-" + str(self.device.id))
         self.assertIsNotNone(button)
-
-
-

interface/urls.py

@@ -2,12 +2,12 @@
 from . import views
 
 urlpatterns = [
-    path('', views.index, name='index'),
-    path('results', views.results, name='results'),
-    path('results/mystrom/<int:id>', views.mystrom_results, name='mystrom_results'),
-    path('results/shelly/<int:id>', views.shelly_results, name='shelly_results'),
-    path('devices/mystrom/<int:id>', views.mystrom_device_info, name='mystrom_device'),
-    path('devices/mystrom', views.mystrom_devices, name='mystrom_devices'),
-    path('devices/shelly/<int:id>', views.shelly_device_info, name='shelly_device'),
-    path('devices/shelly', views.shelly_devices, name='shelly_devices'),
-]
+    path("", views.index, name="index"),
+    path("results", views.results, name="results"),
+    path("results/mystrom/<int:id>", views.mystrom_results, name="mystrom_results"),
+    path("results/shelly/<int:id>", views.shelly_results, name="shelly_results"),
+    path("devices/mystrom/<int:id>", views.mystrom_device_info, name="mystrom_device"),
+    path("devices/mystrom", views.mystrom_devices, name="mystrom_devices"),
+    path("devices/shelly/<int:id>", views.shelly_device_info, name="shelly_device"),
+    path("devices/shelly", views.shelly_devices, name="shelly_devices"),
+]