Hotfix make the lastactive sensor query faster (#59)

* Hotfix make the lastactive sensor query faster

* Add last active tests

Author: karimkawambwa

sensorsafrica/management/commands/cache_lastactive_nodes.py

@@ -0,0 +1,35 @@
+from django.core.management import BaseCommand
+
+from django.db import connection
+from sensorsafrica.api.models import Node, SensorLocation, LastActiveNodes
+
+
+class Command(BaseCommand):
+    help = ""
+
+    def handle(self, *args, **options):
+        with connection.cursor() as cursor:
+            cursor.execute(
+                """
+                    SELECT
+                        sn.id,
+                        sn.location_id,
+                        MAX("timestamp") AS last_active_date
+                    FROM
+                        sensors_sensordata sd
+                        INNER JOIN sensors_sensor s ON s.id = sd.sensor_id
+                            INNER JOIN sensors_node sn ON sn.id = s.node_id
+                            INNER JOIN sensors_sensordatavalue sv ON sv.sensordata_id = sd.id
+                            AND sv.value_type in ('P1', 'P2')
+                        WHERE
+                            "timestamp" >= now() - INTERVAL '5 min'
+                        GROUP BY
+                            sn.id
+                """)
+            latest = cursor.fetchall()
+            for data in latest:
+                LastActiveNodes.objects.update_or_create(
+                    node=Node(pk=data[0]),
+                    location=SensorLocation(pk=data[1]),
+                    defaults={"last_data_received_at": data[2]},
+                )

sensorsafrica/management/commands/cache_lastactive_nodes_data.py

@@ -161,7 +161,7 @@
         "schedule": crontab(hour="*", minute=0)
     },
     "cache-lastactive-nodes-task": {
-        "task": "sensorsafrica.tasks.cache_lastactive_nodes_data",
+        "task": "sensorsafrica.tasks.cache_lastactive_nodes",
         "schedule": crontab(minute="*/5")
     },
     "cache-static-json-data": {

sensorsafrica/settings.py

@@ -13,8 +13,8 @@ def archive_data():
 
 
 @shared_task
-def cache_lastactive_nodes_data():
-    call_command("cache_lastactive_nodes_data")
+def cache_lastactive_nodes():
+    call_command("cache_lastactive_nodes")
 
 
 @shared_task

sensorsafrica/tasks.py

@@ -227,3 +227,36 @@ def additional_sensorsdatastats(sensors, locations, sensorsdatastats):
     from django.core.management import call_command
 
     call_command("calculate_data_statistics")
+
+
+@pytest.fixture
+def last_active(sensors, locations, sensorsdatastats):
+    timestamps = [
+        timezone.now(),
+        timezone.now() + datetime.timedelta(minutes=2),
+        timezone.now() + datetime.timedelta(minutes=4)
+    ]
+    sensordata = SensorData.objects.bulk_create([
+        SensorData(
+            sensor=sensors[0], location=locations[0], timestamp=timestamps[0]),
+        SensorData(
+            sensor=sensors[3], location=locations[3], timestamp=timestamps[1]),
+        SensorData(
+            sensor=sensors[4], location=locations[4], timestamp=timestamps[2]),
+    ])
+
+    SensorDataValue.objects.bulk_create([
+        SensorDataValue(
+            sensordata=sensordata[0], value="4", value_type="P2"),
+        SensorDataValue(
+            sensordata=sensordata[1], value="4", value_type="P1"),
+        # Won't be tracked as last active
+        SensorDataValue(
+            sensordata=sensordata[2], value="4", value_type="Temp"),
+    ])
+
+    from django.core.management import call_command
+
+    call_command("cache_lastactive_nodes")
+
+    return timestamps

sensorsafrica/tests/conftest.py

@@ -0,0 +1,26 @@
+import pytest
+import pytz
+
+from rest_framework.test import APIRequestFactory
+from feinstaub.sensors.views import PostSensorDataView
+from sensorsafrica.api.models import LastActiveNodes
+
+
+@pytest.mark.django_db
+class TestLastActiveSensor:
+    def test_lastactive_command(self, sensors, last_active):
+        sensors0_node = LastActiveNodes.objects.filter(
+            node=sensors[0].node_id
+        ).get()
+        sensors3_node = LastActiveNodes.objects.filter(
+            node=sensors[3].node_id
+        ).get()
+        sensors4_node = LastActiveNodes.objects.filter(
+            node=sensors[4].node_id
+        ).get()
+
+        assert sensors0_node.last_data_received_at == last_active[0]
+        assert sensors3_node.last_data_received_at == last_active[1]
+        # Sensor is last active when they send the P1 and P2
+        # Sensor[4] sent a none P1/P2 at timestamp last_active[2]
+        assert sensors4_node.last_data_received_at != last_active[2]