Replicate lufdetan feinstaub v1 routes (#49)

* /v1/sensors/<sensor_id> all raw measurements of the last 5 minutes for one sensor

* Update pytest

* /v1/filter?city=&country=&type= 5 minutes filtered by query + Tests

* Fix formatting

* /static/v2/*.json routes

* /static/v2/data.dust|temp.min.json routes

* Fix tests + cleanup settings

Author: karimkawambwa

.gitignore

@@ -400,3 +400,4 @@ celerybeat.pid
 logs/
 
 /static
+/sensorsafrica/static/**/*.json

requirements.txt

@@ -17,7 +17,7 @@ greenlet==0.4.12
 whitenoise==4.1.2
 
 pylama==7.6.6
-pytest==4.1.1
+pytest==5.2.1
 pytest-django==3.4.5
 
 ckanapi==4.1

sensorsafrica/api/v1/router.py

@@ -9,6 +9,8 @@
     SensorDataView,
 )
 
+from .views import SensorDataView as SensorsAfricaSensorDataView, FilterView
+
 from rest_framework import routers
 
 router = routers.DefaultRouter()
@@ -19,5 +21,8 @@
 router.register(r"statistics", StatisticsView, basename="statistics")
 router.register(r"now", NowView)
 router.register(r"user", UsersView)
+router.register(r"sensors/(?P<sensor_id>\d+)",
+                SensorsAfricaSensorDataView, basename="sensors")
+router.register(r"filter", FilterView, basename="filter")
 
 api_urls = router.urls

sensorsafrica/api/v1/serializers.py

@@ -0,0 +1,19 @@
+from rest_framework import serializers
+from feinstaub.sensors.models import (
+    SensorData,
+    SensorDataValue
+)
+
+
+class SensorDataValueSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = SensorDataValue
+        fields = ['value_type', 'value']
+
+
+class SensorDataSerializer(serializers.ModelSerializer):
+    sensordatavalues = SensorDataValueSerializer(many=True)
+
+    class Meta:
+        model = SensorData
+        fields = ['timestamp', 'sensordatavalues']

sensorsafrica/api/v1/views.py

@@ -0,0 +1,50 @@
+import datetime
+import pytz
+import json
+
+from rest_framework.exceptions import ValidationError
+
+from django.conf import settings
+from django.utils import timezone
+from dateutil.relativedelta import relativedelta
+from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q
+from django.db.models.functions import Cast, TruncDate
+from rest_framework import mixins, pagination, viewsets
+
+from .serializers import SensorDataSerializer
+from feinstaub.sensors.models import SensorData
+
+
+class SensorDataView(mixins.ListModelMixin, viewsets.GenericViewSet):
+    serializer_class = SensorDataSerializer
+
+    def get_queryset(self):
+        return (
+            SensorData.objects
+            .filter(
+                timestamp__gte=timezone.now() - datetime.timedelta(minutes=5),
+                sensor=self.kwargs["sensor_id"]
+            )
+            .only('sensor', 'timestamp')
+            .prefetch_related('sensordatavalues')
+        )
+
+
+class FilterView(mixins.ListModelMixin, viewsets.GenericViewSet):
+    serializer_class = SensorDataSerializer
+
+    def get_queryset(self):
+        sensor_type = self.request.GET.get('type', r'\w+')
+        country = self.request.GET.get('country', r'\w+')
+        city = self.request.GET.get('city', r'\w+')
+        return (
+            SensorData.objects
+            .filter(
+                timestamp__gte=timezone.now() - datetime.timedelta(minutes=5),
+                sensor__sensor_type__uid__iregex=sensor_type,
+                location__country__iregex=country,
+                location__city__iregex=city
+            )
+            .only('sensor', 'timestamp')
+            .prefetch_related('sensordatavalues')
+        )

sensorsafrica/management/commands/cache_static_json_data.py

@@ -0,0 +1,101 @@
+from django.core.management import BaseCommand
+from django.core.cache import cache
+
+from django.conf import settings
+
+from django.forms.models import model_to_dict
+
+from feinstaub.sensors.models import SensorLocation, Sensor, SensorType
+
+import os
+import json
+import datetime
+from django.utils import timezone
+
+from django.db import connection
+
+from rest_framework import serializers
+
+
+class SensorTypeSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = SensorType
+        fields = "__all__"
+
+
+class SensorSerializer(serializers.ModelSerializer):
+    sensor_type = SensorTypeSerializer()
+
+    class Meta:
+        model = Sensor
+        fields = "__all__"
+
+
+class SensorLocationSerializer(serializers.ModelSerializer):
+    class Meta:
+        model = SensorLocation
+        fields = "__all__"
+
+
+class Command(BaseCommand):
+    help = ""
+
+    def add_arguments(self, parser):
+        parser.add_argument('--interval', type=str)
+
+    def handle(self, *args, **options):
+        intervals = {'5m': '5 minutes', '1h': '1 hour', '24h': '24 hours'}
+        paths = {
+            '5m': [
+                '../../../static/v2/data.json',
+                '../../../static/v2/data.dust.min.json',
+                '../../../static/v2/data.temp.min.json'
+            ],
+            '1h': ['../../../static/v2/data.1h.json'],
+            '24h': ['../../../static/v2/data.24h.json']
+        }
+        cursor = connection.cursor()
+        cursor.execute('''
+            SELECT sd.sensor_id, sdv.value_type, AVG(CAST(sdv."value" AS FLOAT)) as "value", COUNT("value"), sd.location_id
+                FROM sensors_sensordata sd
+                    INNER JOIN sensors_sensordatavalue sdv
+                        ON  sd.id = sdv.sensordata_id WHERE "timestamp" >= (NOW() - interval %s)
+                GROUP BY sd.sensor_id, sdv.value_type, sd.location_id
+        ''', [intervals[options['interval']]])
+
+        data = {}
+        while True:
+            row = cursor.fetchone()
+            if row is None:
+                break
+
+            if row[0] in data:
+                data[row[0]]['sensordatavalues'].append(dict({
+                    'samples': row[3],
+                    'value': row[2],
+                    'value_type': row[1]
+                }))
+            else:
+                data[row[0]] = dict({
+                    'location': SensorLocationSerializer(SensorLocation.objects.get(pk=row[4])).data,
+                    'sensor': SensorSerializer(Sensor.objects.get(pk=row[0])).data,
+                    'sensordatavalues': [{
+                        'samples': row[3],
+                        'value': row[2],
+                        'value_type': row[1]
+                    }]
+                })
+
+        for path in paths[options['interval']]:
+            with open(
+                os.path.join(os.path.dirname(
+                    os.path.abspath(__file__)), path), 'w'
+            ) as f:
+                if 'dust' in path:
+                    json.dump(list(filter(
+                        lambda d: d['sensor']['sensor_type']['uid'] == 'sds011', data.values())), f)
+                elif 'temp' in path:
+                    json.dump(list(filter(
+                        lambda d: d['sensor']['sensor_type']['uid'] == 'dht22', data.values())), f)
+                else:
+                    json.dump(list(data.values()), f)

sensorsafrica/settings.py

@@ -142,7 +142,8 @@
 
 
 # Celery Broker
-CELERY_BROKER_URL = os.environ.get("SENSORSAFRICA_RABBITMQ_URL", "amqp://sensorsafrica:sensorsafrica@localhost//")
+CELERY_BROKER_URL = os.environ.get(
+    "SENSORSAFRICA_RABBITMQ_URL", "amqp://sensorsafrica:sensorsafrica@localhost//")
 CELERY_IGNORE_RESULT = True
 
 CELERY_BEAT_SCHEDULE = {
@@ -158,6 +159,14 @@
         "task": "sensorsafrica.tasks.cache_lastactive_nodes_data",
         "schedule": crontab(minute="*/5")
     },
+    "cache-static-json-data": {
+        "task": "sensorsafrica.tasks.cache_static_json_data",
+        "schedule": crontab(minute="*/5")
+    },
+    "cache-static-json-data-1h-24h": {
+        "task": "sensorsafrica.tasks.cache_static_json_data_1h_24h",
+        "schedule": crontab(hour="*", minute=0)
+    },
 }
 
 

sensorsafrica/static/v2/.gitkeep

@@ -15,3 +15,14 @@ def archive_data():
 @shared_task
 def cache_lastactive_nodes_data():
     call_command("cache_lastactive_nodes_data")
+
+
+@shared_task
+def cache_static_json_data():
+    call_command("cache_static_json_data", interval='5m')
+
+
+@shared_task
+def cache_static_json_data_1h_24h():
+    call_command("cache_static_json_data", interval='1h')
+    call_command("cache_static_json_data", interval='24h')

sensorsafrica/tasks.py

@@ -37,7 +37,8 @@ def location():
 
 @pytest.fixture
 def sensor_type():
-    st, x = SensorType.objects.get_or_create(uid="a", name="b", manufacturer="c")
+    st, x = SensorType.objects.get_or_create(
+        uid="a", name="b", manufacturer="c")
     return st
 
 
@@ -58,38 +59,53 @@ def sensor(logged_in_user, sensor_type, node):
 @pytest.fixture
 def locations():
     return [
-        SensorLocation.objects.get_or_create(city="Dar es Salaam", description="active")[0],
-        SensorLocation.objects.get_or_create(city="Bagamoyo", description="inactive")[0],
-        SensorLocation.objects.get_or_create(city="Mombasa", description="inactive")[0],
-        SensorLocation.objects.get_or_create(city="Nairobi", description="inactive")[0],
-        SensorLocation.objects.get_or_create(city="Dar es Salaam", description="active | some other node location")[0],
+        SensorLocation.objects.get_or_create(
+            city="Dar es Salaam", country="Tanzania", description="active")[0],
+        SensorLocation.objects.get_or_create(
+            city="Bagamoyo", country="Tanzania", description="inactive")[0],
+        SensorLocation.objects.get_or_create(
+            city="Mombasa", country="Kenya", description="inactive")[0],
+        SensorLocation.objects.get_or_create(
+            city="Nairobi", country="Kenya", description="inactive")[0],
+        SensorLocation.objects.get_or_create(
+            city="Dar es Salaam", country="Tanzania", description="active | some other node location")[0],
     ]
 
 
 @pytest.fixture
 def nodes(logged_in_user, locations):
     return [
-        Node.objects.get_or_create(uid="0", owner=logged_in_user, location=locations[0])[0],
-        Node.objects.get_or_create(uid="1", owner=logged_in_user, location=locations[1])[0],
-        Node.objects.get_or_create(uid="2", owner=logged_in_user, location=locations[2])[0],
-        Node.objects.get_or_create(uid="3", owner=logged_in_user, location=locations[3])[0],
-        Node.objects.get_or_create(uid="4", owner=logged_in_user, location=locations[4])[0],
+        Node.objects.get_or_create(
+            uid="0", owner=logged_in_user, location=locations[0])[0],
+        Node.objects.get_or_create(
+            uid="1", owner=logged_in_user, location=locations[1])[0],
+        Node.objects.get_or_create(
+            uid="2", owner=logged_in_user, location=locations[2])[0],
+        Node.objects.get_or_create(
+            uid="3", owner=logged_in_user, location=locations[3])[0],
+        Node.objects.get_or_create(
+            uid="4", owner=logged_in_user, location=locations[4])[0],
     ]
 
 
 @pytest.fixture
 def sensors(sensor_type, nodes):
     return [
         # Active Dar Sensor
-        Sensor.objects.get_or_create(node=nodes[0], sensor_type=sensor_type, public=True)[0],
+        Sensor.objects.get_or_create(
+            node=nodes[0], sensor_type=sensor_type, public=True)[0],
         # Inactive with last data push beyond active threshold
-        Sensor.objects.get_or_create(node=nodes[1], sensor_type=sensor_type)[0],
+        Sensor.objects.get_or_create(
+            node=nodes[1], sensor_type=sensor_type)[0],
         # Inactive without any data
-        Sensor.objects.get_or_create(node=nodes[2], sensor_type=sensor_type)[0],
+        Sensor.objects.get_or_create(
+            node=nodes[2], sensor_type=sensor_type)[0],
         # Active Nairobi Sensor
-        Sensor.objects.get_or_create(node=nodes[3], sensor_type=sensor_type)[0],
+        Sensor.objects.get_or_create(
+            node=nodes[3], sensor_type=sensor_type)[0],
         # Active Dar Sensor another location
-        Sensor.objects.get_or_create(node=nodes[4], sensor_type=sensor_type)[0],
+        Sensor.objects.get_or_create(
+            node=nodes[4], sensor_type=sensor_type)[0],
     ]
 
 
@@ -111,11 +127,13 @@ def sensordata(sensors, locations):
     ]
     # Nairobi SensorData
     for i in range(100):
-        sensor_datas.append(SensorData(sensor=sensors[3], location=locations[3]))
+        sensor_datas.append(SensorData(
+            sensor=sensors[3], location=locations[3]))
 
     # Dar es Salaam another node location SensorData
     for i in range(6):
-        sensor_datas.append(SensorData(sensor=sensors[4], location=locations[4]))
+        sensor_datas.append(SensorData(
+            sensor=sensors[4], location=locations[4]))
 
     data = SensorData.objects.bulk_create(sensor_datas)
 
@@ -130,7 +148,8 @@ def sensordata(sensors, locations):
 def datavalues(sensors, sensordata):
     data_values = [
         # Bagamoyo
-        SensorDataValue(sensordata=sensordata[0], value="2", value_type="humidity"),
+        SensorDataValue(
+            sensordata=sensordata[0], value="2", value_type="humidity"),
         # Dar es salaam a day ago's data
         SensorDataValue(sensordata=sensordata[1], value="1", value_type="P2"),
         SensorDataValue(sensordata=sensordata[2], value="2", value_type="P2"),
@@ -150,12 +169,14 @@ def datavalues(sensors, sensordata):
     # Nairobi SensorDataValues
     for i in range(100):
         data_values.append(
-            SensorDataValue(sensordata=sensordata[9 + i], value="0", value_type="P2")
+            SensorDataValue(
+                sensordata=sensordata[9 + i], value="0", value_type="P2")
         )
 
     # Dar es Salaam another node location SensorDataValues
     for i in range(6):
-        data_values.append(SensorDataValue(sensordata=sensordata[109 + i], value="0.0", value_type="P2"))
+        data_values.append(SensorDataValue(
+            sensordata=sensordata[109 + i], value="0.0", value_type="P2"))
 
     values = SensorDataValue.objects.bulk_create(data_values)