[Fix] /v2/nodes (#38)

* cleanup

* select valid nodes + valid prev and current locations + average 5 mins

* remove unneeded seed

* cached data command

* celery task to cache data every 5 minutes

* refactor

* format cleanup

* use database model instead of json file

* add admin visibility

* cleanup

* cleanup

Author: karimkawambwa

sensorsafrica/admin.py

@@ -1,6 +1,6 @@
 from django.contrib import admin
 
-from .api.models import SensorDataStat, City
+from .api.models import LastActiveNodes, SensorDataStat, City
 
 from feinstaub.sensors.admin import (
     SensorLocationAdmin,
@@ -14,44 +14,43 @@
 import django.utils.timezone
 
 
-def last_data_received_at(self, obj):
-    then = (
-        SensorData.objects.filter(location=obj)
-        .values_list("timestamp", flat=True)
-        .last()
-    )
-    now = datetime.datetime.now(django.utils.timezone.utc)
+@admin.register(LastActiveNodes)
+class LastActiveNodesAdmin(admin.ModelAdmin):
+    readonly_fields = ["node", "location", "last_data_received_at"]
+    list_display = ["node", "location", "received"]
+    search_fields = ["node", "location", "last_data_received_at"]
+    list_filter = ["node", "location", "last_data_received_at"]
 
-    if not then:
-        return "Unknown"
+    def received(self, obj):
+        now = datetime.datetime.now(django.utils.timezone.utc)
 
-    return "( %s ) %s" % (
-        timeago.format(then, now),
-        SensorData.objects.filter(location=obj)
-        .values_list("timestamp", flat=True)
-        .last(),
-    )
+        if not obj.last_data_received_at:
+            return "Unknown"
 
+        return "( %s ) %s" % (
+            timeago.format(obj.last_data_received_at, now),
+            obj.last_data_received_at,
+        )
 
-def latitude_and_longitude(self, obj):
-    return "%s,%s" % (obj.latitude, obj.longitude)
+    def get_actions(self, request):
+        actions = super(LastActiveNodesAdmin, self).get_actions(request)
+        del actions["delete_selected"]
+        return actions
 
+    def has_add_permission(self, request):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
 
-def node_UID(self, obj):
-    return Node.objects.filter(location=obj).values_list("uid", flat=True).first()
+    def save_model(self, request, obj, form, change):
+        pass
 
+    def delete_model(self, request, obj):
+        pass
 
-SensorLocation._meta.verbose_name_plural = "Sensor Node Locations"
-SensorLocationAdmin.list_display = [
-    "node_UID",
-    "location",
-    "city",
-    "latitude_and_longitude",
-    "last_data_received_at",
-]
-SensorLocationAdmin.last_data_received_at = last_data_received_at
-SensorLocationAdmin.latitude_and_longitude = latitude_and_longitude
-SensorLocationAdmin.node_UID = node_UID
+    def save_related(self, request, form, formsets, change):
+        pass
 
 
 @admin.register(SensorDataStat)

sensorsafrica/api/models.py

@@ -48,3 +48,12 @@ def __str__(self):
             self.minimum,
             self.maximum,
         )
+
+
+class LastActiveNodes(TimeStampedModel):
+    node = models.ForeignKey(Node)
+    location = models.ForeignKey(SensorLocation)
+    last_data_received_at = models.DateTimeField()
+
+    class Meta:
+        unique_together = ['node', 'location']

sensorsafrica/api/v2/views.py

@@ -1,14 +1,16 @@
 import datetime
 import pytz
+import json
 
 from rest_framework.exceptions import ValidationError
 
+from django.conf import settings
 from django.utils import timezone
 from django.db.models import ExpressionWrapper, F, FloatField, Max, Min, Sum, Avg, Q
 from django.db.models.functions import Cast, TruncDate
 from rest_framework import mixins, pagination, viewsets
 
-from ..models import SensorDataStat, City, Node
+from ..models import SensorDataStat, LastActiveNodes, City, Node
 from .serializers import SensorDataStatSerializer, CitySerializer
 
 from feinstaub.sensors.views import StandardResultsSetPagination
@@ -205,36 +207,24 @@ class CityView(mixins.ListModelMixin, viewsets.GenericViewSet):
 
 
 class NodesView(viewsets.ViewSet):
-
-    # Cache requested url for each user for 1 hour
-    @method_decorator(cache_page(3600))
     def list(self, request):
         nodes = []
-        for location in SensorLocation.objects.iterator():
-            Node.objects.filter(location=location)
-            result = (
-                SensorData.objects.filter(location=location)
-                .values(
-                    "sensor__node__location",
-                    "sensor__node__location__location",
-                    "sensor__node__location__city",
-                    "sensor__node__location__longitude",
-                    "sensor__node__location__latitude",
-                    "timestamp",
-                )
-                .last()
-            )
+        for last_active_node in LastActiveNodes.objects.iterator():
+            node = Node.objects.filter(
+                Q(id=last_active_node.node.id), ~Q(sensors=None)
+            ).get()
+            last_data_received_at = last_active_node.last_data_received_at
 
+            # last_data_received_at
             stats = []
             prev_location = None
-            if result:
-                last_data_datetime = result["timestamp"]
-                last_24_hours = last_data_datetime - datetime.timedelta(hours=24)
+            if last_data_received_at:
+                last_5_mins = last_data_received_at - datetime.timedelta(minutes=5)
                 stats = (
                     SensorDataValue.objects.filter(
-                        Q(sensordata__location=location),
-                        Q(sensordata__timestamp__gte=last_24_hours),
-                        Q(sensordata__timestamp__lte=last_data_datetime),
+                        Q(sensordata__location=last_active_node.location.id),
+                        Q(sensordata__timestamp__gte=last_5_mins),
+                        Q(sensordata__timestamp__lte=last_data_received_at),
                         # Ignore timestamp values
                         ~Q(value_type="timestamp"),
                         # Match only valid float text
@@ -252,28 +242,31 @@ def list(self, request):
                     )
                 )
 
-                if result["sensor__node__location"] != location.id:
-                    prev_location = {
-                        "name": result["sensor__node__location__location"],
-                        "longitude": result["sensor__node__location__longitude"],
-                        "latitude": result["sensor__node__location__latitude"],
-                        "city": {
-                            "name": result["sensor__node__location__city"],
-                            "slug": slugify(result["sensor__node__location__city"]),
-                        },
-                    }
+            if last_active_node.location.id != node.location.id:
+                prev_location = {
+                    "name": node.location.location,
+                    "longitude": node.location.longitude,
+                    "latitude": node.location.latitude,
+                    "city": {
+                        "name": node.location.city,
+                        "slug": slugify(node.location.city),
+                    },
+                }
 
             nodes.append(
                 {
                     "node_moved": prev_location is not None,
                     "prev_location": prev_location,
                     "location": {
-                        "longitude": location.longitude,
-                        "latitude": location.latitude,
-                        "name": location.location,
-                        "city": {"name": location.city, "slug": slugify(location.city)},
+                        "name": last_active_node.location.location,
+                        "longitude": last_active_node.location.longitude,
+                        "latitude": last_active_node.location.latitude,
+                        "city": {
+                            "name": last_active_node.location.city,
+                            "slug": slugify(last_active_node.location.city),
+                        },
                     },
-                    "last_data_received_at": last_data_datetime,
+                    "last_data_received_at": last_data_received_at,
                     "stats": stats,
                 }
             )

sensorsafrica/management/commands/cache_lastactive_nodes_data.py

@@ -0,0 +1,25 @@
+from django.core.management import BaseCommand
+from django.core.cache import cache
+
+from django.conf import settings
+
+from django.db.models import Max
+
+from sensorsafrica.api.models import Node, SensorLocation, LastActiveNodes
+from feinstaub.sensors.models import SensorData
+
+
+class Command(BaseCommand):
+    help = ""
+
+    def handle(self, *args, **options):
+        for data in (
+            SensorData.objects.values("sensor__node", "location")
+            .order_by("sensor__node__id", "location__id")
+            .annotate(timestamp=Max("timestamp"))
+        ):
+            LastActiveNodes.objects.update_or_create(
+                node=Node(pk=data["sensor__node"]),
+                location=SensorLocation(pk=data["location"]),
+                last_data_received_at=data["timestamp"],
+            )

sensorsafrica/management/commands/seed.py

@@ -0,0 +1,33 @@
+# -*- coding: utf-8 -*-
+# Generated by Django 1.11.18 on 2019-05-09 11:45
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import django.db.models.deletion
+import django_extensions.db.fields
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('sensors', '0020_auto_20190314_1232'),
+        ('sensorsafrica', '0003_auto_20190222_1137'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='LastActiveNodes',
+            fields=[
+                ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('created', django_extensions.db.fields.CreationDateTimeField(auto_now_add=True, verbose_name='created')),
+                ('modified', django_extensions.db.fields.ModificationDateTimeField(auto_now=True, verbose_name='modified')),
+                ('last_data_received_at', models.DateTimeField()),
+                ('location', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sensors.SensorLocation')),
+                ('node', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='sensors.Node')),
+            ],
+        ),
+        migrations.AlterUniqueTogether(
+            name='lastactivenodes',
+            unique_together=set([('node', 'location')]),
+        ),
+    ]

sensorsafrica/migrations/0004_auto_20190509_1145.py

@@ -153,7 +153,11 @@
     # "archive-task": {
     #     "task": "sensorsafrica.tasks.archive_data",
     #     "schedule": crontab(hour="*", minute=0)
-    # }
+    # },
+    "cache-lastactive-nodes-task": {
+        "task": "sensorsafrica.tasks.cache_lastactive_nodes_data",
+        "schedule": crontab(minute="*/5")
+    },
 }
 
 

sensorsafrica/settings.py

@@ -10,3 +10,8 @@ def calculate_data_statistics():
 @shared_task
 def archive_data():
     call_command("upload_to_ckan")
+
+
+@shared_task
+def cache_lastactive_nodes_data():
+    call_command("cache_lastactive_nodes_data")