Backend/metrics (#74)

* added board type & completed metadata response

* added base work for tests

* setup tests

* moved server files not related to classification to backend package

* added time passed in stage metrics

* added time passed in each sleep stages tests

* tests pass for time passed in stage

* added latency tests

* latency tests passes

* added tests to backend CI

* added onset tests

* completed onsets

* added sleep offset

* added wake after sleep offset

* added efficient tests

* added efficiency

* fixed warning

* added tests

* moved all metrics to single module

* refactored and added stage shifts

* fixed awakenings tests

* return -1 timestamps in case user doesnt sleep

* refactored metrics for the 2nd time ......

* fixed server errors

* removed acquisition board (now we autodetect) && added tests instructions to readme

* extracted stream duration

* fixed tests

* code review

* code review

Co-authored-by: Anes Belfodil <abelfodil@users.noreply.github.com>

Author: conorato

.github/workflows/backend.yml

@@ -26,6 +26,7 @@ jobs:
         with:
           python-version: 3.8
       - run: python -m pip install -r requirements.txt -r requirements-dev.txt
+      - run: python -m pytest
       - run: python -m PyInstaller --onefile app.py
       - uses: actions/upload-artifact@v2
         with:

backend/app.py

@@ -3,14 +3,14 @@
 from waitress import serve
 from http import HTTPStatus
 
+from backend.request import ClassificationRequest
+from backend.response import ClassificationResponse
+from backend.spectrogram_generator import SpectrogramGenerator
 from classification.parser import get_raw_array
 from classification.exceptions import ClassificationError
 from classification.config.constants import Sex, ALLOWED_FILE_EXTENSIONS
 from classification.model import SleepStagesClassifier
-from classification.request import ClassificationRequest
-from classification.response import ClassificationResponse
 from classification.features.preprocessing import preprocess
-from classification.spectrogram_generator import SpectrogramGenerator
 
 app = Flask(__name__)
 sleep_stage_classifier = SleepStagesClassifier()

backend/assets/readme.md

@@ -18,21 +18,25 @@
   "report": {
     "sleepOnset": 1602211380, // Time at which the subject fell asleep (time of the first non-wake epoch)
     "sleepOffset": 1602242425, // Time at which the subject woke up (time of the epoch after the last non-wake epoch)
-    "wakeAfterSleepOffset": 500, // [seconds] (wakeUpTime - sleepOffset)
-    "totalSleepTime": 31045, // Total amount of time sleeping including nocturnal awakenings (sleepOffset - sleepOnset)
-    "efficientSleepTime": 27113, // Total amount of seconds passed in non-wake stages
-    "sleepEffeciency": 0.8733, // Overall sense of how well the patient slept (totalSleepTime/bedTime)
-    "totalWASO": 3932, // Total amount of time passed in nocturnal awakenings. It is the total time passed in non-wake stage from sleep Onset to sleep offset (totalSleepTime - efficientSleepTime)
-    "sleepLatency": 1000, // Time to fall asleep [seconds] (sleepOnset - bedTime)
     "remOnset": 1602214232, // First REM epoch
+
+    "sleepLatency": 1000, // Time to fall asleep [seconds] (sleepOnset - bedTime)
     "remLatency": 3852, // [seconds] (remOnset- bedTime)
+
+    "sleepEfficiency": 0.8733, // Overall sense of how well the patient slept (totalSleepTime/bedTime)
     "awakenings": 7, // number of times the subject woke up between sleep onset & offset
     "stageShifts": 89, // number of times the subject transitionned from one stage to another between sleep onset & offset
-    "totalWTime": 3932, // [seconds] time passed in this stage between bedTime to wakeUpTime
-    "totalREMTime": 2370,
-    "totalN1Time": 3402,
-    "totalN2Time": 16032,
-    "totalN3Time": 5309
+
+
+    "wakeAfterSleepOffset": 500, // [seconds] (wakeUpTime - sleepOffset)
+    "efficientSleepTime": 27113, // Total amount of seconds passed in non-wake stages
+    "WASO": 3932, // Total amount of time passed in nocturnal awakenings. It is the total time passed in non-wake stage from sleep Onset to sleep offset (totalSleepTime - efficientSleepTime)
+    "WTime": 3932, // [seconds] time passed in this stage between bedTime to wakeUpTime
+    "SleepTime": 31045, // Total amount of time sleeping including nocturnal awakenings (sleepOffset - sleepOnset)
+    "REMTime": 2370,
+    "N1Time": 3402,
+    "N2Time": 16032,
+    "N3Time": 5309
   },
   "epochs": {
     "timestamps": [

backend/backend/metric.py

@@ -0,0 +1,153 @@
+from collections import Counter
+import numpy as np
+
+from classification.config.constants import SleepStage, EPOCH_DURATION
+
+
+class Metrics():
+    def __init__(self, sleep_stages, bedtime):
+        self.sleep_stages = sleep_stages
+        self.bedtime = bedtime
+        self.has_slept = len(np.unique(self.sleep_stages)) != 1 or np.unique(self.sleep_stages)[0] != SleepStage.W.name
+
+        self.is_sleeping_stages = self.sleep_stages != SleepStage.W.name
+        self.sleep_indexes = np.where(self.is_sleeping_stages)[0]
+        self.is_last_stage_sleep = self.sleep_stages[-1] != SleepStage.W.name
+
+        self._initialize_sleep_offset()
+        self._initialize_sleep_latency()
+        self._initialize_rem_latency()
+        self._initialize_transition_based_metrics()
+
+    @property
+    def report(self):
+        report = {
+            'sleepOffset': self._sleep_offset,
+            'sleepLatency': self._sleep_latency,
+            'remLatency': self._rem_latency,
+            'awakenings': self._awakenings,
+            'stageShifts': self._stage_shifts,
+            'sleepTime': self._sleep_time,
+            'WASO': self._wake_after_sleep_onset,
+            'sleepEfficiency': self._sleep_efficiency,
+            'efficientSleepTime': self._efficient_sleep_time,
+            'wakeAfterSleepOffset': self._wake_after_sleep_offset,
+            'sleepOnset': self._sleep_onset,
+            'remOnset': self._rem_onset,
+            **self._time_passed_in_stage,
+        }
+
+        for metric in report:
+            #  json does not recognize NumPy data types
+            if isinstance(report[metric], np.int64):
+                report[metric] = int(report[metric])
+
+        return report
+
+    @property
+    def _sleep_time(self):
+        if not self.has_slept:
+            return 0
+
+        return self._sleep_offset - self._sleep_onset
+
+    @property
+    def _wake_after_sleep_onset(self):
+        if not self.has_slept:
+            return 0
+
+        return self._sleep_time - self._efficient_sleep_time
+
+    @property
+    def _time_passed_in_stage(self):
+        """Calculates time passed in each stage for all of the sequence"""
+        nb_epoch_passed_by_stage = Counter(self.sleep_stages)
+
+        def get_time_passed(stage):
+            return EPOCH_DURATION * nb_epoch_passed_by_stage[stage] if stage in nb_epoch_passed_by_stage else 0
+
+        return {
+            f"{stage.upper()}Time": get_time_passed(stage)
+            for stage in SleepStage.tolist()
+        }
+
+    @property
+    def _sleep_efficiency(self):
+        return len(self.sleep_indexes) / len(self.sleep_stages)
+
+    @property
+    def _efficient_sleep_time(self):
+        return len(self.sleep_indexes) * EPOCH_DURATION
+
+    @property
+    def _wake_after_sleep_offset(self):
+        if not self.has_slept:
+            return 0
+
+        wake_after_sleep_offset_nb_epochs = (
+            len(self.sleep_stages) - self.sleep_indexes[-1] - 1
+        ) if not self.is_last_stage_sleep else 0
+
+        return wake_after_sleep_offset_nb_epochs * EPOCH_DURATION
+
+    @property
+    def _sleep_onset(self):
+        if not self.has_slept:
+            return None
+
+        return self._sleep_latency + self.bedtime
+
+    @property
+    def _rem_onset(self):
+        rem_latency = self._rem_latency
+        if rem_latency is None:
+            return None
+
+        return rem_latency + self.bedtime
+
+    def _initialize_sleep_offset(self):
+        if not self.has_slept:
+            sleep_offset = None
+        else:
+            sleep_nb_epochs = (self.sleep_indexes[-1] + 1) if len(self.sleep_indexes) else len(self.sleep_stages)
+            sleep_offset = sleep_nb_epochs * EPOCH_DURATION + self.bedtime
+
+        self._sleep_offset = sleep_offset
+
+    def _initialize_sleep_latency(self):
+        self._sleep_latency = self._get_latency_of_stage(self.is_sleeping_stages)
+
+    def _initialize_rem_latency(self):
+        """Time it took to enter REM stage"""
+        self._rem_latency = self._get_latency_of_stage(self.sleep_stages == SleepStage.REM.name)
+
+    def _initialize_transition_based_metrics(self):
+        consecutive_stages_occurences = Counter(zip(self.sleep_stages[:-1], self.sleep_stages[1:]))
+        occurences_by_transition = {
+            consecutive_stages: consecutive_stages_occurences[consecutive_stages]
+            for consecutive_stages in consecutive_stages_occurences if consecutive_stages[0] != consecutive_stages[1]
+        }
+        transition_occurences = list(occurences_by_transition.values())
+        awakenings_occurences = [
+            occurences_by_transition[transition_stages]
+            for transition_stages in occurences_by_transition
+            if transition_stages[0] != SleepStage.W.name
+            and transition_stages[1] == SleepStage.W.name
+        ]
+        nb_stage_shifts = sum(transition_occurences)
+        nb_awakenings = sum(awakenings_occurences)
+
+        if self.is_last_stage_sleep and self.has_slept:
+            nb_stage_shifts += 1
+            nb_awakenings += 1
+
+        self._stage_shifts = nb_stage_shifts
+        self._awakenings = nb_awakenings
+
+    def _get_latency_of_stage(self, sequence_is_stage):
+        epochs_of_stage_of_interest = np.where(sequence_is_stage)[0]
+
+        if len(epochs_of_stage_of_interest) == 0:
+            return None
+
+        return epochs_of_stage_of_interest[0] * EPOCH_DURATION

backend/classification/request.py renamed to backend/backend/request.py

@@ -12,18 +12,21 @@
 
 
 class ClassificationRequest():
-    def __init__(self, sex, age, stream_start, bedtime, wakeup, raw_eeg):
+    def __init__(self, sex, age, stream_start, bedtime, wakeup, raw_eeg, stream_duration=None):
         self.sex = sex
         self.age = age
         self.stream_start = stream_start
         self.bedtime = bedtime
         self.wakeup = wakeup
-
-        self.stream_duration = raw_eeg.times[-1]
         self.raw_eeg = raw_eeg
+        self.stream_duration = stream_duration if stream_duration else self._get_stream_duration()
 
         self._validate()
 
+    def _get_stream_duration(self):
+        PERIOD_DURATION = 1 / self.raw_eeg.info['sfreq']
+        return self.raw_eeg.times[-1] + PERIOD_DURATION
+
     @property
     def in_bed_seconds(self):
         """timespan, in seconds, from which the subject started the recording and went to bed"""

backend/classification/response.py renamed to backend/backend/response.py

@@ -1,5 +1,6 @@
 import numpy as np
 
+from backend.metric import Metrics
 from classification.config.constants import EPOCH_DURATION, SleepStage
 
 
@@ -15,42 +16,46 @@ def __init__(self, request, predictions, spectrogram):
 
         self.spectrogram = spectrogram
         self.predictions = predictions
+        self.metrics = Metrics(self.sleep_stages, self.bedtime)
 
     @property
     def sleep_stages(self):
-        ordered_sleep_stage_names = np.array([SleepStage(stage_index).name for stage_index in range(len(SleepStage))])
+        ordered_sleep_stage_names = np.array(SleepStage.tolist())
         return ordered_sleep_stage_names[self.predictions]
 
     @property
-    def epochs(self):
+    def response(self):
+        return {
+            'epochs': self._epochs,
+            'report': self._report,
+            'metadata': self._metadata,
+            'subject': self._subject,
+            'spectrograms': self.spectrogram,
+        }
+
+    @property
+    def _epochs(self):
         timestamps = np.arange(self.n_epochs * EPOCH_DURATION, step=EPOCH_DURATION) + self.bedtime
         return {'timestamps': timestamps.tolist(), 'stages': self.sleep_stages.tolist()}
 
     @property
-    def metadata(self):
+    def _metadata(self):
         return {
             "sessionStartTime": self.stream_start,
             "sessionEndTime": self.stream_duration + self.stream_start,
             "totalSessionTime": self.stream_duration,
             "bedTime": self.bedtime,
-            "wakeUpTime": None,
-            "totalBedTime": None,
+            "wakeUpTime": self.wakeup,
+            "totalBedTime": self.wakeup - self.bedtime,
         }
 
     @property
-    def subject(self):
+    def _subject(self):
         return {
             'age': self.age,
             'sex': self.sex.name,
         }
 
     @property
-    def response(self):
-        return {
-            'epochs': self.epochs,
-            'report': None,
-            'metadata': self.metadata,
-            'subject': self.subject,
-            'board': None,
-            'spectrograms': self.spectrogram,
-        }
+    def _report(self):
+        return self.metrics.report

backend/classification/spectrogram_generator.py renamed to backend/backend/spectrogram_generator.py

@@ -15,6 +15,10 @@ class SleepStage(Enum):
     N3 = 3
     REM = 4
 
+    @staticmethod
+    def tolist():
+        return [e.name for e in SleepStage]
+
 
 class HiddenMarkovModelProbability(Enum):
     emission = auto()

backend/classification/config/constants.py

@@ -41,6 +41,14 @@ If you want to run the backend with hot reload enabled (you must have installed
 hupper -m waitress app:app
 ```
 
+## Run the tests
+
+You can run our unit tests with the following command, after installing the development requirements:
+
+```bash
+pytest
+```
+
 ## Profile application
 
 - Run `python profiler.py`

backend/readme.md

@@ -1,4 +1,5 @@
 hupper==1.10.2
 pyinstaller==4.0
+pytest==6.1.2
 snakeviz==2.1.0
 Werkzeug==1.0.1

backend/requirements-dev.txt

@@ -0,0 +1,28 @@
+from unittest.mock import patch
+
+from backend.request import ClassificationRequest
+from classification.config.constants import Sex
+
+
+def pytest_generate_tests(metafunc):
+    # called once per each test function
+    funcarglist = metafunc.cls.params[metafunc.function.__name__]
+    argnames = sorted(funcarglist[0])
+    metafunc.parametrize(
+        argnames, [[funcargs[name] for name in argnames] for funcargs in funcarglist]
+    )
+
+
+def get_mock_request():
+    with patch.object(ClassificationRequest, '_validate', lambda *x, **y: None):
+        mock_request = ClassificationRequest(
+            sex=Sex.M,
+            age=22,
+            stream_start=1582418280,
+            bedtime=1582423980,
+            wakeup=1582452240,
+            raw_eeg=None,
+            stream_duration=35760,
+        )
+
+    return mock_request