Merge pull request #455 from smart-on-fhir/mikix/upload-date

upload-notes: add a date metadata field, for downstream scripting

Author: mikix

cumulus_etl/upload_notes/cli.py

@@ -300,6 +300,7 @@ def group_notes_by_unique_id(notes: Collection[LabelStudioNote]) -> list[LabelSt
                 group_notes[0].encounter_id,
                 group_notes[0].anon_encounter_id,
                 text=grouped_text,
+                date=group_notes[0].date,
                 doc_mappings=grouped_doc_mappings,
                 doc_spans=grouped_doc_spans,
                 ctakes_matches=grouped_ctakes_matches,

cumulus_etl/upload_notes/labelstudio.py

@@ -160,6 +160,7 @@ def _format_task_for_note(self, note: LabelStudioNote) -> dict:
                 "anon_patient_id": note.anon_patient_id,
                 "encounter_id": note.encounter_id,
                 "anon_encounter_id": note.anon_encounter_id,
+                "date": note.date and note.date.isoformat(),
                 "docref_mappings": note.doc_mappings,
                 # json doesn't natively have tuples, so convert spans to lists
                 "docref_spans": {k: list(v) for k, v in note.doc_spans.items()},

tests/upload_notes/test_upload_cli.py

@@ -502,6 +502,8 @@ async def test_grouped_datetime(self):
         self.assertEqual(1, len(notes))
         note = notes[0]
 
+        self.assertEqual(note.date.isoformat(), "2018-01-01T00:00:00")
+
         # The order will be oldest->newest (None placed last)
         self.assertEqual(
             self.wrap_note("Document", "DocRef 3", date="01/01/18")

tests/upload_notes/test_upload_labelstudio.py

@@ -1,5 +1,6 @@
 """Tests for cumulus.upload_notes.labelstudio.py"""
 
+import datetime
 from unittest import mock
 
 import ddt
@@ -26,7 +27,11 @@ def setUp(self):
 
     @staticmethod
     def make_note(
-        *, unique_id: str = "unique", ctakes: bool = True, philter_label: bool = True
+        *,
+        unique_id: str = "unique",
+        ctakes: bool = True,
+        philter_label: bool = True,
+        **kwargs,
     ) -> LabelStudioNote:
         text = "Normal note text"
         note = LabelStudioNote(
@@ -38,6 +43,7 @@ def make_note(
             doc_mappings={"doc": "doc-anon"},
             doc_spans={"doc": (0, len(text))},
             text=text,
+            **kwargs,
         )
         if ctakes:
             note.ctakes_matches = ctakesmock.fake_ctakes_extract(note.text).list_match(
@@ -71,7 +77,7 @@ def get_pushed_task(self) -> dict:
         return imported_tasks[0]
 
     async def test_basic_push(self):
-        await self.push_tasks(self.make_note())
+        await self.push_tasks(self.make_note(date=datetime.datetime(2010, 10, 10)))
         self.assertEqual(
             {
                 "data": {
@@ -81,6 +87,7 @@ async def test_basic_push(self):
                     "anon_patient_id": "patient-anon",
                     "encounter_id": "enc",
                     "anon_encounter_id": "enc-anon",
+                    "date": "2010-10-10T00:00:00",
                     "docref_mappings": {"doc": "doc-anon"},
                     "docref_spans": {"doc": [0, 16]},
                     "mylabel": [{"value": "Itch"}, {"value": "Nausea"}],
@@ -174,6 +181,7 @@ async def test_no_predictions(self):
                     "anon_patient_id": "patient-anon",
                     "encounter_id": "enc",
                     "anon_encounter_id": "enc-anon",
+                    "date": None,
                     "docref_mappings": {"doc": "doc-anon"},
                     "docref_spans": {"doc": [0, 16]},
                     "mylabel": [],
@@ -198,6 +206,7 @@ async def test_dynamic_labels(self, label_type):
                 "anon_patient_id": "patient-anon",
                 "encounter_id": "enc",
                 "anon_encounter_id": "enc-anon",
+                "date": None,
                 "docref_mappings": {"doc": "doc-anon"},
                 "docref_spans": {"doc": [0, 16]},
                 "mylabel": [
@@ -218,6 +227,7 @@ async def test_dynamic_labels_no_predictions(self):
                 "anon_patient_id": "patient-anon",
                 "encounter_id": "enc",
                 "anon_encounter_id": "enc-anon",
+                "date": None,
                 "docref_mappings": {"doc": "doc-anon"},
                 "docref_spans": {"doc": [0, 16]},
                 "mylabel": [],  # this needs to be sent, or the server will complain
@@ -276,6 +286,7 @@ async def test_push_highlights(self):
                     "anon_patient_id": "patient-anon",
                     "encounter_id": "enc",
                     "anon_encounter_id": "enc-anon",
+                    "date": None,
                     "docref_mappings": {"doc": "doc-anon"},
                     "docref_spans": {"doc": [0, 16]},
                     "mylabel": [{"value": "Label1"}, {"value": "Label2"}],