Merge master into diabetic_retinopathy_for_merge

Author: dehall

src/main/java/App.java

@@ -280,14 +280,15 @@ public static void main(String[] args) throws Exception {
    * Reset the fields of the provided options to the current values in the Config.
    */
   private static void resetOptionsFromConfig(Generator.GeneratorOptions options,
-		  Exporter.ExporterRuntimeOptions exportOptions) {
-      // Any options that are automatically set by reading the configuration
-      // file during options initialization need to be reset here.
-      options.population = Config.getAsInteger("generate.default_population", 1);
-      options.threadPoolSize = Config.getAsInteger("generate.thread_pool_size", -1);
+      Exporter.ExporterRuntimeOptions exportOptions) {
+    // Any options that are automatically set by reading the configuration
+    // file during options initialization need to be reset here.
+    options.population = Config.getAsInteger("generate.default_population", 1);
+    options.threadPoolSize = Config.getAsInteger("generate.thread_pool_size", -1);
 
-      exportOptions.yearsOfHistory = Config.getAsInteger("exporter.years_of_history", 10);
-      exportOptions.terminologyService = !Config.get("generate.terminology_service_url", "").isEmpty();
+    exportOptions.yearsOfHistory = Config.getAsInteger("exporter.years_of_history", 10);
+    exportOptions.terminologyService = !Config.get("generate.terminology_service_url", "")
+            .isEmpty();
   }
 
   private static boolean validateConfig(Generator.GeneratorOptions options,

src/main/java/org/mitre/synthea/engine/Generator.java

@@ -218,6 +218,7 @@ private void init() {
       CDWExporter.getInstance().setKeyStart((stateIndex * 1_000_000) + 1);
     }
     Exporter.loadCustomExporters();
+    Exporter.loadCodeMappers();
 
     this.populationRandom = new DefaultRandomNumberGenerator(options.seed);
     this.clinicianRandom = new DefaultRandomNumberGenerator(options.clinicianSeed);

src/main/java/org/mitre/synthea/export/Exporter.java

@@ -14,14 +14,17 @@
 import java.nio.file.StandardOpenOption;
 import java.util.ArrayList;
 import java.util.Collections;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Map;
 import java.util.ServiceLoader;
 import java.util.concurrent.BlockingQueue;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.LinkedBlockingQueue;
 import java.util.function.Predicate;
+import java.util.stream.Collectors;
 
 import org.apache.commons.lang3.tuple.ImmutablePair;
 import org.apache.commons.lang3.tuple.Pair;
@@ -34,6 +37,7 @@
 import org.mitre.synthea.export.flexporter.FlexporterJavascriptContext;
 import org.mitre.synthea.export.flexporter.Mapping;
 import org.mitre.synthea.export.rif.BB2RIFExporter;
+import org.mitre.synthea.export.rif.CodeMapper;
 import org.mitre.synthea.helpers.Config;
 import org.mitre.synthea.helpers.TransitionMetrics;
 import org.mitre.synthea.helpers.Utilities;
@@ -69,6 +73,7 @@ public enum SupportedFhirVersion {
 
   private static List<PatientExporter> patientExporters;
   private static List<PostCompletionExporter> postCompletionExporters;
+  private static Map<String, CodeMapper> codeMappers;
 
   /**
    * If the config setting "exporter.enable_custom_exporters" is enabled,
@@ -95,6 +100,45 @@ public static void loadCustomExporters() {
     }
   }
 
+  /**
+   * Load any configured code mappers. Code mappers are configured via the
+   * synthea.properties file and a sample configuration is shown below:
+   * <pre>
+   * exporter.code_map.icd_10=export/anti_amyloid_code_map.json
+   * exporter.code_map.cpt=export/phlebotomy_code_map.json,export/neurology_code_map.json
+   * </pre>
+   * The above define a single code map for ICD-10 codes and two code maps for CPT codes.
+   */
+  public static void loadCodeMappers() {
+    codeMappers = new HashMap<String, CodeMapper>();
+    List<String> codeSystemProperties = Config.allPropertyNames()
+            .stream()
+            .filter((key) -> key.startsWith("exporter.code_map"))
+            .collect(Collectors.toList());
+    codeSystemProperties.forEach(codeSystemProperty -> {
+      String codeSystem = codeSystemProperty.strip().replace(
+              "exporter.code_map.", "").toUpperCase();
+      String[] resources = Config.get(codeSystemProperty).split(",");
+      for (String resource: resources) {
+        CodeMapper mapper = new CodeMapper(resource);
+        if (codeMappers.containsKey(codeSystem)) {
+          codeMappers.get(codeSystem).merge(mapper);
+        } else {
+          codeMappers.put(codeSystem, mapper);
+        }
+      }
+    });
+  }
+
+  /**
+   * Get the code mapper for the supplied code system.
+   * @param codeSystem the code system
+   * @return the corresponding code mapper or null if none configured
+   */
+  public static CodeMapper getCodeMapper(String codeSystem) {
+    return codeMappers.get(codeSystem);
+  }
+
   /**
    * Runtime configuration of the record exporter.
    */

src/main/java/org/mitre/synthea/export/FhirR4.java

@@ -10,6 +10,7 @@
 
 import java.awt.geom.Point2D;
 import java.io.IOException;
+import java.util.AbstractMap;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Calendar;
@@ -139,7 +140,10 @@
 
 import org.mitre.synthea.engine.Components;
 import org.mitre.synthea.engine.Components.Attachment;
+import org.mitre.synthea.export.rif.CodeMapper;
 import org.mitre.synthea.helpers.Config;
+import org.mitre.synthea.helpers.RandomNumberGenerator;
+import org.mitre.synthea.helpers.RandomValueGenerator;
 import org.mitre.synthea.helpers.SimpleCSV;
 import org.mitre.synthea.helpers.Utilities;
 import org.mitre.synthea.identity.Entity;
@@ -404,7 +408,7 @@ public static Bundle convertToFHIR(Person person, long stopTime) {
 
       if (shouldExport(Condition.class)) {
         for (HealthRecord.Entry condition : encounter.conditions) {
-          condition(personEntry, bundle, encounterEntry, condition);
+          condition(person, personEntry, bundle, encounterEntry, condition);
         }
       }
 
@@ -431,7 +435,7 @@ public static Bundle convertToFHIR(Person person, long stopTime) {
 
       if (shouldExport(org.hl7.fhir.r4.model.Procedure.class)) {
         for (Procedure procedure : encounter.procedures) {
-          procedure(personEntry, bundle, encounterEntry, procedure);
+          procedure(person, personEntry, bundle, encounterEntry, procedure);
         }
       }
 
@@ -849,6 +853,27 @@ private static BundleEntryComponent basicInfo(Person person, Bundle bundle, long
     return newEntry(bundle, patientResource, (String) person.attributes.get(Person.ID));
   }
 
+  /**
+   * Add a code translation (if available) of the supplied source code to the
+   * supplied CodeableConcept.
+   * @param codeSystem the code system of the translated code
+   * @param from the source code
+   * @param to the CodeableConcept to add the translation to
+   * @param rand a source of randomness
+   */
+  private static void addTranslation(String codeSystem, Code from,
+          CodeableConcept to, RandomNumberGenerator rand) {
+    CodeMapper mapper = Exporter.getCodeMapper(codeSystem);
+    if (mapper != null && mapper.canMap(from)) {
+      Coding coding = new Coding();
+      Map.Entry<String, String> mappedCode = mapper.mapToCodeAndDescription(from, rand);
+      coding.setCode(mappedCode.getKey());
+      coding.setDisplay(mappedCode.getValue());
+      coding.setSystem(ExportHelper.getSystemURI("ICD10-CM"));
+      to.addCoding(coding);
+    }
+  }
+
   /**
    * Map the given Encounter into a FHIR Encounter resource, and add it to the given Bundle.
    *
@@ -894,6 +919,8 @@ private static BundleEntryComponent encounter(Person person, BundleEntryComponen
     if (encounter.reason != null) {
       encounterResource.addReasonCode().addCoding().setCode(encounter.reason.code)
           .setDisplay(encounter.reason.display).setSystem(SNOMED_URI);
+      addTranslation("ICD10-CM", encounter.reason,
+              encounterResource.getReasonCodeFirstRep(), person);
     }
 
     Provider provider = encounter.provider;
@@ -1607,6 +1634,7 @@ private static BundleEntryComponent explanationOfBenefit(BundleEntryComponent pe
    * @return The added Entry
    */
   private static BundleEntryComponent condition(
+          RandomNumberGenerator rand,
           BundleEntryComponent personEntry, Bundle bundle, BundleEntryComponent encounterEntry,
           HealthRecord.Entry condition) {
     Condition conditionResource = new Condition();
@@ -1630,7 +1658,9 @@ private static BundleEntryComponent condition(
     conditionResource.setEncounter(new Reference(encounterEntry.getFullUrl()));
 
     Code code = condition.codes.get(0);
-    conditionResource.setCode(mapCodeToCodeableConcept(code, SNOMED_URI));
+    CodeableConcept concept = mapCodeToCodeableConcept(code, SNOMED_URI);
+    addTranslation("ICD10-CM", code, concept, rand);
+    conditionResource.setCode(concept);
 
     CodeableConcept verification = new CodeableConcept();
     verification.getCodingFirstRep()
@@ -1964,13 +1994,14 @@ static org.hl7.fhir.r4.model.SampledData mapValueToSampledData(
   /**
    * Map the given Procedure into a FHIR Procedure resource, and add it to the given Bundle.
    *
+   * @param person         The Person
    * @param personEntry    The Person entry
    * @param bundle         Bundle to add to
    * @param encounterEntry The current Encounter entry
    * @param procedure      The Procedure
    * @return The added Entry
    */
-  private static BundleEntryComponent procedure(
+  private static BundleEntryComponent procedure(Person person,
           BundleEntryComponent personEntry, Bundle bundle, BundleEntryComponent encounterEntry,
           Procedure procedure) {
     org.hl7.fhir.r4.model.Procedure procedureResource = new org.hl7.fhir.r4.model.Procedure();
@@ -2013,6 +2044,7 @@ private static BundleEntryComponent procedure(
         // we didn't find a matching Condition,
         // fallback to just reason code
         procedureResource.addReasonCode(mapCodeToCodeableConcept(reason, SNOMED_URI));
+        addTranslation("ICD10-CM", reason, procedureResource.getReasonCodeFirstRep(), person);
       }
     }
 
@@ -2322,6 +2354,8 @@ && shouldExport(org.hl7.fhir.r4.model.Medication.class)) {
         // we didn't find a matching Condition,
         // fallback to just reason code
         medicationResource.addReasonCode(mapCodeToCodeableConcept(reason, SNOMED_URI));
+        addTranslation("ICD10-CM", reason, medicationResource.getReasonCodeFirstRep(),
+                person);
       }
     }
 
@@ -2474,6 +2508,8 @@ private static BundleEntryComponent medicationAdministration(
         // we didn't find a matching Condition,
         // fallback to just reason code
         medicationResource.addReasonCode(mapCodeToCodeableConcept(reason, SNOMED_URI));
+        addTranslation("ICD10-CM", reason, medicationResource.getReasonCodeFirstRep(),
+                person);
       }
     }
 
@@ -2717,6 +2753,8 @@ private static BundleEntryComponent carePlan(Person person,
           activityDetailComponent.addReasonReference().setReference(reasonCondition.getFullUrl());
         } else if (reason != null) {
           activityDetailComponent.addReasonCode(mapCodeToCodeableConcept(reason, SNOMED_URI));
+          addTranslation("ICD10-CM", reason, activityDetailComponent.getReasonCodeFirstRep(),
+                  person);
         }
 
         activityComponent.setDetail(activityDetailComponent);
@@ -2863,7 +2901,9 @@ private static BundleEntryComponent careTeam(Person person,
 
     if (carePlan.reasons != null && !carePlan.reasons.isEmpty()) {
       for (Code code : carePlan.reasons) {
-        careTeam.addReasonCode(mapCodeToCodeableConcept(code, SNOMED_URI));
+        CodeableConcept concept = mapCodeToCodeableConcept(code, SNOMED_URI);
+        addTranslation("ICD10-CM", code, concept, person);
+        careTeam.addReasonCode(concept);
       }
     }
 

src/main/java/org/mitre/synthea/export/rif/CodeMapper.java

@@ -5,6 +5,7 @@
 import com.google.gson.reflect.TypeToken;
 import java.io.IOException;
 import java.lang.reflect.Type;
+import java.util.AbstractMap.SimpleEntry;
 import java.util.ArrayList;
 import java.util.Collections;
 import java.util.HashMap;
@@ -86,6 +87,26 @@ public CodeMapper(String jsonMapResource) {
     }
   }
 
+  /**
+   * Merge the mappings from the supplied code mapper. This method performs a deep
+   * merge such that destination codes will be merged if this and other contain
+   * mappings for the same source code. Care is required with the use of weights
+   * such that this and other use the same weighting scale (e.g. 0-1) otherwise one
+   * set of mappings could overwhelm the other.
+   * @param other the other code mapper from which mappings will be copied.
+   */
+  public void merge(CodeMapper other) {
+    if (other.hasMap()) {
+      other.map.forEach((source, mapped) -> {
+        if (map.containsKey(source)) {
+          map.get(source).addAll(mapped);
+        } else {
+          map.put(source, mapped);
+        }
+      });
+    }
+  }
+
   /**
    * Determines whether this mapper has an entry for the supplied code.
    * @param codeToMap the Synthea code to look for
@@ -249,6 +270,24 @@ public String map(Code codeToMap, String bfdCodeType, RandomNumberGenerator rand
     return map(codeToMap.code, bfdCodeType, rand, stripDots);
   }
 
+  /**
+   * Get one of the BFD codes for the supplied Synthea code. Equivalent to
+   * {@code map(codeToMap, "code", rand)}.
+   * @param codeToMap the Synthea code to look for
+   * @param rand a source of random numbers used to pick one of the list of BFD codes
+   * @return the BFD code and display string or null if the code can't be mapped
+   */
+  public SimpleEntry<String, String> mapToCodeAndDescription(Code codeToMap,
+          RandomNumberGenerator rand) {
+    if (!canMap(codeToMap)) {
+      return null;
+    }
+    RandomCollection<Map<String, String>> options = map.get(codeToMap.code);
+    Map<String, String> mappedCode = options.next(rand);
+    return new SimpleEntry<>(mappedCode.get("code"),
+            mappedCode.get("description"));
+  }
+
   /**
    * Get the missing code as a list of maps, where each map includes the mapper name, a missing
    * code, a description, and the count of times the code was requested.

src/main/java/org/mitre/synthea/helpers/RandomCollection.java

@@ -2,15 +2,14 @@
 
 import java.io.Serializable;
 import java.util.Map.Entry;
-import java.util.NavigableMap;
 import java.util.TreeMap;
 
 /**
  * Random collection of objects, with weightings. Intended to be an equivalent to the ruby Pickup
  * gem. Adapted from https://stackoverflow.com/a/6409791/630384
  */
 public class RandomCollection<E> implements Serializable {
-  private final NavigableMap<Double, E> map = new TreeMap<Double, E>();
+  private final TreeMap<Double, E> map = new TreeMap<Double, E>();
   private double total = 0;
 
   /**
@@ -28,6 +27,18 @@ public void add(double weight, E result) {
     map.put(total, result);
   }
 
+  /**
+   * Add all of the entries from the supplied RandomCollection.
+   * @param other the collection from which to copy entries.
+   */
+  public void addAll(RandomCollection<E> other) {
+    Double weightAdj = 0.0;
+    for (Entry<Double, E> e: other.map.entrySet()) {
+      add(e.getKey() - weightAdj, e.getValue());
+      weightAdj = e.getKey();
+    }
+  }
+
   /**
    * Select an item from the collection at random by the weight of the items.
    * Selecting an item from one draw, does not remove the item from the collection