Merge branch 'master' into A53

Author: schbetsy

README.md

@@ -225,6 +225,25 @@ The Public API will run on `$(docker-machine ip):8082`
 By default, the `HDMA Platform` runs with a log level of `INFO`. This can be changed by establishing a different log level in the `HMDA_LOGLEVEL` environment variable.
 For the different logging options, see the [reference.conf](https://github.com/akka/akka/blob/master/akka-actor/src/main/resources/reference.conf#L38) default configuration file for `Akka`.
 
+#### API Load Testing
+
+A load testing scenario for the API has been built using [Gatling](https://gatling.io/). The parameters for this test can be configured in the `cluster/src/test/resources/application.conf` file.
+To run the default scenario:
+
+```shell
+$ sbt
+> project cluster
+> gatling:test
+```
+
+After the test is run, some statistics will be presented on the screen. To further study the results of the test, a report can be opened on a browser from the sbt prompt:
+
+```shell
+> gatling:lastReport
+```
+
+** NOTE: The load test requires the sample panel [file](loader/src/main/resources/inst_data_2017_dummy.csv) to be loaded before running it.
+
 #### To run the entire platform
 
 1. Ensure you have a Docker Machine with sufficient resources, as described in the [Docker](#docker) section above.

build.sbt

@@ -3,8 +3,9 @@ import Dependencies._
 import sbtassembly.AssemblyPlugin.autoImport._
 import spray.revolver.RevolverPlugin.autoImport.Revolver
 import org.scalajs.sbtplugin.ScalaJSPlugin.autoImport._
+import io.gatling.sbt.GatlingPlugin
 
-val commonDeps = Seq(logback, scalaTest, scalaCheck)
+val commonDeps = Seq(logback, scalaTest, scalaCheck, gatling, gatlingHighcharts)
 
 val akkaDeps = commonDeps ++ Seq(akka, akkaCluster, akkaClusterTools, akkaSlf4J, akkaStream, akkaTestkit, constructr, constructrZookeeper, akkaClusterManagement)
 
@@ -58,14 +59,15 @@ lazy val hmda = (project in file("."))
 
 
 lazy val cluster = (project in file("cluster"))
+  .enablePlugins(GatlingPlugin)
   .settings(hmdaBuildSettings: _*)
   .settings(
     Seq(
       libraryDependencies ++= akkaDeps ++ configDeps
     )
   )
   .dependsOn(
-    modelJVM,
+    modelJVM % "compile->compile;test->test",
     parserJVM,
     apiModel,
     api,

census/src/main/resources/jul_2015_cbsa.csv

@@ -10,7 +10,7 @@ import com.github.tototoshi.csv.CSVParser.parse
 object CbsaLookup extends CbsaResourceUtils {
   val values: Seq[Cbsa] = {
     val lines = resourceLines("/jul_2015_cbsa.csv", "ISO-8859-1")
-    lines.map { line =>
+    lines.drop(3).map { line =>
       val values = parse(line, '\\', ',', '"').getOrElse(List())
       val cbsaCode = values(0)
       val metroDivCode = values(1)

census/src/main/scala/hmda/census/model/CbsaLookup.scala

@@ -0,0 +1,13 @@
+hmda {
+  benchmark {
+    host = "localhost"
+    host = ${?HMDA_BENCHMARK_HOST}
+    port = 8080
+    port = ${?HMDA_BENCHMARK_PORT}
+    adminPort = 8081
+    adminPort = ${?HMDA_BENCHMARK_ADMIN_PORT}
+    nrOfUsers = 50
+    rampUpTime = 30
+    feederFile = "institutions100.csv"
+  }
+}

cluster/src/test/resources/application.conf

@@ -0,0 +1,11 @@
+institutionId,fileName
+74140,clean_164-lars.txt
+24949,clean_164-lars.txt
+212465,clean_164-lars.txt
+288853,clean_164-lars.txt
+18827,clean_164-lars.txt
+23456,clean_164-lars.txt
+35570,clean_164-lars.txt
+886,clean_164-lars.txt
+60143,clean_164-lars.txt
+4277,clean_164-lars.txt

cluster/src/test/resources/clean_164-lars.txt

@@ -0,0 +1,101 @@
+institutionId,fileName
+74140,clean_164-lars.txt
+24949,clean_164-lars.txt
+212465,clean_164-lars.txt
+288853,clean_164-lars.txt
+18827,clean_164-lars.txt
+23456,clean_164-lars.txt
+35570,clean_164-lars.txt
+886,clean_164-lars.txt
+60143,clean_164-lars.txt
+3458,clean_164-lars.txt
+792,clean_164-lars.txt
+27614,clean_164-lars.txt
+11640,clean_164-lars.txt
+16551,clean_164-lars.txt
+2277093,clean_164-lars.txt
+2027364,clean_164-lars.txt
+5210,clean_164-lars.txt
+2318482,clean_164-lars.txt
+2282,clean_164-lars.txt
+13082,clean_164-lars.txt
+77422,clean_164-lars.txt
+208244,clean_164-lars.txt
+2350222,clean_164-lars.txt
+457,clean_164-lars.txt
+13103,clean_164-lars.txt
+12281,clean_164-lars.txt
+63573,clean_164-lars.txt
+339858,clean_164-lars.txt
+34742,clean_164-lars.txt
+143662,clean_164-lars.txt
+1864722,clean_164-lars.txt
+2161,clean_164-lars.txt
+4192,clean_164-lars.txt
+27070,clean_164-lars.txt
+17259,clean_164-lars.txt
+24453,clean_164-lars.txt
+18489,clean_164-lars.txt
+16748,clean_164-lars.txt
+4231,clean_164-lars.txt
+14874,clean_164-lars.txt
+14865,clean_164-lars.txt
+1072246,clean_164-lars.txt
+7979,clean_164-lars.txt
+26176,clean_164-lars.txt
+370271,clean_164-lars.txt
+27847,clean_164-lars.txt
+7287,clean_164-lars.txt
+10849,clean_164-lars.txt
+11387,clean_164-lars.txt
+3720,clean_164-lars.txt
+23504,clean_164-lars.txt
+9357,clean_164-lars.txt
+2351078,clean_164-lars.txt
+64150,clean_164-lars.txt
+4839,clean_164-lars.txt
+2399940,clean_164-lars.txt
+6329,clean_164-lars.txt
+31255,clean_164-lars.txt
+379920,clean_164-lars.txt
+1223440,clean_164-lars.txt
+17147,clean_164-lars.txt
+229801,clean_164-lars.txt
+58243,clean_164-lars.txt
+75633,clean_164-lars.txt
+48374,clean_164-lars.txt
+2376,clean_164-lars.txt
+26242,clean_164-lars.txt
+9807,clean_164-lars.txt
+85052,clean_164-lars.txt
+2554,clean_164-lars.txt
+18788,clean_164-lars.txt
+249612,clean_164-lars.txt
+1655797,clean_164-lars.txt
+354,clean_164-lars.txt
+12030,clean_164-lars.txt
+1829,clean_164-lars.txt
+1578449,clean_164-lars.txt
+1074736,clean_164-lars.txt
+266271,clean_164-lars.txt
+13390,clean_164-lars.txt
+17473,clean_164-lars.txt
+33147,clean_164-lars.txt
+29878,clean_164-lars.txt
+233031,clean_164-lars.txt
+41245,clean_164-lars.txt
+1454,clean_164-lars.txt
+2213046,clean_164-lars.txt
+12937,clean_164-lars.txt
+28255,clean_164-lars.txt
+17464,clean_164-lars.txt
+214807,clean_164-lars.txt
+26765,clean_164-lars.txt
+2318437,clean_164-lars.txt
+3588,clean_164-lars.txt
+242,clean_164-lars.txt
+28013,clean_164-lars.txt
+18854,clean_164-lars.txt
+2198,clean_164-lars.txt
+66154,clean_164-lars.txt
+41629,clean_164-lars.txt

cluster/src/test/resources/institutions10.csv

@@ -0,0 +1,105 @@
+package api.http.benchmark
+
+import com.typesafe.config.ConfigFactory
+import io.gatling.core.Predef._
+import io.gatling.http.Predef._
+import scala.concurrent.duration._
+import scala.language.postfixOps
+
+class FilingSimulation extends Simulation {
+
+  val config = ConfigFactory.load()
+  val host = config.getString("hmda.benchmark.host")
+  val port = config.getInt("hmda.benchmark.port")
+  val adminPort = config.getInt("hmda.benchmark.adminPort")
+  val nrOfUsers = config.getInt("hmda.benchmark.nrOfUsers")
+  val rampUpTime = config.getInt("hmda.benchmark.rampUpTime")
+  val feeder = csv(config.getString("hmda.benchmark.feederFile")).random
+
+  val httpProtocol = http
+    .baseURL(s"http://$host:$port")
+    .acceptHeader("text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8")
+    .acceptEncodingHeader("gzip, deflate")
+    .acceptLanguageHeader("en-US,en;q=0.5")
+    .acceptLanguageHeader("en-US,en;q=0.5")
+    .userAgentHeader("Mozilla/5.0 (Macintosh; Intel Mac OS X 10.8; rv:16.0) Gecko/20100101 Firefox/16.0")
+    .header("cfpb-hmda-username", "user")
+    .disableCaching
+
+  object Institutions {
+
+    val hmdaFilingApi =
+      feed(feeder)
+        .exec(http("Search Institutions")
+          .get("/institutions")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .check(status is 200))
+        .pause(1)
+        .exec(http("Institution by id")
+          .get("/institutions/${institutionId}")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .check(status is 200)
+          .check(jsonPath("$.institution.id") is "${institutionId}"))
+        .pause(1)
+        .exec(http("List Filings")
+          .get("/institutions/${institutionId}/filings/2017")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .check(status is 200))
+        .pause(1)
+        .exec(http("Create Submission")
+          .post("/institutions/${institutionId}/filings/2017/submissions")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .check(status is 201)
+          .check(jsonPath("$.id.sequenceNumber").saveAs("submissionId")))
+        .pause(2)
+        .exec(http("Upload file")
+          .post("/institutions/${institutionId}/filings/2017/submissions/${submissionId}")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .bodyPart(RawFileBodyPart("file", "${fileName}"))
+          .check(status is 202))
+        .pause(10)
+        .exec(http("View edits")
+          .get("/institutions/${institutionId}/filings/2017/submissions/${submissionId}/edits")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .check(status is 200)
+          .check(jsonPath("$.status.code") is "8"))
+        .pause(5)
+        .exec(http("Validate Quality Edits")
+          .post("/institutions/${institutionId}/filings/2017/submissions/${submissionId}/edits/quality")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .body(StringBody("""{ "verified": true }""")).asJSON
+          .check(status is 200))
+        .pause(1)
+        .exec(http("Validate Macro Edits")
+          .post("/institutions/${institutionId}/filings/2017/submissions/${submissionId}/edits/macro")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .body(StringBody("""{ "verified": true }""")).asJSON
+          .check(status is 200))
+        .pause(1)
+        .exec(http("Get Summary")
+          .get("/institutions/${institutionId}/filings/2017/submissions/${submissionId}/summary")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .check(status is 200))
+        .pause(1)
+        .exec(http("IRS Report")
+          .get("/institutions/${institutionId}/filings/2017/submissions/${submissionId}/irs")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .check(status is 200))
+        .pause(1)
+        .exec(http("Sign Submission")
+          .post("/institutions/${institutionId}/filings/2017/submissions/${submissionId}/sign")
+          .header("cfpb-hmda-institutions", "${institutionId}")
+          .body(StringBody(""" { "signed": true } """)).asJSON
+          .check(status is 200)
+          .check(jsonPath("$.status.code") is "10"))
+
+  }
+
+  val user = scenario("HMDA User")
+    .exec(Institutions.hmdaFilingApi)
+
+  setUp(
+    user.inject(rampUsers(nrOfUsers) over (rampUpTime seconds)).protocols(httpProtocol)
+  )
+
+}

cluster/src/test/resources/institutions100.csv

@@ -2,8 +2,26 @@ package hmda.model.publication.reports
 
 sealed trait BorrowerCharacteristic
 
-case class RaceBorrowerCharacteristic(races: List[RaceCharacteristic]) extends BorrowerCharacteristic
+case class RaceBorrowerCharacteristic(races: List[RaceCharacteristic]) extends BorrowerCharacteristic {
+  def +(rbc: RaceBorrowerCharacteristic) = {
+    val combined = races.map(r =>
+      r + rbc.races.find(_.race == r.race).get)
+    RaceBorrowerCharacteristic(combined)
+  }
+}
 
-case class EthnicityBorrowerCharacteristic(ethnicities: List[EthnicityCharacteristic]) extends BorrowerCharacteristic
+case class EthnicityBorrowerCharacteristic(ethnicities: List[EthnicityCharacteristic]) extends BorrowerCharacteristic {
+  def +(ebc: EthnicityBorrowerCharacteristic) = {
+    val combined = ethnicities.map(e =>
+      e + ebc.ethnicities.find(_.ethnicity == e.ethnicity).get)
+    EthnicityBorrowerCharacteristic(combined)
+  }
+}
 
-case class MinorityStatusBorrowerCharacteristic(minoritystatus: List[MinorityStatusCharacteristic]) extends BorrowerCharacteristic
+case class MinorityStatusBorrowerCharacteristic(minoritystatus: List[MinorityStatusCharacteristic]) extends BorrowerCharacteristic {
+  def +(msbc: MinorityStatusBorrowerCharacteristic) = {
+    val combined = minoritystatus.map(m =>
+      m + msbc.minoritystatus.find(_.minorityStatus == m.minorityStatus).get)
+    MinorityStatusBorrowerCharacteristic(combined)
+  }
+}

cluster/src/test/scala/api/http/benchmark/FilingSimulation.scala

@@ -1,9 +1,29 @@
 package hmda.model.publication.reports
 
-sealed trait Characteristic
+sealed trait Characteristic {
+  def dispositions: List[Disposition]
+}
 
-case class RaceCharacteristic(race: RaceEnum, dispositions: List[Disposition]) extends Characteristic
+case class RaceCharacteristic(race: RaceEnum, dispositions: List[Disposition]) extends Characteristic {
+  def +(rc: RaceCharacteristic) = {
+    val combined = dispositions.map(d =>
+      d + rc.dispositions.find(_.disposition == d.disposition).get)
+    RaceCharacteristic(race, combined)
+  }
+}
 
-case class EthnicityCharacteristic(ethnicity: EthnicityEnum, dispositions: List[Disposition]) extends Characteristic
+case class EthnicityCharacteristic(ethnicity: EthnicityEnum, dispositions: List[Disposition]) extends Characteristic {
+  def +(ec: EthnicityCharacteristic) = {
+    val combined = dispositions.map(d =>
+      d + ec.dispositions.find(_.disposition == d.disposition).get)
+    EthnicityCharacteristic(ethnicity, combined)
+  }
+}
 
-case class MinorityStatusCharacteristic(minorityStatus: MinorityStatusEnum, dispositions: List[Disposition]) extends Characteristic
+case class MinorityStatusCharacteristic(minorityStatus: MinorityStatusEnum, dispositions: List[Disposition]) extends Characteristic {
+  def +(msc: MinorityStatusCharacteristic) = {
+    val combined = dispositions.map(d =>
+      d + msc.dispositions.find(_.disposition == d.disposition).get)
+    MinorityStatusCharacteristic(minorityStatus, combined)
+  }
+}

model/shared/src/main/scala/hmda/model/publication/reports/BorrowerCharacteristic.scala

@@ -1,7 +1,11 @@
 package hmda.model.publication.reports
 
 case class Disposition(
-  disposition: ActionTakenTypeEnum,
-  count: Int,
-  value: Int
-)
+    disposition: ActionTakenTypeEnum,
+    count: Int,
+    value: Int
+) {
+  def +(disp: Disposition): Disposition = {
+    Disposition(disposition, count + disp.count, value + disp.value)
+  }
+}

model/shared/src/main/scala/hmda/model/publication/reports/Characteristic.scala

@@ -41,4 +41,7 @@ object Dependencies {
   val alpakkaCassandra = "com.lightbend.akka" %% "akka-stream-alpakka-cassandra" % Version.alpakka
   val cassandraDriver = "com.datastax.cassandra" % "cassandra-driver-core" % Version.cassandraDriver
   val javaMail = "javax.mail" % "mail" % Version.javaMail
+  val gatlingHighcharts = "io.gatling.highcharts" % "gatling-charts-highcharts" % Version.gatling % "test"
+  val gatling = "io.gatling" % "gatling-test-framework" % Version.gatling % "test"
+
 }

model/shared/src/main/scala/hmda/model/publication/reports/Disposition.scala

@@ -23,4 +23,5 @@ object Version {
   val alpakka = "0.13"
   val cassandraDriver = "3.2.0"
   val javaMail = "1.4.7"
+  val gatling = "2.3.0"
 }

project/Dependencies.scala

@@ -0,0 +1 @@
+addSbtPlugin("io.gatling" % "gatling-sbt" % "2.2.2")

project/Version.scala

@@ -3,3 +3,4 @@ A52,Aggregate,5-2,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Cl
 A53,Aggregate,5-3,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications to Refinance Loans on 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"
 D51,Disclosure,5-1,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of applications for FHA, FSA/RHS, and VA home-purchase loans, 1- to 4-family and manufactured home dwellings, by income, race and ethnicity of applicant"
 D53,Disclosure,5-3,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications to Refinance Loans on 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"
+N52,National Aggregate,5-2,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications for Conventional Home-Purchase Loans, 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"