Merge pull request cfpb#1156 from nickgrippin/n52

N52

Author: schbetsy

census/src/main/resources/jul_2015_cbsa.csv

@@ -10,7 +10,7 @@ import com.github.tototoshi.csv.CSVParser.parse
 object CbsaLookup extends CbsaResourceUtils {
   val values: Seq[Cbsa] = {
     val lines = resourceLines("/jul_2015_cbsa.csv", "ISO-8859-1")
-    lines.map { line =>
+    lines.drop(3).map { line =>
       val values = parse(line, '\\', ',', '"').getOrElse(List())
       val cbsaCode = values(0)
       val metroDivCode = values(1)

census/src/main/scala/hmda/census/model/CbsaLookup.scala

@@ -2,8 +2,26 @@ package hmda.model.publication.reports
 
 sealed trait BorrowerCharacteristic
 
-case class RaceBorrowerCharacteristic(races: List[RaceCharacteristic]) extends BorrowerCharacteristic
+case class RaceBorrowerCharacteristic(races: List[RaceCharacteristic]) extends BorrowerCharacteristic {
+  def +(rbc: RaceBorrowerCharacteristic) = {
+    val combined = races.map(r =>
+      r + rbc.races.find(_.race == r.race).get)
+    RaceBorrowerCharacteristic(combined)
+  }
+}
 
-case class EthnicityBorrowerCharacteristic(ethnicities: List[EthnicityCharacteristic]) extends BorrowerCharacteristic
+case class EthnicityBorrowerCharacteristic(ethnicities: List[EthnicityCharacteristic]) extends BorrowerCharacteristic {
+  def +(ebc: EthnicityBorrowerCharacteristic) = {
+    val combined = ethnicities.map(e =>
+      e + ebc.ethnicities.find(_.ethnicity == e.ethnicity).get)
+    EthnicityBorrowerCharacteristic(combined)
+  }
+}
 
-case class MinorityStatusBorrowerCharacteristic(minoritystatus: List[MinorityStatusCharacteristic]) extends BorrowerCharacteristic
+case class MinorityStatusBorrowerCharacteristic(minoritystatus: List[MinorityStatusCharacteristic]) extends BorrowerCharacteristic {
+  def +(msbc: MinorityStatusBorrowerCharacteristic) = {
+    val combined = minoritystatus.map(m =>
+      m + msbc.minoritystatus.find(_.minorityStatus == m.minorityStatus).get)
+    MinorityStatusBorrowerCharacteristic(combined)
+  }
+}

model/shared/src/main/scala/hmda/model/publication/reports/BorrowerCharacteristic.scala

@@ -1,9 +1,29 @@
 package hmda.model.publication.reports
 
-sealed trait Characteristic
+sealed trait Characteristic {
+  def dispositions: List[Disposition]
+}
 
-case class RaceCharacteristic(race: RaceEnum, dispositions: List[Disposition]) extends Characteristic
+case class RaceCharacteristic(race: RaceEnum, dispositions: List[Disposition]) extends Characteristic {
+  def +(rc: RaceCharacteristic) = {
+    val combined = dispositions.map(d =>
+      d + rc.dispositions.find(_.disposition == d.disposition).get)
+    RaceCharacteristic(race, combined)
+  }
+}
 
-case class EthnicityCharacteristic(ethnicity: EthnicityEnum, dispositions: List[Disposition]) extends Characteristic
+case class EthnicityCharacteristic(ethnicity: EthnicityEnum, dispositions: List[Disposition]) extends Characteristic {
+  def +(ec: EthnicityCharacteristic) = {
+    val combined = dispositions.map(d =>
+      d + ec.dispositions.find(_.disposition == d.disposition).get)
+    EthnicityCharacteristic(ethnicity, combined)
+  }
+}
 
-case class MinorityStatusCharacteristic(minorityStatus: MinorityStatusEnum, dispositions: List[Disposition]) extends Characteristic
+case class MinorityStatusCharacteristic(minorityStatus: MinorityStatusEnum, dispositions: List[Disposition]) extends Characteristic {
+  def +(msc: MinorityStatusCharacteristic) = {
+    val combined = dispositions.map(d =>
+      d + msc.dispositions.find(_.disposition == d.disposition).get)
+    MinorityStatusCharacteristic(minorityStatus, combined)
+  }
+}

model/shared/src/main/scala/hmda/model/publication/reports/Characteristic.scala

@@ -1,7 +1,11 @@
 package hmda.model.publication.reports
 
 case class Disposition(
-  disposition: ActionTakenTypeEnum,
-  count: Int,
-  value: Int
-)
+    disposition: ActionTakenTypeEnum,
+    count: Int,
+    value: Int
+) {
+  def +(disp: Disposition): Disposition = {
+    Disposition(disposition, count + disp.count, value + disp.value)
+  }
+}

model/shared/src/main/scala/hmda/model/publication/reports/Disposition.scala

@@ -2,3 +2,4 @@ ReportId,Type,ReportTable,Dispositions,Description
 A52,Aggregate,5-2,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications for Conventional Home-Purchase Loans, 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"
 D51,Disclosure,5-1,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of applications for FHA, FSA/RHS, and VA home-purchase loans, 1- to 4-family and manufactured home dwellings, by income, race and ethnicity of applicant"
 D53,Disclosure,5-3,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications to Refinance Loans on 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"
+N52,National Aggregate,5-2,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications for Conventional Home-Purchase Loans, 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"

publication/src/main/resources/reports-metadata.txt

@@ -56,7 +56,6 @@ object A52 {
       year <- yearF
       total <- totalF
     } yield {
-
       A52(
         year,
         msa,

publication/src/main/scala/hmda/publication/reports/aggregate/A52.scala

@@ -0,0 +1,77 @@
+package hmda.publication.reports.national
+
+import akka.NotUsed
+import akka.stream.scaladsl.Source
+import hmda.census.model.MsaIncomeLookup
+import hmda.model.publication.reports._
+import hmda.publication.reports._
+import hmda.publication.reports.aggregate.A52
+import hmda.publication.reports.util.ReportUtil._
+import hmda.publication.reports.util.ReportsMetaDataLookup
+import hmda.query.model.filing.LoanApplicationRegisterQuery
+
+import scala.concurrent.Future
+
+case class N52(
+    year: Int = 0,
+    reportDate: String = formattedCurrentDate,
+    applicantIncomes: List[ApplicantIncome] = List(),
+    total: List[Disposition] = List(),
+    table: String = N52.metaData.reportTable,
+    description: String = N52.metaData.description
+) extends NationalAggregateReport {
+  def +(a52: A52): N52 = {
+    if (applicantIncomes.isEmpty) {
+      N52(a52.year, a52.reportDate, a52.applicantIncomes, a52.total)
+    } else {
+      val combinedIncomes = a52.applicantIncomes.map(income => {
+        val newC = applicantIncomes.find(i => i.applicantIncome == income.applicantIncome).get.borrowerCharacteristics
+        val originalC = income.borrowerCharacteristics
+
+        val nR = newC.find(_.isInstanceOf[RaceBorrowerCharacteristic]).get.asInstanceOf[RaceBorrowerCharacteristic]
+        val oR = originalC.find(_.isInstanceOf[RaceBorrowerCharacteristic]).get.asInstanceOf[RaceBorrowerCharacteristic]
+        val cR = oR + nR
+
+        val nE = newC.find(_.isInstanceOf[EthnicityBorrowerCharacteristic]).get.asInstanceOf[EthnicityBorrowerCharacteristic]
+        val oE = originalC.find(_.isInstanceOf[EthnicityBorrowerCharacteristic]).get.asInstanceOf[EthnicityBorrowerCharacteristic]
+        val cE = oE + nE
+
+        val nM = newC.find(_.isInstanceOf[MinorityStatusBorrowerCharacteristic]).get.asInstanceOf[MinorityStatusBorrowerCharacteristic]
+        val oM = originalC.find(_.isInstanceOf[MinorityStatusBorrowerCharacteristic]).get.asInstanceOf[MinorityStatusBorrowerCharacteristic]
+        val cM = oM + nM
+
+        ApplicantIncome(income.applicantIncome, List(cR, cE, cM))
+      })
+
+      val combinedDispositions = a52.total.map(disposition => {
+        val originalDisposition = total.find(d => d.disposition == disposition.disposition).get
+        disposition + originalDisposition
+      })
+
+      N52(year, reportDate, combinedIncomes, combinedDispositions)
+    }
+  }
+}
+
+object N52 {
+  val metaData = ReportsMetaDataLookup.values("N52")
+  val dispositions = metaData.dispositions
+
+  // Table filters:
+  // Loan Type 1
+  // Property Type 1,2
+  // Purpose of Loan 1
+  def generate[ec: EC, mat: MAT, as: AS](larSource: Source[LoanApplicationRegisterQuery, NotUsed]): Future[N52] = {
+    val fipsList = MsaIncomeLookup.values.map(_.fips).filterNot(_ == 99999)
+
+    val a52List = fipsList.map(fipsCode => A52.generate(larSource, fipsCode))
+    val n52f = Future.sequence(a52List).map(seq => {
+      seq.foldLeft(N52())((n52, a52) => n52 + a52)
+    })
+
+    for {
+      n <- n52f
+      y <- calculateYear(larSource)
+    } yield N52(y, n.reportDate, n.applicantIncomes, n.total)
+  }
+}

publication/src/main/scala/hmda/publication/reports/national/N52.scala

@@ -0,0 +1,14 @@
+package hmda.publication.reports.national
+
+import hmda.model.publication.reports.ReportTypeEnum.NationalAggregate
+import hmda.model.publication.reports.ReportTypeEnum
+
+trait NationalAggregateReport {
+
+  val description: String
+  val table: String
+  val year: Int
+
+  val reportType: ReportTypeEnum = NationalAggregate
+
+}

publication/src/main/scala/hmda/publication/reports/national/NationalAggregateReport.scala

@@ -0,0 +1,30 @@
+package hmda.publication.reports.national
+
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
+import akka.util.Timeout
+import com.typesafe.config.ConfigFactory
+import hmda.query.repository.filing.FilingCassandraRepository
+import hmda.publication.reports.protocol.national.N52Protocol._
+
+import scala.concurrent.Future
+import scala.concurrent.duration._
+import spray.json._
+
+class NationalAggregateReports(val sys: ActorSystem, val mat: ActorMaterializer) extends FilingCassandraRepository {
+  override implicit def system: ActorSystem = sys
+  override implicit def materializer: ActorMaterializer = mat
+  val config = ConfigFactory.load()
+  val duration = config.getInt("hmda.actor-lookup-timeout")
+  implicit val timeout = Timeout(duration.seconds)
+
+  val larSource = readData(1000)
+
+  def generateReports(fipsCode: Int, respId: String): Future[Unit] = {
+
+    val n52F = N52.generate(larSource)
+    n52F.map { n52 =>
+      println(n52.toJson.prettyPrint)
+    }
+  }
+}

publication/src/main/scala/hmda/publication/reports/national/NationalAggregateReports.scala

@@ -0,0 +1,47 @@
+package hmda.publication.reports.protocol.national
+
+import hmda.model.publication.reports.ReportTypeEnum._
+import hmda.model.publication.reports.{ ApplicantIncome, Disposition }
+import hmda.publication.reports.national.N52
+import hmda.publication.reports.protocol.{ ApplicantIncomeProtocol, MSAReportProtocol, ReportTypeEnumProtocol }
+import spray.json._
+
+object N52Protocol
+    extends DefaultJsonProtocol
+    with ReportTypeEnumProtocol
+    with MSAReportProtocol
+    with ApplicantIncomeProtocol {
+
+  implicit object N52Format extends RootJsonFormat[N52] {
+    override def write(obj: N52): JsValue = {
+      JsObject(
+        "table" -> JsString("5-2"),
+        "type" -> JsString(NationalAggregate.toString),
+        "desc" -> JsString(obj.description),
+        "year" -> JsNumber(obj.year),
+        "reportDate" -> JsString(obj.reportDate),
+        "applicantIncomes" -> obj.applicantIncomes.toJson,
+        "total" -> obj.total.toJson
+      )
+    }
+
+    override def read(json: JsValue): N52 = json.asJsObject.getFields(
+      "table",
+      "type",
+      "desc",
+      "year",
+      "reportDate",
+      "applicantIncomes",
+      "total"
+    ) match {
+        case Seq(table, reportType, description, year, reportDate, applicantIncomes, total) =>
+          N52(
+            year.convertTo[Int],
+            reportDate.convertTo[String],
+            applicantIncomes.convertTo[List[ApplicantIncome]],
+            total.convertTo[List[Disposition]]
+          )
+      }
+  }
+
+}

publication/src/main/scala/hmda/publication/reports/protocol/national/N52Protocol.scala

@@ -0,0 +1,31 @@
+package hmda.publication.reports.national
+
+import org.scalatest.prop.PropertyChecks
+import org.scalatest.{ MustMatchers, PropSpec }
+import spray.json._
+import NationalAggregateReportGenerators._
+import hmda.publication.reports.protocol.national.N52Protocol._
+
+class N52ProtocolSpec extends PropSpec with PropertyChecks with MustMatchers {
+
+  property("N52 Report must convert to and from JSON") {
+    forAll(n52Gen) { n52 =>
+      n52.toJson.convertTo[N52] mustBe n52
+    }
+  }
+
+  property("N52 Report must serialize to the correct JSON format") {
+    forAll(n52Gen) { n52 =>
+      n52.toJson mustBe JsObject(
+        "table" -> JsString("5-2"),
+        "type" -> JsString("NationalAggregate"),
+        "desc" -> JsString("Disposition of Applications for Conventional Home-Purchase Loans, 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"),
+        "year" -> JsNumber(n52.year),
+        "reportDate" -> JsString(n52.reportDate),
+        "applicantIncomes" -> n52.applicantIncomes.toJson,
+        "total" -> n52.total.toJson
+      )
+    }
+  }
+}
+

publication/src/test/scala/hmda/publication/reports/national/N52ProtocolSpec.scala

@@ -0,0 +1,66 @@
+package hmda.publication.reports.national
+
+import akka.NotUsed
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.Source
+import hmda.census.model.CbsaLookup
+import hmda.model.fi.lar.{ LarGenerators, LoanApplicationRegister }
+import hmda.model.publication.reports.ActionTakenTypeEnum._
+import hmda.model.publication.reports.ApplicantIncomeEnum.LessThan50PercentOfMSAMedian
+import hmda.model.publication.reports.{ EthnicityBorrowerCharacteristic, MinorityStatusBorrowerCharacteristic, RaceBorrowerCharacteristic }
+import hmda.query.model.filing.LoanApplicationRegisterQuery
+import hmda.query.repository.filing.LarConverter._
+import org.scalacheck.Gen
+import org.scalatest.{ AsyncWordSpec, BeforeAndAfterAll, MustMatchers }
+
+class N52Spec extends AsyncWordSpec with MustMatchers with LarGenerators with BeforeAndAfterAll {
+
+  implicit val system = ActorSystem()
+  implicit val ec = system.dispatcher
+  implicit val materializer = ActorMaterializer()
+
+  override def afterAll(): Unit = {
+    super.afterAll()
+    system.terminate()
+  }
+
+  val fips = CbsaLookup.values.map(_.cbsa).filterNot(f => f.isEmpty || f == "99999")
+  val fipsGen = Gen.oneOf(fips)
+  def propType = Gen.oneOf(1, 2).sample.get
+
+  val lars = larListGen.sample.get.map { lar: LoanApplicationRegister =>
+    val geo = lar.geography.copy(msa = fipsGen.sample.get)
+    val loan = lar.loan.copy(loanType = 1, propertyType = propType, purpose = 1)
+    lar.copy(geography = geo, loan = loan)
+  }
+
+  val source: Source[LoanApplicationRegisterQuery, NotUsed] = Source
+    .fromIterator(() => lars.toIterator)
+    .map(lar => toLoanApplicationRegisterQuery(lar))
+
+  val expectedDispositions = List(ApplicationReceived, LoansOriginated, ApprovedButNotAccepted, ApplicationsDenied, ApplicationsWithdrawn, ClosedForIncompleteness)
+
+  "Generate an National Aggregate 5-2 report" in {
+    N52.generate(source).map { result =>
+
+      result.table mustBe "5-2"
+      result.applicantIncomes.size mustBe 5
+
+      val lowestIncome = result.applicantIncomes.head
+      lowestIncome.applicantIncome mustBe LessThan50PercentOfMSAMedian
+
+      val races = lowestIncome.borrowerCharacteristics.head.asInstanceOf[RaceBorrowerCharacteristic].races
+      races.size mustBe 8
+
+      val ethnicities = lowestIncome.borrowerCharacteristics(1).asInstanceOf[EthnicityBorrowerCharacteristic].ethnicities
+      ethnicities.size mustBe 4
+
+      val minorityStatuses = lowestIncome.borrowerCharacteristics(2).asInstanceOf[MinorityStatusBorrowerCharacteristic].minoritystatus
+      minorityStatuses.size mustBe 2
+
+      races.head.dispositions.map(_.disposition) mustBe expectedDispositions
+    }
+  }
+
+}

publication/src/test/scala/hmda/publication/reports/national/N52Spec.scala

@@ -0,0 +1,18 @@
+package hmda.publication.reports.national
+
+import java.util.Calendar
+
+import hmda.publication.reports.ReportGenerators._
+import hmda.publication.reports.util.DateUtil._
+import org.scalacheck.Gen
+
+object NationalAggregateReportGenerators {
+  implicit def n52Gen: Gen[N52] = {
+    for {
+      total <- totalDispositionGen
+      year = Calendar.getInstance().get(Calendar.YEAR)
+      reportDate = formatDate(Calendar.getInstance().toInstant)
+      applicantIncomes <- Gen.listOfN(5, applicantIncomeGen)
+    } yield N52(year, reportDate, applicantIncomes, total)
+  }
+}