Merge pull request cfpb#1165 from schbetsy/A53

A53

Author: Nick Grippin

publication/src/main/resources/reports-metadata.txt

@@ -1,5 +1,6 @@
 ReportId,Type,ReportTable,Dispositions,Description
 A52,Aggregate,5-2,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications for Conventional Home-Purchase Loans, 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"
+A53,Aggregate,5-3,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications to Refinance Loans on 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"
 D51,Disclosure,5-1,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of applications for FHA, FSA/RHS, and VA home-purchase loans, 1- to 4-family and manufactured home dwellings, by income, race and ethnicity of applicant"
 D52,Disclosure,5-2,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications for Conventional Home-Purchase Loans, 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"
 D53,Disclosure,5-3,Received;Originated;ApprovedButNotAccepted;Denied;Withdrawn;Closed,"Disposition of Applications to Refinance Loans on 1-to-4 Family and Manufactured Home Dwellings, by Income, Race, and Ethnicity of Applicant"

publication/src/main/scala/hmda/publication/reports/aggregate/A52.scala

@@ -2,66 +2,21 @@ package hmda.publication.reports.aggregate
 
 import akka.NotUsed
 import akka.stream.scaladsl.Source
-import hmda.model.publication.reports.{ ApplicantIncome, Disposition, MSAReport }
 import hmda.publication.reports._
-import hmda.publication.reports.util.ReportUtil._
-import hmda.publication.reports.util.ReportsMetaDataLookup
 import hmda.query.model.filing.LoanApplicationRegisterQuery
 
 import scala.concurrent.Future
 
-case class A52(
-  year: Int,
-  msa: MSAReport,
-  applicantIncomes: List[ApplicantIncome],
-  total: List[Disposition],
-  reportDate: String = formattedCurrentDate,
-  table: String = A52.metaData.reportTable,
-  description: String = A52.metaData.description
-) extends AggregateReport
-
 object A52 {
-  val metaData = ReportsMetaDataLookup.values("A52")
-  val dispositions = metaData.dispositions
 
-  // Table filters:
-  // Loan Type 1
-  // Property Type 1,2
-  // Purpose of Loan 1
+  def filters(lar: LoanApplicationRegisterQuery): Boolean = {
+    (lar.loanType == 1) &&
+      (lar.propertyType == 1 || lar.propertyType == 2) &&
+      (lar.purpose == 1)
+  }
+
   def generate[ec: EC, mat: MAT, as: AS](
     larSource: Source[LoanApplicationRegisterQuery, NotUsed],
     fipsCode: Int
-  ): Future[A52] = {
-    val lars = larSource
-      .filter(lar => lar.msa != "NA")
-      .filter(lar => lar.msa.toInt == fipsCode)
-      .filter { lar =>
-        (lar.loanType == 1) &&
-          (lar.propertyType == 1 || lar.propertyType == 2) &&
-          (lar.purpose == 1)
-      }
-
-    val larsWithIncome = lars.filter(lar => lar.income != "NA")
-
-    val msa = msaReport(fipsCode.toString)
-
-    val incomeIntervals = calculateMedianIncomeIntervals(fipsCode)
-    val applicantIncomesF = applicantIncomesWithBorrowerCharacteristics(larsWithIncome, incomeIntervals, dispositions)
-
-    val yearF = calculateYear(larSource)
-    val totalF = calculateDispositions(lars, dispositions)
-
-    for {
-      applicantIncomes <- applicantIncomesF
-      year <- yearF
-      total <- totalF
-    } yield {
-      A52(
-        year,
-        msa,
-        applicantIncomes,
-        total
-      )
-    }
-  }
+  ): Future[A5X] = A5X.generate("A52", larSource, fipsCode, filters)
 }

publication/src/main/scala/hmda/publication/reports/aggregate/A53.scala

@@ -0,0 +1,22 @@
+package hmda.publication.reports.aggregate
+
+import akka.NotUsed
+import akka.stream.scaladsl.Source
+import hmda.publication.reports._
+import hmda.query.model.filing.LoanApplicationRegisterQuery
+
+import scala.concurrent.Future
+
+object A53 {
+
+  def filters(lar: LoanApplicationRegisterQuery): Boolean = {
+    (lar.propertyType == 1 || lar.propertyType == 2) &&
+      (lar.purpose == 3)
+  }
+
+  def generate[ec: EC, mat: MAT, as: AS](
+    larSource: Source[LoanApplicationRegisterQuery, NotUsed],
+    fipsCode: Int
+  ): Future[A5X] = A5X.generate("A53", larSource, fipsCode, filters)
+
+}

publication/src/main/scala/hmda/publication/reports/aggregate/A5X.scala

@@ -0,0 +1,65 @@
+package hmda.publication.reports.aggregate
+
+import akka.NotUsed
+import akka.stream.scaladsl.Source
+import hmda.model.publication.reports.{ ApplicantIncome, Disposition, MSAReport }
+import hmda.publication.reports._
+import hmda.publication.reports.util.ReportUtil._
+import hmda.publication.reports.util.ReportsMetaDataLookup
+import hmda.query.model.filing.LoanApplicationRegisterQuery
+
+import scala.concurrent.Future
+
+case class A5X(
+  year: Int,
+  msa: MSAReport,
+  applicantIncomes: List[ApplicantIncome],
+  total: List[Disposition],
+  table: String,
+  description: String,
+  reportDate: String = formattedCurrentDate
+) extends AggregateReport
+
+object A5X {
+  def generate[ec: EC, mat: MAT, as: AS](
+    reportId: String,
+    larSource: Source[LoanApplicationRegisterQuery, NotUsed],
+    fipsCode: Int,
+    filters: LoanApplicationRegisterQuery => Boolean
+  ): Future[A5X] = {
+
+    val metaData = ReportsMetaDataLookup.values(reportId)
+    val dispositions = metaData.dispositions
+
+    val lars = larSource
+      .filter(lar => lar.msa != "NA")
+      .filter(lar => lar.msa.toInt == fipsCode)
+      .filter(filters)
+
+    val larsWithIncome = lars.filter(lar => lar.income != "NA")
+
+    val msa = msaReport(fipsCode.toString)
+
+    val incomeIntervals = calculateMedianIncomeIntervals(fipsCode)
+    val applicantIncomesF = applicantIncomesWithBorrowerCharacteristics(larsWithIncome, incomeIntervals, dispositions)
+
+    val yearF = calculateYear(larSource)
+    val totalF = calculateDispositions(lars, dispositions)
+
+    for {
+      applicantIncomes <- applicantIncomesF
+      year <- yearF
+      total <- totalF
+    } yield {
+
+      A5X(
+        year,
+        msa,
+        applicantIncomes,
+        total,
+        metaData.reportTable,
+        metaData.description
+      )
+    }
+  }
+}

publication/src/main/scala/hmda/publication/reports/aggregate/AggregateReport.scala

@@ -9,6 +9,7 @@ trait AggregateReport {
   val description: String
   val year: Int
   val msa: MSAReport
+  val reportDate: String
 
   val reportType: ReportTypeEnum = Aggregate
 }

publication/src/main/scala/hmda/publication/reports/aggregate/AggregateReports.scala

@@ -5,7 +5,7 @@ import akka.stream.ActorMaterializer
 import akka.util.Timeout
 import com.typesafe.config.ConfigFactory
 import hmda.query.repository.filing.FilingCassandraRepository
-import hmda.publication.reports.protocol.aggregate.A52Protocol._
+import hmda.publication.reports.protocol.aggregate.A5XProtocol._
 
 import scala.concurrent.Future
 import scala.concurrent.duration._
@@ -27,5 +27,7 @@ class AggregateReports(val sys: ActorSystem, val mat: ActorMaterializer) extends
     a52F.map { a52 =>
       println(a52.toJson.prettyPrint)
     }
+
+    // A53.generate(larSource, fipsCode)
   }
 }

publication/src/main/scala/hmda/publication/reports/national/N52.scala

@@ -5,7 +5,7 @@ import akka.stream.scaladsl.Source
 import hmda.census.model.MsaIncomeLookup
 import hmda.model.publication.reports._
 import hmda.publication.reports._
-import hmda.publication.reports.aggregate.A52
+import hmda.publication.reports.aggregate.{ A52, A5X }
 import hmda.publication.reports.util.ReportUtil._
 import hmda.publication.reports.util.ReportsMetaDataLookup
 import hmda.query.model.filing.LoanApplicationRegisterQuery
@@ -20,11 +20,11 @@ case class N52(
     table: String = N52.metaData.reportTable,
     description: String = N52.metaData.description
 ) extends NationalAggregateReport {
-  def +(a52: A52): N52 = {
+  def +(a5X: A5X): N52 = {
     if (applicantIncomes.isEmpty) {
-      N52(a52.year, a52.reportDate, a52.applicantIncomes, a52.total)
+      N52(a5X.year, a5X.reportDate, a5X.applicantIncomes, a5X.total)
     } else {
-      val combinedIncomes = a52.applicantIncomes.map(income => {
+      val combinedIncomes = a5X.applicantIncomes.map(income => {
         val newC = applicantIncomes.find(i => i.applicantIncome == income.applicantIncome).get.borrowerCharacteristics
         val originalC = income.borrowerCharacteristics
 
@@ -43,7 +43,7 @@ case class N52(
         ApplicantIncome(income.applicantIncome, List(cR, cE, cM))
       })
 
-      val combinedDispositions = a52.total.map(disposition => {
+      val combinedDispositions = a5X.total.map(disposition => {
         val originalDisposition = total.find(d => d.disposition == disposition.disposition).get
         disposition + originalDisposition
       })

publication/src/main/scala/hmda/publication/reports/protocol/aggregate/A52Protocol.scala renamed to publication/src/main/scala/hmda/publication/reports/protocol/aggregate/A5XProtocol.scala

@@ -2,20 +2,20 @@ package hmda.publication.reports.protocol.aggregate
 
 import hmda.model.publication.reports.{ ApplicantIncome, Disposition, MSAReport }
 import hmda.model.publication.reports.ReportTypeEnum._
-import hmda.publication.reports.aggregate.A52
+import hmda.publication.reports.aggregate.A5X
 import hmda.publication.reports.protocol.{ ApplicantIncomeProtocol, MSAReportProtocol, ReportTypeEnumProtocol }
 import spray.json._
 
-object A52Protocol
+object A5XProtocol
     extends DefaultJsonProtocol
     with ReportTypeEnumProtocol
     with MSAReportProtocol
     with ApplicantIncomeProtocol {
 
-  implicit object A52Format extends RootJsonFormat[A52] {
-    override def write(obj: A52): JsValue = {
+  implicit object A5XFormat extends RootJsonFormat[A5X] {
+    override def write(obj: A5X): JsValue = {
       JsObject(
-        "table" -> JsString("5-2"),
+        "table" -> JsString(obj.table),
         "type" -> JsString(Aggregate.toString),
         "desc" -> JsString(obj.description),
         "year" -> JsNumber(obj.year),
@@ -26,7 +26,7 @@ object A52Protocol
       )
     }
 
-    override def read(json: JsValue): A52 = json.asJsObject.getFields(
+    override def read(json: JsValue): A5X = json.asJsObject.getFields(
       "table",
       "type",
       "desc",
@@ -37,11 +37,13 @@ object A52Protocol
       "total"
     ) match {
         case Seq(table, reportType, description, year, reportDate, msa, applicantIncomes, total) =>
-          A52(
+          A5X(
             year.convertTo[Int],
             msa.convertTo[MSAReport],
             applicantIncomes.convertTo[List[ApplicantIncome]],
             total.convertTo[List[Disposition]],
+            table.convertTo[String],
+            description.convertTo[String],
             reportDate.convertTo[String]
           )
       }

publication/src/test/scala/hmda/publication/reports/aggregate/A52ProtocolSpec.scala

@@ -0,0 +1,65 @@
+package hmda.publication.reports.aggregate
+
+import akka.NotUsed
+import akka.actor.ActorSystem
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.Source
+import hmda.model.fi.lar.{ LarGenerators, LoanApplicationRegister }
+import hmda.model.publication.reports.ActionTakenTypeEnum._
+import hmda.model.publication.reports.ApplicantIncomeEnum.LessThan50PercentOfMSAMedian
+import hmda.model.publication.reports.{ EthnicityBorrowerCharacteristic, MSAReport, MinorityStatusBorrowerCharacteristic, RaceBorrowerCharacteristic }
+import hmda.query.model.filing.LoanApplicationRegisterQuery
+import hmda.query.repository.filing.LarConverter._
+import org.scalacheck.Gen
+import org.scalatest.{ AsyncWordSpec, BeforeAndAfterAll, MustMatchers }
+
+class A53Spec extends AsyncWordSpec with MustMatchers with LarGenerators with BeforeAndAfterAll {
+
+  implicit val system = ActorSystem()
+  implicit val ec = system.dispatcher
+  implicit val materializer = ActorMaterializer()
+
+  override def afterAll(): Unit = {
+    super.afterAll()
+    system.terminate()
+  }
+
+  val fips = 18700 //Corvallis, OR
+  def propType = Gen.oneOf(1, 2).sample.get
+
+  val lars = lar100ListGen.sample.get.map { lar: LoanApplicationRegister =>
+    val geo = lar.geography.copy(msa = fips.toString)
+    val loan = lar.loan.copy(propertyType = propType, purpose = 3)
+    lar.copy(geography = geo, loan = loan)
+  }
+
+  val source: Source[LoanApplicationRegisterQuery, NotUsed] = Source
+    .fromIterator(() => lars.toIterator)
+    .map(lar => toLoanApplicationRegisterQuery(lar))
+
+  val expectedDispositions = List(ApplicationReceived, LoansOriginated, ApprovedButNotAccepted, ApplicationsDenied, ApplicationsWithdrawn, ClosedForIncompleteness)
+
+  "Generate an Aggregate 5-3 report" in {
+    A53.generate(source, fips).map { result =>
+
+      result.msa mustBe MSAReport("18700", "Corvallis, OR", "OR", "Oregon")
+      result.table mustBe "5-3"
+      result.applicantIncomes.size mustBe 5
+
+      val lowestIncome = result.applicantIncomes.head
+      lowestIncome.applicantIncome mustBe LessThan50PercentOfMSAMedian
+
+      val races = lowestIncome.borrowerCharacteristics.head.asInstanceOf[RaceBorrowerCharacteristic].races
+      races.size mustBe 8
+
+      val ethnicities = lowestIncome.borrowerCharacteristics(1).asInstanceOf[EthnicityBorrowerCharacteristic].ethnicities
+      ethnicities.size mustBe 4
+
+      val minorityStatuses = lowestIncome.borrowerCharacteristics(2).asInstanceOf[MinorityStatusBorrowerCharacteristic].minoritystatus
+      minorityStatuses.size mustBe 2
+
+      races.head.dispositions.map(_.disposition) mustBe expectedDispositions
+    }
+  }
+
+}