Add batch ULI validation tests

Author: jmarin

api/src/main/scala/hmda/api/http/public/PublicHttpApi.scala

@@ -55,7 +55,6 @@ trait PublicHttpApi extends PublicLarHttpApi with HmdaCustomDirectives with ApiE
                     fileUpload("file") {
                       case (_, byteSource) =>
                         val validatedF = processUliFile(byteSource).runWith(Sink.seq)
-
                         onComplete(validatedF) {
                           case Success(validated) =>
                             complete(ToResponseMarshallable(ULIBatchValidatedResponse(validated)))

api/src/test/scala/hmda/api/http/FileUploadUtils.scala

@@ -0,0 +1,12 @@
+package hmda.api.http
+
+import akka.http.scaladsl.model.{ ContentTypes, HttpEntity, Multipart }
+
+trait FileUploadUtils {
+  def multiPartFile(contents: String, fileName: String) =
+    Multipart.FormData(Multipart.FormData.BodyPart.Strict(
+      "file",
+      HttpEntity(ContentTypes.`text/plain(UTF-8)`, contents),
+      Map("filename" -> fileName)
+    ))
+}

api/src/test/scala/hmda/api/http/InstitutionHttpSpec.scala

@@ -23,7 +23,7 @@ import scala.concurrent.duration._
 import akka.pattern.ask
 import scala.concurrent._
 
-trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with RequestHeaderUtils with InstitutionsHttpApi with ScalatestRouteTest { suite: Suite =>
+trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with RequestHeaderUtils with InstitutionsHttpApi with FileUploadUtils with ScalatestRouteTest { suite: Suite =>
   val configuration: Config = ConfigFactory.load()
 
   val validationStats = ValidationStats.createValidationStats(system)
@@ -55,11 +55,4 @@ trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with Reque
     FileUtils.deleteRecursively(snapshotStore)
   }
 
-  def multiPartFile(contents: String, fileName: String) =
-    Multipart.FormData(Multipart.FormData.BodyPart.Strict(
-      "file",
-      HttpEntity(ContentTypes.`text/plain(UTF-8)`, contents),
-      Map("filename" -> fileName)
-    ))
-
 }

api/src/test/scala/hmda/api/http/public/PublicHttpApiSpec.scala

@@ -5,17 +5,18 @@ import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.testkit.ScalatestRouteTest
 import akka.util.Timeout
 import hmda.api.RequestHeaderUtils
-import hmda.api.model.public.ULIModel.{ Loan, ULI, ULICheck, ULIValidated }
+import hmda.api.model.public.ULIModel._
 import hmda.api.protocol.public.ULIProtocol
 import hmda.model.fi.lar.LarGenerators
 import hmda.query.repository.filing.LarConverter._
 import org.scalatest.{ BeforeAndAfterAll, MustMatchers, WordSpec }
 import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
+import hmda.api.http.FileUploadUtils
 
 import scala.concurrent.duration._
 
 class PublicHttpApiSpec extends WordSpec with MustMatchers with BeforeAndAfterAll
-    with ScalatestRouteTest with RequestHeaderUtils with PublicHttpApi with LarGenerators with ULIProtocol {
+    with ScalatestRouteTest with RequestHeaderUtils with PublicHttpApi with FileUploadUtils with LarGenerators with ULIProtocol {
 
   override val log: LoggingAdapter = NoLogging
   implicit val ec = system.dispatcher
@@ -27,6 +28,10 @@ class PublicHttpApiSpec extends WordSpec with MustMatchers with BeforeAndAfterAl
   val l1 = toLoanApplicationRegisterQuery(sampleLar).copy(period = p, institutionId = "0")
   val l2 = toLoanApplicationRegisterQuery(sampleLar).copy(period = p, institutionId = "0")
 
+  val uliTxt = "10Cx939c5543TqA1144M999143X10\n" +
+    "10Bx939c5543TqA1144M999143X38\n" +
+    "10Bx939c5543TqA1144M999133X38\n"
+
   "Modified LAR Http API" must {
     "return list of modified LARs in proper format" in {
       Get("/institutions/0/filings/2017/lar") ~> publicHttpRoutes ~> check {
@@ -39,6 +44,7 @@ class PublicHttpApiSpec extends WordSpec with MustMatchers with BeforeAndAfterAl
   }
 
   "ULI API" must {
+    val uliFile = multiPartFile(uliTxt, "ulis.txt")
     val loanId = "10Bx939c5543TqA1144M999143X"
     val checkDigit = 38
     val uli = "10Bx939c5543TqA1144M999143X" + checkDigit
@@ -55,6 +61,25 @@ class PublicHttpApiSpec extends WordSpec with MustMatchers with BeforeAndAfterAl
         responseAs[ULIValidated] mustBe ULIValidated(true)
       }
     }
+    "validate a file of ULIs and return csv" in {
+      Post("/uli/validate/csv", uliFile) ~> publicHttpRoutes ~> check {
+        status mustBe StatusCodes.OK
+        val csv = responseAs[String]
+        csv must include("10Cx939c5543TqA1144M999143X10,true")
+        csv must include("10Bx939c5543TqA1144M999143X38,true")
+        csv must include("10Bx939c5543TqA1144M999133X38,false")
+      }
+    }
+    "validate a file of ULIs" in {
+      Post("/uli/validate", uliFile) ~> publicHttpRoutes ~> check {
+        status mustBe StatusCodes.OK
+        responseAs[ULIBatchValidatedResponse].ulis mustBe Seq(
+          ULIBatchValidated("10Cx939c5543TqA1144M999143X10", true),
+          ULIBatchValidated("10Bx939c5543TqA1144M999143X38", true),
+          ULIBatchValidated("10Bx939c5543TqA1144M999133X38", false)
+        )
+      }
+    }
   }
 
 }

validation/src/main/scala/hmda/validation/engine/lar/ULI.scala

@@ -1,8 +1,5 @@
 package hmda.validation.engine.lar
 
-import akka.NotUsed
-import akka.stream.scaladsl.Source
-
 //See https://www.consumerfinance.gov/eregulations/1003-C/2015-26607_20180101#1003-C-1
 
 object ULI {
@@ -61,18 +58,8 @@ object ULI {
     loanId + checkDigit(loanId).toString()
   }
 
-  def checkDigitBatch(loanIdSource: Source[String, NotUsed]): Source[(String, BigInt), NotUsed] = {
-    loanIdSource
-      .map(loanId => (loanId, checkDigit(loanId)))
-  }
-
   def validateULI(uli: String): Boolean = {
     calculateMod(BigInt(convert(uli))) == 1
   }
 
-  def validateULIBatch(uliSource: Source[String, NotUsed]): Source[(String, Boolean), NotUsed] = {
-    uliSource
-      .map(uli => (uli, validateULI(uli)))
-  }
-
 }

validation/src/test/scala/hmda/validation/engine/lar/ULISpec.scala

@@ -34,23 +34,5 @@ class ULISpec extends AsyncWordSpec with MustMatchers with BeforeAndAfterAll {
       ULI.validateULI(validULI2) mustBe true
       ULI.validateULI(invalidULI) mustBe false
     }
-    "produce valid check digits in batch" in {
-      val loanIt = List(loan1, loan2).toIterator
-      val loanSource = Source.fromIterator(() => loanIt)
-      val checkDigitF = ULI.checkDigitBatch(loanSource).runWith(Sink.seq)
-      checkDigitF.map { checkDigit =>
-        checkDigit.head._2 mustBe 38
-        checkDigit.tail.head._2 mustBe 10
-      }
-    }
-    "Validate a list of ULIs" in {
-      val uliIt = List(validULI1, invalidULI).toIterator
-      val uliSource = Source.fromIterator(() => uliIt)
-      val validatedF = ULI.validateULIBatch(uliSource).runWith(Sink.seq)
-      validatedF.map { validated =>
-        validated.head._2 mustBe true
-        validated.tail.head._2 mustBe false
-      }
-    }
   }
 }