Skip to content

Commit d51b752

Browse files
author
Nick Grippin
authored
Merge pull request cfpb#1225 from jmarin/check-digit
Check digit
2 parents a5e63b3 + 1155744 commit d51b752

File tree

9 files changed

+479
-12
lines changed

9 files changed

+479
-12
lines changed

‎Documents/public-api.md

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,151 @@ This documenatation describes de public HMDA Platform HTTP API
8484
For a definition of these fields, please consult the [HMDA Filing Instructions Guide](http://www.consumerfinance.gov/data-research/hmda/static/for-filers/2017/2017-HMDA-FIG.pdf).
8585
Please note that the Modified LAR does not include the fields `Loan Application Number`, `Date Application Received` or `Date of Action` described in HMDA Filing Instructions Guide.
8686

87+
## Check Digit
8788

89+
### Check digit generation
90+
91+
* `/uli/checkDigit`
92+
93+
* `POST` - Calculates check digit and full ULI from a loan id.
94+
95+
Example payload, in `JSON` format:
96+
97+
```json
98+
{
99+
"loanId": "10Bx939c5543TqA1144M999143X"
100+
}
101+
```
102+
103+
Example response:
104+
105+
```json
106+
{
107+
"loanId": "10Cx939c5543TqA1144M999143X",
108+
"checkDigit": 10,
109+
"uli": "10Cx939c5543TqA1144M999143X10"
110+
}
111+
```
112+
113+
A file with a list of Loan Ids can also be uploaded to this endpoint for batch check digit generation.
114+
115+
Example file contents:
116+
117+
```
118+
10Cx939c5543TqA1144M999143X
119+
10Bx939c5543TqA1144M999143X
120+
```
121+
122+
Example response in `JSON` format:
123+
124+
```json
125+
{
126+
"loanIds": [
127+
{
128+
"loanId": "10Bx939c5543TqA1144M999143X",
129+
"checkDigit": 38,
130+
"uli": "10Bx939c5543TqA1144M999143X38"
131+
},
132+
{
133+
"loanId": "10Cx939c5543TqA1144M999143X",
134+
"checkDigit": 10,
135+
"uli": "10Cx939c5543TqA1144M999143X10"
136+
}
137+
]
138+
}
139+
```
140+
141+
* `/uli/checkDigit/csv`
142+
143+
* `POST` - calculates check digits for loan ids submitted as a file
144+
145+
Example file contents:
146+
147+
```
148+
10Cx939c5543TqA1144M999143X
149+
10Bx939c5543TqA1144M999143X
150+
```
151+
152+
Example response in `CSV` format:
153+
154+
```csv
155+
loanId,checkDigit,uli
156+
10Bx939c5543TqA1144M999143X,38,10Bx939c5543TqA1144M999143X38
157+
10Cx939c5543TqA1144M999143X,10,10Cx939c5543TqA1144M999143X10
158+
```
159+
160+
### ULI Validation
161+
162+
* `/uli/validate`
163+
164+
* `POST` - Validates a ULI (correct check digit)
165+
166+
Example payload, in `JSON` format:
167+
168+
```json
169+
{
170+
"uli": "10Bx939c5543TqA1144M999143X38"
171+
}
172+
```
173+
174+
Example response:
175+
176+
```json
177+
{
178+
"isValid": true
179+
}
180+
```
181+
182+
A file with a list of ULIs can also be uploaded to this endpoint for batch ULI validation.
183+
184+
Example file contents:
185+
186+
```
187+
10Cx939c5543TqA1144M999143X10
188+
10Bx939c5543TqA1144M999143X38
189+
10Bx939c5543TqA1144M999133X38
190+
```
191+
192+
Example response in `JSON` format:
193+
194+
```json
195+
{
196+
"ulis": [
197+
{
198+
"uli": "10Cx939c5543TqA1144M999143X10",
199+
"isValid": true
200+
},
201+
{
202+
"uli": "10Bx939c5543TqA1144M999143X38",
203+
"isValid": true
204+
},
205+
{
206+
"uli": "10Bx939c5543TqA1144M999133X38",
207+
"isValid": false
208+
}
209+
]
210+
}
211+
```
212+
213+
* `/uli/validate/csv`
214+
215+
* `POST` - Batch validation of ULIs
216+
217+
Example file contents:
218+
219+
```
220+
10Cx939c5543TqA1144M999143X10
221+
10Bx939c5543TqA1144M999143X38
222+
10Bx939c5543TqA1144M999133X38
223+
```
224+
225+
Example response in `CSV` format:
226+
227+
```csv
228+
uli,isValid
229+
10Cx939c5543TqA1144M999143X10,true
230+
10Bx939c5543TqA1144M999143X38,true
231+
10Bx939c5543TqA1144M999133X38,false
232+
```
88233

89234

‎api-model/src/main/scala/hmda/api/model/public/ULIModel.scala

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package hmda.api.model.public
2+
3+
object ULIModel {
4+
5+
case class Loan(loanId: String)
6+
case class ULI(loanId: String, checkDigit: Int, uli: String) {
7+
def toCSV: String = s"$loanId,$checkDigit,$uli"
8+
}
9+
case class LoanCheckDigitResponse(loanIds: Seq[ULI])
10+
case class ULICheck(uli: String)
11+
case class ULIValidated(isValid: Boolean)
12+
case class ULIBatchValidated(uli: String, isValid: Boolean) {
13+
def toCSV: String = s"$uli,$isValid"
14+
}
15+
case class ULIBatchValidatedResponse(ulis: Seq[ULIBatchValidated])
16+
}

‎api-model/src/main/scala/hmda/api/protocol/public/ULIProtocol.scala

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
package hmda.api.protocol.public
2+
3+
import hmda.api.model.public.ULIModel._
4+
import spray.json.DefaultJsonProtocol
5+
6+
trait ULIProtocol extends DefaultJsonProtocol {
7+
8+
implicit val loanFormat = jsonFormat1(Loan.apply)
9+
implicit val uliFormat = jsonFormat3(ULI.apply)
10+
implicit val loanCheckDigitResponse = jsonFormat1(LoanCheckDigitResponse.apply)
11+
implicit val uliCheckFormat = jsonFormat1(ULICheck.apply)
12+
implicit val uliValidatedFormat = jsonFormat1(ULIValidated.apply)
13+
implicit val uliBatchValidatedFormat = jsonFormat2(ULIBatchValidated.apply)
14+
implicit val uliBatchValidatedResponseFormat = jsonFormat1(ULIBatchValidatedResponse.apply)
15+
16+
}

‎api/src/main/scala/hmda/api/http/public/PublicHttpApi.scala

Lines changed: 115 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,24 @@ package hmda.api.http.public
33
import akka.actor.ActorSystem
44
import akka.event.LoggingAdapter
55
import akka.stream.ActorMaterializer
6-
import akka.util.Timeout
6+
import akka.util.{ ByteString, Timeout }
77
import hmda.api.http.HmdaCustomDirectives
88
import akka.http.scaladsl.server.Directives._
9+
import hmda.api.model.public.ULIModel._
10+
import hmda.validation.engine.lar.ULI._
11+
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
12+
import akka.http.scaladsl.marshalling.ToResponseMarshallable
13+
import akka.http.scaladsl.model.{ HttpCharsets, HttpEntity, StatusCodes }
14+
import akka.http.scaladsl.model.MediaTypes.`text/csv`
15+
import akka.stream.scaladsl.{ Sink, Source }
16+
import hmda.api.protocol.processing.ApiErrorProtocol
17+
import hmda.api.protocol.public.ULIProtocol
18+
import hmda.api.util.FlowUtils
919

1020
import scala.concurrent.ExecutionContext
21+
import scala.util.{ Failure, Success }
1122

12-
trait PublicHttpApi extends PublicLarHttpApi with HmdaCustomDirectives {
23+
trait PublicHttpApi extends PublicLarHttpApi with HmdaCustomDirectives with ApiErrorProtocol with ULIProtocol with FlowUtils {
1324
implicit val system: ActorSystem
1425
implicit val materializer: ActorMaterializer
1526
implicit val timeout: Timeout
@@ -22,7 +33,108 @@ trait PublicHttpApi extends PublicLarHttpApi with HmdaCustomDirectives {
2233
encodeResponse {
2334
pathPrefix("institutions" / Segment) { instId =>
2435
modifiedLar(instId)
25-
}
36+
} ~
37+
pathPrefix("uli") {
38+
path("checkDigit") {
39+
timedPost { _ =>
40+
entity(as[Loan]) { loan =>
41+
val loanId = loan.loanId
42+
val check = checkDigit(loanId)
43+
val uli = ULI(loanId, check.toInt, loanId + check)
44+
complete(ToResponseMarshallable(uli))
45+
} ~
46+
fileUpload("file") {
47+
case (_, byteSource) =>
48+
val checkDigitF = processLoanIdFile(byteSource).runWith(Sink.seq)
49+
onComplete(checkDigitF) {
50+
case Success(checkDigits) => {
51+
complete(ToResponseMarshallable(LoanCheckDigitResponse(checkDigits)))
52+
}
53+
case Failure(error) =>
54+
log.error(error.getLocalizedMessage)
55+
complete(ToResponseMarshallable(StatusCodes.InternalServerError))
56+
}
57+
case _ =>
58+
complete(ToResponseMarshallable(StatusCodes.BadRequest))
59+
}
60+
}
61+
} ~
62+
path("checkDigit" / "csv") {
63+
timedPost { _ =>
64+
fileUpload("file") {
65+
case (_, byteSource) =>
66+
val headerSource = Source.fromIterator(() => List("loanId,checkDigit,uli\n").toIterator)
67+
val checkDigit = processLoanIdFile(byteSource)
68+
.map(l => l.toCSV)
69+
.map(l => l + "\n")
70+
.map(s => ByteString(s))
71+
72+
val csv = headerSource.map(s => ByteString(s)).concat(checkDigit)
73+
complete(HttpEntity.Chunked.fromData(`text/csv`.toContentType(HttpCharsets.`UTF-8`), csv))
74+
75+
case _ =>
76+
complete(ToResponseMarshallable(StatusCodes.BadRequest))
77+
}
78+
}
79+
} ~
80+
path("validate") {
81+
timedPost { _ =>
82+
entity(as[ULICheck]) { uc =>
83+
val uli = uc.uli
84+
val isValid = validateULI(uli)
85+
val validated = ULIValidated(isValid)
86+
complete(ToResponseMarshallable(validated))
87+
} ~
88+
fileUpload("file") {
89+
case (_, byteSource) =>
90+
val validatedF = processUliFile(byteSource).runWith(Sink.seq)
91+
onComplete(validatedF) {
92+
case Success(validated) =>
93+
complete(ToResponseMarshallable(ULIBatchValidatedResponse(validated)))
94+
case Failure(error) =>
95+
log.error(error.getLocalizedMessage)
96+
complete(ToResponseMarshallable(StatusCodes.InternalServerError))
97+
}
98+
99+
case _ =>
100+
complete(ToResponseMarshallable(StatusCodes.BadRequest))
101+
}
102+
}
103+
} ~
104+
path("validate" / "csv") {
105+
timedPost { _ =>
106+
fileUpload("file") {
107+
case (_, byteSource) =>
108+
val headerSource = Source.fromIterator(() => List("uli,isValid\n").toIterator)
109+
val validated = processUliFile(byteSource)
110+
.map(u => u.toCSV)
111+
.map(l => l + "\n")
112+
.map(s => ByteString(s))
113+
114+
val csv = headerSource.map(s => ByteString(s)).concat(validated)
115+
complete(HttpEntity.Chunked.fromData(`text/csv`.toContentType(HttpCharsets.`UTF-8`), csv))
116+
117+
case _ =>
118+
complete(ToResponseMarshallable(StatusCodes.BadRequest))
119+
}
120+
}
121+
}
122+
}
26123
}
27124
}
125+
126+
private def processLoanIdFile(byteSource: Source[ByteString, Any]) = {
127+
byteSource
128+
.via(framing)
129+
.map(_.utf8String)
130+
.map(loanId => ULI(loanId, checkDigit(loanId).toInt, loanId + checkDigit(loanId)))
131+
}
132+
133+
private def processUliFile(byteSource: Source[ByteString, Any]) = {
134+
byteSource
135+
.via(framing)
136+
.map(_.utf8String)
137+
.map(uli => (uli, validateULI(uli)))
138+
.map(validated => ULIBatchValidated(validated._1, validated._2))
139+
}
28140
}

‎api/src/test/scala/hmda/api/http/FileUploadUtils.scala

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
package hmda.api.http
2+
3+
import akka.http.scaladsl.model.{ ContentTypes, HttpEntity, Multipart }
4+
5+
trait FileUploadUtils {
6+
def multiPartFile(contents: String, fileName: String) =
7+
Multipart.FormData(Multipart.FormData.BodyPart.Strict(
8+
"file",
9+
HttpEntity(ContentTypes.`text/plain(UTF-8)`, contents),
10+
Map("filename" -> fileName)
11+
))
12+
}

‎api/src/test/scala/hmda/api/http/InstitutionHttpSpec.scala

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ import scala.concurrent.duration._
2323
import akka.pattern.ask
2424
import scala.concurrent._
2525

26-
trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with RequestHeaderUtils with InstitutionsHttpApi with ScalatestRouteTest { suite: Suite =>
26+
trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with RequestHeaderUtils with InstitutionsHttpApi with FileUploadUtils with ScalatestRouteTest { suite: Suite =>
2727
val configuration: Config = ConfigFactory.load()
2828

2929
val validationStats = ValidationStats.createValidationStats(system)
@@ -55,11 +55,4 @@ trait InstitutionHttpSpec extends MustMatchers with BeforeAndAfterAll with Reque
5555
FileUtils.deleteRecursively(snapshotStore)
5656
}
5757

58-
def multiPartFile(contents: String, fileName: String) =
59-
Multipart.FormData(Multipart.FormData.BodyPart.Strict(
60-
"file",
61-
HttpEntity(ContentTypes.`text/plain(UTF-8)`, contents),
62-
Map("filename" -> fileName)
63-
))
64-
6558
}

0 commit comments

Comments
 (0)