Skip to content

Commit 7d91fb4

Browse files
author
Nick Grippin
authored
Merge pull request cfpb#1129 from schbetsy/submission-filename
Include file name in Submissions API response
2 parents 9937968 + 1881244 commit 7d91fb4

File tree

27 files changed

+171
-126
lines changed

27 files changed

+171
-126
lines changed

‎Documents/API.md

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,8 @@ Example response:
141141
"message": "created",
142142
"description": "The filing period is now open and available to accept HMDA data."
143143
},
144+
"fileName": "bank1_hmda_2017.txt",
145+
"receipt": "",
144146
"start": 1483287071000,
145147
"end": 0
146148
},
@@ -155,6 +157,8 @@ Example response:
155157
"message": "created",
156158
"description": "The filing period is now open and available to accept HMDA data."
157159
},
160+
"fileName": "bank1_lars_2017.txt",
161+
"receipt": "",
158162
"start": 1483287071000,
159163
"end": 0
160164
},
@@ -169,6 +173,8 @@ Example response:
169173
"message": "created",
170174
"description": "The filing period is now open and available to accept HMDA data."
171175
},
176+
"fileName": "bank1_lars_2017.txt",
177+
"receipt": "",
172178
"start": 1483287071000,
173179
"end": 0
174180
}
@@ -196,6 +202,8 @@ Example response, with HTTP code 201:
196202
"message": "created",
197203
"description": "The filing period is now open and available to accept HMDA data."
198204
},
205+
"fileName": "bank1_lars_2017.txt",
206+
"receipt": "",
199207
"start": 1483287071000,
200208
"end": 0
201209
}
@@ -221,6 +229,8 @@ Example response:
221229
"message": "created",
222230
"description": "The filing period is now open and available to accept HMDA data."
223231
},
232+
"fileName": "bank1_lars_2017.txt",
233+
"receipt": "",
224234
"start": 1483287071000,
225235
"end": 1514736671000
226236
}
@@ -245,7 +255,11 @@ Example response:
245255
"code": 3,
246256
"message": "uploaded",
247257
"description": "The data have finished uploading and are ready to be analyzed."
248-
}
258+
},
259+
"fileName": "bank1_lars_2017.txt",
260+
"receipt": "",
261+
"start": 1505142349962,
262+
"end": 0
249263
}
250264
```
251265

@@ -262,7 +276,11 @@ Example response, with HTTP code 400:
262276
"code": -1,
263277
"message": "Submission 4848484 not available for upload",
264278
"description": "An error occurred during the process of submitting the data. Please re-upload your file."
265-
}
279+
},
280+
"fileName": "",
281+
"receipt": "",
282+
"start": 0,
283+
"end": 0
266284
}
267285
```
268286

‎api-model/src/main/scala/hmda/api/protocol/processing/SubmissionProtocol.scala

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ trait SubmissionProtocol extends ValidationResultProtocol {
4242
}
4343

4444
implicit val submissionIdProtocol = jsonFormat3(SubmissionId.apply)
45-
implicit val submissionFormat = jsonFormat5(Submission.apply)
45+
implicit val submissionFormat = jsonFormat6(Submission.apply)
4646
implicit val submissionsFormat = jsonFormat1(Submissions.apply)
4747
implicit val receiptFormat = jsonFormat3(Receipt.apply)
4848

‎api-model/src/test/scala/hmda/api/protocol/processing/SubmissionProtocolSpec.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ class SubmissionProtocolSpec extends PropSpec with PropertyChecks with MustMatch
3131
("description", JsString(s.status.description))
3232
)),
3333
("receipt", JsString(s.receipt)),
34+
("fileName", JsString(s.fileName)),
3435
("start", JsNumber(s.start)),
3536
("end", JsNumber(s.end))
3637
)

‎api/src/main/scala/hmda/api/http/institutions/UploadPaths.scala

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,10 @@ import hmda.model.fi.{ Created, Failed, Submission, SubmissionId, Uploaded }
2222
import hmda.persistence.HmdaSupervisor.{ FindHmdaFiling, FindProcessingActor, FindSubmissions }
2323
import hmda.persistence.institutions.SubmissionPersistence
2424
import hmda.persistence.institutions.SubmissionPersistence.GetSubmissionById
25-
import hmda.persistence.processing.HmdaRawFile.{ AddFileName, AddLine }
25+
import hmda.persistence.processing.HmdaRawFile.AddLine
2626
import hmda.persistence.processing.ProcessingMessages.{ CompleteUpload, Persisted, StartUpload }
2727
import hmda.persistence.processing.SubmissionManager
28+
import hmda.persistence.processing.SubmissionManager.AddFileName
2829
import hmda.query.HmdaQuerySupervisor.FindHmdaFilingView
2930
import hmda.query.projections.filing.HmdaFilingDBProjection.{ CreateSchema, DeleteLars }
3031

‎api/src/main/scala/hmda/api/http/institutions/submissions/SubmissionSummaryPaths.scala

Lines changed: 9 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -13,15 +13,15 @@ import hmda.api.http.ValidationErrorConverter
1313
import hmda.api.model.ErrorResponse
1414
import hmda.api.model.institutions.submissions.{ ContactSummary, FileSummary, RespondentSummary, SubmissionSummary }
1515
import hmda.api.protocol.processing.{ ApiErrorProtocol, EditResultsProtocol, InstitutionProtocol, SubmissionProtocol }
16-
import hmda.model.fi.SubmissionId
16+
import hmda.model.fi.{ Submission, SubmissionId }
1717
import hmda.model.fi.ts.TransmittalSheet
1818
import hmda.model.institution.Agency
19-
import hmda.persistence.HmdaSupervisor.FindProcessingActor
19+
import hmda.persistence.HmdaSupervisor.{ FindProcessingActor, FindSubmissions }
20+
import hmda.persistence.institutions.SubmissionPersistence
21+
import hmda.persistence.institutions.SubmissionPersistence.GetSubmissionById
2022
import hmda.persistence.messages.CommonMessages.GetState
21-
import hmda.persistence.processing.{ HmdaFileValidator, HmdaRawFile, SubmissionManager }
23+
import hmda.persistence.processing.HmdaFileValidator
2224
import hmda.persistence.processing.HmdaFileValidator._
23-
import hmda.persistence.processing.SubmissionManager.GetActorRef
24-
import hmda.persistence.processing.HmdaRawFile.HmdaRawFileState
2525

2626
import scala.concurrent.ExecutionContext
2727
import scala.util.{ Failure, Success, Try }
@@ -48,20 +48,15 @@ trait SubmissionSummaryPaths
4848
timedGet { uri =>
4949
completeVerified(supervisor, querySupervisor, institutionId, period, seqNr, uri) {
5050
val submissionId = SubmissionId(institutionId, period, seqNr)
51-
val submissionManagerF = (supervisor ? FindProcessingActor(SubmissionManager.name, submissionId)).mapTo[ActorRef]
5251
val validatorF = (supervisor ? FindProcessingActor(HmdaFileValidator.name, submissionId)).mapTo[ActorRef]
53-
val hmdaRawF = submissionManagerF.flatMap(actorRef => (actorRef ? GetActorRef(HmdaRawFile.name)).mapTo[ActorRef])
52+
val submissionPersistenceF = (supervisor ? FindSubmissions(SubmissionPersistence.name, submissionId.institutionId, submissionId.period)).mapTo[ActorRef]
5453

5554
val tsF = for {
5655
validator <- validatorF
57-
hmdaRaw <- hmdaRawF
58-
u <- (hmdaRaw ? GetState).mapTo[HmdaRawFileState]
56+
submissions <- submissionPersistenceF
5957
s <- (validator ? GetState).mapTo[HmdaFileValidationState]
60-
fileDetails = u.fileName
61-
larSize = s.lars.size
62-
ts = s.ts
63-
tsLarSummary = TsLarSummary(ts, larSize, fileDetails)
64-
} yield tsLarSummary
58+
sub <- (submissions ? GetSubmissionById(submissionId)).mapTo[Submission]
59+
} yield TsLarSummary(s.ts, s.lars.size, sub.fileName)
6560

6661
onComplete(tsF) {
6762
case Success(x) => x.ts match {

‎api/src/test/scala/hmda/api/http/institutions/submissions/SubmissionSummaryPathsSpec.scala

Lines changed: 21 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,51 @@
11
package hmda.api.http.institutions.submissions
22

3-
import akka.actor.ActorRef
4-
import akka.pattern.ask
3+
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
54
import akka.http.scaladsl.model.StatusCodes
65
import hmda.api.http.InstitutionHttpApiSpec
76
import hmda.api.model.institutions.submissions.{ ContactSummary, FileSummary, RespondentSummary, SubmissionSummary }
8-
import hmda.model.fi.SubmissionId
9-
import hmda.persistence.HmdaSupervisor.FindProcessingActor
10-
import hmda.persistence.processing.{ HmdaFileValidator, SubmissionManager }
11-
import org.scalatest.BeforeAndAfterAll
12-
import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
137
import hmda.api.model.ErrorResponse
14-
import hmda.parser.fi.lar.LarCsvParser
15-
16-
import scala.concurrent.Await
17-
import hmda.parser.fi.ts.TsCsvParser
18-
import hmda.persistence.processing.HmdaRawFile.AddFileName
8+
import org.scalatest.BeforeAndAfterAll
199

2010
class SubmissionSummaryPathsSpec extends InstitutionHttpApiSpec with BeforeAndAfterAll {
21-
import hmda.model.util.FITestData._
2211

2312
val institutionId = "0"
2413
val period = "2017"
2514
val seqNr = 1
26-
val submissionId = SubmissionId(institutionId, period, seqNr)
2715

28-
val fileName = "lars.dat"
29-
val lines = fiCSV.split("\n")
30-
val ts = TsCsvParser(lines(0)).right.get
31-
val lars = lines.tail.map(line => LarCsvParser(line).right.get)
16+
val csv = "1|externalTest0|3|201502221111|2017|35-0704860|10|Passes Bank|555 Passes Court|Passes City|CA|92130|Passes Bank Parent|555 Passes Court Parent|Passes City|CA|92130|Passes Person|555-555-5555|555-555-5555|pperson@passes.com\n" +
17+
"2|externalTest0|3|10164 |20170224|1|1|3|1|21|3|1|20170326|45460|18|153|0501.00|2|2|5| | | | |5| | | | |1|2|31|0| | | |NA |2|1\n" +
18+
"2|externalTest0|3|10174 |20170224|1|1|2|1|60|3|1|20170402|45460|18|153|0503.00|2|2|5| | | | |5| | | | |1|2|210|0| | | |NA |2|2\n" +
19+
"2|externalTest0|3|10370 |20170228|1|1|3|1|73|3|3|20170326|45460|18|153|0505.00|2|2|5| | | | |5| | | | |1|2|89|0|4| | |NA |2|1"
20+
21+
val fileName = "2017_lars_bank_1.txt"
22+
val file = multiPartFile(csv, fileName)
3223

3324
override def beforeAll(): Unit = {
3425
super.beforeAll()
35-
val validatorF = (supervisor ? FindProcessingActor(HmdaFileValidator.name, submissionId)).mapTo[ActorRef]
36-
val submissionManagerF = (supervisor ? FindProcessingActor(SubmissionManager.name, submissionId)).mapTo[ActorRef]
37-
val validator = Await.result(validatorF, duration)
38-
val submissionManager = Await.result(submissionManagerF, duration)
39-
validator ! ts
40-
lars.foreach(lar => validator ! lar)
41-
submissionManager ! AddFileName(fileName)
4226
}
4327

4428
"Submission Summary Paths" must {
29+
30+
"Set up: upload a file" in {
31+
postWithCfpbHeaders(s"/institutions/$institutionId/filings/$period/submissions/$seqNr", file) ~> institutionsRoutes(supervisor, querySupervisor, validationStats) ~> check {
32+
Thread.sleep(5000) // wait for the submission to complete validation
33+
status mustBe StatusCodes.Accepted
34+
}
35+
}
36+
4537
"return a 200" in {
4638
getWithCfpbHeaders(s"/institutions/$institutionId/filings/$period/submissions/$seqNr/summary") ~> institutionsRoutes(supervisor, querySupervisor, validationStats) ~> check {
47-
val contactSummary = ContactSummary(ts.contact.name, ts.contact.phone, ts.contact.email)
48-
val respondentSummary = RespondentSummary(ts.respondent.name, ts.respondent.id, ts.taxId, "cfpb", contactSummary)
49-
val fileSummary = FileSummary(fileName, "2013", lars.size)
39+
val contactSummary = ContactSummary("Passes Person", "555-555-5555", "pperson@passes.com")
40+
val respondentSummary = RespondentSummary("Passes Bank", "externalTest0", "35-0704860", "fdic", contactSummary)
41+
val fileSummary = FileSummary(name = fileName, year = "2017", totalLARS = 3)
5042
val submissionSummary = SubmissionSummary(respondentSummary, fileSummary)
5143

5244
status mustBe StatusCodes.OK
5345
responseAs[SubmissionSummary] mustBe submissionSummary
5446
}
5547
}
48+
5649
"return 404 for nonexistent institution" in {
5750
getWithCfpbHeaders(s"/institutions/xxxxx/filings/$period/submissions/$seqNr/summary") ~> institutionsRoutes(supervisor, querySupervisor, validationStats) ~> check {
5851
status mustBe StatusCodes.NotFound

‎model/shared/src/main/scala/hmda/model/fi/Submission.scala

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,5 +72,6 @@ case class Submission(
7272
status: SubmissionStatus = Created,
7373
start: Long = 0L,
7474
end: Long = 0L,
75+
fileName: String = "",
7576
receipt: String = ""
7677
)

‎persistence-model/src/main/protobuf/FileUpload.proto

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,3 @@ message LineAddedMessage {
77
int64 timestamp = 1;
88
string data = 2;
99
}
10-
11-
message FileNameAddedMessage {
12-
string fileName = 1;
13-
}
14-

‎persistence-model/src/main/protobuf/SubmissionEvents.proto

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ message SubmissionMessage {
2121
int64 start = 3;
2222
int64 end = 4;
2323
string receipt = 5;
24+
string fileName = 6;
2425
}
2526

2627
message SubmissionCreatedMessage {
@@ -30,4 +31,9 @@ message SubmissionCreatedMessage {
3031
message SubmissionStatusUpdatedMessage {
3132
SubmissionIdMessage id = 1;
3233
SubmissionStatusMessage status = 2;
34+
}
35+
36+
message SubmissionFileNameAddedMessage {
37+
SubmissionIdMessage id = 1;
38+
string fileName = 2;
3339
}

‎persistence-model/src/main/resources/application-dev.conf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,10 @@ akka {
2323
"hmda.persistence.messages.events.institutions.FilingEvents$FilingStatusUpdated" = filing
2424
"hmda.persistence.messages.events.institutions.SubmissionEvents$SubmissionCreated" = submission
2525
"hmda.persistence.messages.events.institutions.SubmissionEvents$SubmissionStatusUpdated" = submission
26+
"hmda.persistence.messages.events.institutions.SubmissionEvents$SubmissionFileNameAdded" = submission
2627
"hmda.persistence.messages.events.processing.CommonHmdaValidatorEvents$LarValidated" = hmdaValidator
2728
"hmda.persistence.messages.events.processing.CommonHmdaValidatorEvents$TsValidated" = hmdaValidator
2829
"hmda.persistence.messages.events.processing.FileUploadEvents$LineAdded" = upload
29-
"hmda.persistence.messages.events.processing.FileUploadEvents$FileNameAdded" = upload
3030
"hmda.persistence.messages.events.processing.HmdaFileParserEvents$TsParsed" = parser
3131
"hmda.persistence.messages.events.processing.HmdaFileParserEvents$TsParsedErrors" = parser
3232
"hmda.persistence.messages.events.processing.HmdaFileParserEvents$LarParsed" = parser

0 commit comments

Comments
 (0)