@@ -3,8 +3,8 @@ package hmda.loader.lar
3
3
import java .io .File
4
4
import java .time .Instant
5
5
6
- import akka .actor .{ ActorPath , ActorRef , ActorSystem }
7
6
import akka .pattern .ask
7
+ import akka .actor .{ ActorPath , ActorRef , ActorSystem }
8
8
import akka .cluster .client .{ ClusterClient , ClusterClientSettings }
9
9
import akka .stream .{ ActorMaterializer , IOResult }
10
10
import akka .stream .scaladsl .{ FileIO , Sink , Source }
@@ -13,8 +13,8 @@ import hmda.api.util.FlowUtils
13
13
import hmda .model .fi .{ Created , Filing , Submission }
14
14
import hmda .persistence .HmdaSupervisor .{ FindFilings , FindHmdaFiling , FindProcessingActor , FindSubmissions }
15
15
import hmda .persistence .institutions .FilingPersistence .CreateFiling
16
- import hmda .persistence .institutions .{ FilingPersistence , SubmissionPersistence }
17
16
import hmda .persistence .institutions .SubmissionPersistence .CreateSubmission
17
+ import hmda .persistence .institutions .{ FilingPersistence , SubmissionPersistence }
18
18
import hmda .persistence .processing .HmdaRawFile .AddLine
19
19
import hmda .persistence .processing .ProcessingMessages .{ CompleteUpload , Persisted , StartUpload }
20
20
import hmda .persistence .processing .SubmissionManager
@@ -23,15 +23,16 @@ import hmda.persistence.processing.SubmissionManager.AddFileName
23
23
import hmda .query .HmdaQuerySupervisor .FindHmdaFilingView
24
24
import org .slf4j .LoggerFactory
25
25
26
- import scala .concurrent .{ Await , Future }
27
26
import scala .concurrent .duration ._
27
+ import scala .concurrent .{ ExecutionContext , Future }
28
28
import scala .util .{ Failure , Success }
29
29
30
- object HmdaLarLoader extends FlowUtils {
30
+ object HmdaBatchLarLoader extends FlowUtils {
31
+
32
+ override implicit val system : ActorSystem = ActorSystem (" hmda-cluster-client" )
33
+ override implicit val materializer : ActorMaterializer = ActorMaterializer ()
34
+ override implicit val ec : ExecutionContext = system.dispatcher
31
35
32
- override implicit val system = ActorSystem (" hmda-cluster-client" )
33
- override implicit val materializer = ActorMaterializer ()
34
- override implicit val ec = system.dispatcher
35
36
val hmdaClusterName = config.getString(" hmda.clusterName" )
36
37
val hmdaClusterIP = config.getString(" hmda.lar.host" )
37
38
val hmdaClusterPort = config.getInt(" hmda.lar.port" )
@@ -53,21 +54,21 @@ object HmdaLarLoader extends FlowUtils {
53
54
54
55
def main (args : Array [String ]): Unit = {
55
56
56
- if (args.length < 1 ) {
57
- exitSys(log, " No File argument provided " , 1 )
57
+ if (args.length < 2 ) {
58
+ exitSys(log, " Please provide a directory containing files with LAR data and period to process " , 1 )
58
59
}
59
60
60
- val file = new File (args(0 ))
61
- if (! file.exists() || ! file.isFile ) {
62
- exitSys(log, " File does not exist " , 2 )
61
+ val path = new File (args(0 ))
62
+ if (! path.isDirectory ) {
63
+ exitSys(log, " Argument must be the full path to a directory containing files with LAR data " , 2 )
63
64
}
64
65
65
- val fileName = file.getName
66
- val parts = fileName.split( " _ " )
67
- val institutionId = parts.head
68
- val finalPart = parts.tail.head
69
- val period = finalPart.substring( 0 , finalPart.indexOf( " . " ))
70
- processLars(file, fileName, institutionId, period)
66
+ val period = args( 1 )
67
+
68
+ val fileList = path.listFiles().toSet
69
+ val fileNames = fileList.map(file => file.getName).filter(name => name.endsWith( " .txt " ))
70
+
71
+ fileNames.foreach(fileName => processLars(new File ( s " $path / $fileName " ), s " $ fileName" , s " ${fileName.substring( 0 , fileName.indexOf( " . " ))} " , period) )
71
72
72
73
}
73
74
0 commit comments