1
1
package hmda .loader .lar
2
2
3
3
import java .io .File
4
+ import java .time .Instant
4
5
5
- import akka .actor .{ActorPath , ActorRef , ActorSystem }
6
+ import akka .actor .{ ActorPath , ActorRef , ActorSystem }
6
7
import akka .pattern .ask
7
- import akka .cluster .client .{ClusterClient , ClusterClientSettings }
8
- import akka .stream .ActorMaterializer
9
- import akka .stream .scaladsl .{FileIO , Sink }
10
- import akka .util .Timeout
11
- import com .typesafe .config .ConfigFactory
8
+ import akka .cluster .client .{ ClusterClient , ClusterClientSettings }
9
+ import akka .stream .{ ActorMaterializer , IOResult }
10
+ import akka .stream .scaladsl .{ FileIO , Sink , Source }
11
+ import akka .util .{ ByteString , Timeout }
12
12
import hmda .api .util .FlowUtils
13
- import hmda .model .fi .{Created , Submission , SubmissionId }
14
- import hmda .persistence .HmdaSupervisor .{FindProcessingActor , FindSubmissions }
13
+ import hmda .model .fi .{ Created , Submission }
14
+ import hmda .persistence .HmdaSupervisor .{ FindHmdaFiling , FindProcessingActor , FindSubmissions }
15
15
import hmda .persistence .institutions .SubmissionPersistence
16
- import hmda .persistence .institutions .SubmissionPersistence .GetSubmissionById
16
+ import hmda .persistence .institutions .SubmissionPersistence .CreateSubmission
17
+ import hmda .persistence .processing .HmdaRawFile .AddLine
18
+ import hmda .persistence .processing .ProcessingMessages .{ CompleteUpload , Persisted , StartUpload }
17
19
import hmda .persistence .processing .SubmissionManager
20
+ import hmda .persistence .messages .CommonMessages ._
21
+ import hmda .persistence .processing .SubmissionManager .AddFileName
22
+ import hmda .query .HmdaQuerySupervisor .FindHmdaFilingView
18
23
import org .slf4j .LoggerFactory
19
24
25
+ import scala .concurrent .{ Await , Future }
20
26
import scala .concurrent .duration ._
21
- import scala .util .{Failure , Success }
27
+ import scala .util .{ Failure , Success }
22
28
23
29
object HmdaLarLoader extends FlowUtils {
24
30
@@ -29,6 +35,7 @@ object HmdaLarLoader extends FlowUtils {
29
35
val hmdaClusterIP = config.getString(" hmda.lar.host" )
30
36
val hmdaClusterPort = config.getInt(" hmda.lar.port" )
31
37
val actorTimeout = config.getInt(" hmda.actorTimeout" )
38
+ val flowParallelism = config.getInt(" hmda.lar.parallelism" )
32
39
33
40
implicit val timeout = Timeout (actorTimeout.seconds)
34
41
@@ -54,48 +61,44 @@ object HmdaLarLoader extends FlowUtils {
54
61
exitSys(log, " File does not exist" , 2 )
55
62
}
56
63
57
- val source = FileIO .fromPath(file.toPath)
58
-
59
- source.take(1 )
60
- .runWith(Sink .foreach(println))
61
-
62
-
63
-
64
-
65
-
66
- val institutionId = " institutionID"
67
- val period = " 2017"
68
-
69
- // processLars(institutionId, period)
64
+ val fileName = file.getName
65
+ val parts = fileName.split(" _" )
66
+ val institutionId = parts.head
67
+ val finalPart = parts.tail.head
68
+ val period = finalPart.substring(0 , finalPart.indexOf(" ." ))
69
+ processLars(file, fileName, institutionId, period)
70
70
71
71
}
72
72
73
- private def processLars (institutionId : String , period : String ) = {
74
- val submissionId = SubmissionId (institutionId, period)
75
-
76
- val message = FindProcessingActor (SubmissionManager .name, submissionId)
73
+ private def processLars (file : File , fileName : String , institutionId : String , period : String ) = {
74
+ val uploadTimestamp = Instant .now.toEpochMilli
75
+ val source = FileIO .fromPath(file.toPath)
77
76
78
- val fProcessingActor = (clusterClient ? ClusterClient .Send (" /user/supervisor/singleton" , message, localAffinity = true )).mapTo[ActorRef ]
79
77
val fSubmissionsActor = (clusterClient ? ClusterClient
80
- .Send (" /user/supervisor/singleton" ,
78
+ .Send (
79
+ " /user/supervisor/singleton" ,
81
80
FindSubmissions (SubmissionPersistence .name, institutionId, period),
82
- localAffinity = true )).mapTo[ActorRef ]
81
+ localAffinity = true
82
+ )).mapTo[ActorRef ]
83
83
84
- // TODO: Do we need this to load previous year's data?
85
- // (clusterClient ? ClusterClient.Send("/user/supervisor/singleton", FindHmdaFiling(period), localAffinity = true)).mapTo[ActorRef]
86
- // (clusterClient ? ClusterClient.Send("/user/query-supervisor", FindHmdaFilingView(period), localAffinity = true)).mapTo[ActorRef]
84
+ (clusterClient ? ClusterClient .Send (" /user/supervisor/singleton" , FindHmdaFiling (period), localAffinity = true )).mapTo[ActorRef ]
85
+ (clusterClient ? ClusterClient .Send (" /user/query-supervisor" , FindHmdaFilingView (period), localAffinity = true )).mapTo[ActorRef ]
87
86
88
87
val fUploadSubmission = for {
89
- p <- fProcessingActor
90
88
s <- fSubmissionsActor
91
- fSubmission <- (s ? GetSubmissionById (submissionId)).mapTo[Submission ]
92
- } yield (fSubmission, fSubmission.status == Created , p)
89
+ fSubmission <- (s ? CreateSubmission ).mapTo[Option [Submission ]]
90
+ submission = fSubmission.getOrElse(Submission ())
91
+ } yield (submission, submission.status == Created )
93
92
94
93
fUploadSubmission.onComplete {
95
- case Success ((submission, true , processingActor)) =>
96
- uploadData(processingActor, 0L , submission)
97
-
98
- case Success ((_, false , _)) =>
94
+ case Success ((submission, true )) =>
95
+ val message = FindProcessingActor (SubmissionManager .name, submission.id)
96
+ val fProcessingActor = (clusterClient ? ClusterClient .Send (" /user/supervisor/singleton" , message, localAffinity = true )).mapTo[ActorRef ]
97
+ fProcessingActor.onComplete { processingActor =>
98
+ uploadData(processingActor.getOrElse(ActorRef .noSender), uploadTimestamp, fileName, submission, source)
99
+ }
100
+
101
+ case Success ((_, false )) =>
99
102
log.error(" submission not available for upload" )
100
103
sys.exit(0 )
101
104
@@ -106,6 +109,23 @@ object HmdaLarLoader extends FlowUtils {
106
109
}
107
110
}
108
111
109
- private def uploadData (processingActor : ActorRef , uploadTimestamp : Long , submission: Submission ): Unit = ???
112
+ private def uploadData (processingActor : ActorRef , uploadTimestamp : Long , fileName : String , submission : Submission , source : Source [ByteString , Future [IOResult ]]): Unit = {
113
+ processingActor ! AddFileName (fileName)
114
+ processingActor ! StartUpload
115
+ val uploadedF = source
116
+ .via(framing)
117
+ .map(_.utf8String)
118
+ .mapAsync(parallelism = flowParallelism)(line => (processingActor ? AddLine (uploadTimestamp, line)).mapTo[Persisted .type ])
119
+ .runWith(Sink .ignore)
120
+
121
+ uploadedF.onComplete {
122
+ case Success (_) =>
123
+ processingActor ! CompleteUpload
124
+
125
+ case Failure (error) =>
126
+ processingActor ! Shutdown
127
+ log.error(error.getLocalizedMessage)
128
+ }
129
+ }
110
130
111
131
}
0 commit comments