Merge branch 'master' into n52

Author: Nick Grippin

README.md

@@ -157,6 +157,11 @@ docker-compose -f docker-dev.yml up
 
 When finished, use `docker-compose down` to gracefully stop the running containers.
 
+In order to not get Postgres connection error upon signing a submission, you may want to start up the keycloak image as well.
+```shell
+docker-compose up keycloak
+```
+Not having the keycloak image running will produce an error in the console, but will not cause the platform to crash.
 
 #### Running the API
 
@@ -214,13 +219,13 @@ The Filing API will run on `$(docker-machine ip):8080`
 The Public API will run on `$(docker-machine ip):8082`
 
 By default, the `HDMA Platform` runs with a log level of `INFO`. This can be changed by establishing a different log level in the `HMDA_LOGLEVEL` environment variable.
-For the different logging options, see the [reference.conf](https://github.com/akka/akka/blob/master/akka-actor/src/main/resources/reference.conf#L38) default configuration file for `Akka`. 
+For the different logging options, see the [reference.conf](https://github.com/akka/akka/blob/master/akka-actor/src/main/resources/reference.conf#L38) default configuration file for `Akka`.
 
 #### To run the entire platform
 
 1. Ensure you have a Docker Machine with sufficient resources, as described in the [Docker](#docker) section above.
 
-1. Clone [hmda-platform-ui](https://github.com/cfpb/hmda-platform-ui) and 
+1. Clone [hmda-platform-ui](https://github.com/cfpb/hmda-platform-ui) and
     [hmda-platform-auth](https://github.com/cfpb/hmda-platform-auth) into the same
     directory as hmda-platform.
 
@@ -270,7 +275,7 @@ For the different logging options, see the [reference.conf](https://github.com/a
         **Note:** You must register with an email address from our whitelist of email domains.
         For convenience, `bank0.com` and `bank1.com` address should be available automatically.
 
-    1. Browse to the mock email server at https://192.168.99.100:8443/mail/, and select the 
+    1. Browse to the mock email server at https://192.168.99.100:8443/mail/, and select the
         verification link in the email found there.  This should take you back to the HMDA
         filing web app, now logged in.
 
@@ -281,7 +286,7 @@ For the different logging options, see the [reference.conf](https://github.com/a
 
 ##### Updating an existing system
 
-If you've updated any of the hmda-platform services, and would like to see those 
+If you've updated any of the hmda-platform services, and would like to see those
 changes reflected in the Docker Compose setup, the simplest way to do this is to
 rebuild everything from scratch.  The following command should be executed from
 within the `hmda-platform` directory.

api-model/src/main/resources/application.conf

@@ -0,0 +1,3 @@
+hmda {
+  connectionFlowParallelism = 5
+}

api-model/src/main/scala/hmda/api/util/FlowUtils.scala

@@ -0,0 +1,52 @@
+package hmda.api.util
+
+import akka.NotUsed
+import akka.actor.ActorSystem
+import akka.http.scaladsl.Http
+import akka.http.scaladsl.model.{ HttpMethods, HttpRequest, HttpResponse, Uri }
+import akka.http.scaladsl.unmarshalling.Unmarshal
+import akka.stream.ActorMaterializer
+import akka.stream.scaladsl.{ Flow, Framing }
+import akka.util.ByteString
+import com.typesafe.config.ConfigFactory
+import org.slf4j.Logger
+import scala.concurrent.ExecutionContext
+
+trait FlowUtils {
+
+  implicit val system: ActorSystem
+  implicit val materializer: ActorMaterializer
+  implicit val ec: ExecutionContext
+  val config = ConfigFactory.load()
+  val parallelism = config.getInt("hmda.connectionFlowParallelism")
+
+  def singleConnectionFlow: Flow[HttpRequest, HttpResponse, NotUsed] =
+    Flow[HttpRequest]
+      .mapAsync[HttpResponse](parallelism)(request => {
+        for {
+          response <- Http().singleRequest(request)
+        } yield response
+      })
+
+  def framing: Flow[ByteString, ByteString, NotUsed] = {
+    Framing.delimiter(ByteString("\n"), maximumFrameLength = 65536, allowTruncation = true)
+  }
+
+  def byte2StringFlow: Flow[ByteString, String, NotUsed] =
+    Flow[ByteString].map(bs => bs.utf8String)
+
+  def sendGetRequest(req: String, url: Uri) = {
+    val request = HttpRequest(HttpMethods.GET, uri = s"$url/$req")
+    for {
+      response <- Http().singleRequest(request)
+      content <- Unmarshal(response.entity).to[String]
+    } yield content
+  }
+
+  def exitSys(log: Logger, errorMessage: String, code: Int) = {
+    log.error(errorMessage)
+    system.terminate()
+    Thread.sleep(100)
+    sys.exit(code)
+  }
+}

api/src/main/resources/application-dev.conf

@@ -28,6 +28,15 @@ hmda {
     timeout = ${?HMDA_HTTP_TIMEOUT}
   }
   isDemo = true
+  panel {
+    tcp {
+      host = "0.0.0.0"
+      host = ${?HMDA_PANEL_LOADER_HOST}
+      port = "8888"
+      port = ${?HMDA_PANEL_LOADER_PORT}
+      timeout = 5
+    }
+  }
   actor-flow-parallelism = 4
 }
 

api/src/main/resources/application.conf

@@ -27,8 +27,26 @@ hmda {
     timeout = 10
     timeout = ${?HMDA_HTTP_TIMEOUT}
   }
+  mail {
+    host = "mail_dev"
+    host = ${?HMDA_MAIL_HOST}
+    port = "25"
+    port = ${?HMDA_MAIL_PORT}
+    senderAddress = "no-reply@cfpb.gov"
+    senderAddress = ${?HMDA_MAIL_SENDER_ADDRESS}
+  }
   isDemo = false
   isDemo = ${?HMDA_IS_DEMO}
+  panel {
+    tcp {
+      host = "0.0.0.0"
+      host = ${?HMDA_PANEL_LOADER_HOST}
+      port = "8888"
+      port = ${?HMDA_PANEL_LOADER_PORT}
+      timeout = 5
+      parallelism = 5
+    }
+  }
 }
 
 api-dispatcher {

api/src/main/resources/logback.xml

@@ -18,14 +18,9 @@ import hmda.persistence.HmdaSupervisor.FindFilings
 import hmda.persistence.institutions.FilingPersistence.CreateFiling
 import hmda.persistence.institutions.{ FilingPersistence, InstitutionPersistence }
 import hmda.persistence.institutions.InstitutionPersistence.{ CreateInstitution, ModifyInstitution }
-import hmda.persistence.messages.CommonMessages.Event
 import hmda.persistence.model.HmdaSupervisorActor.FindActorByName
-import hmda.query.projections.institutions.InstitutionDBProjection.{ CreateSchema, DeleteSchema }
-import hmda.query.view.institutions.InstitutionView
-import hmda.query.view.messages.CommonViewMessages.GetProjectionActorRef
 
 import scala.concurrent.ExecutionContext
-import scala.util.matching.Regex
 import scala.util.{ Failure, Success }
 
 trait InstitutionAdminHttpApi
@@ -85,31 +80,5 @@ trait InstitutionAdminHttpApi
       }
     }
 
-  private val cdRegex = new Regex("create|delete")
-
-  def institutionsSchemaPath(querySupervisor: ActorRef) =
-    path("institutions" / cdRegex) { command =>
-      extractExecutionContext { executor =>
-        implicit val ec: ExecutionContext = executor
-        val fInstitutionsActor = (querySupervisor ? FindActorByName(InstitutionView.name)).mapTo[ActorRef]
-        val message = command match {
-          case "create" => CreateSchema
-          case "delete" => DeleteSchema
-        }
-        timedGet { uri =>
-          val event = for {
-            instAct <- fInstitutionsActor
-            dbAct <- (instAct ? GetProjectionActorRef).mapTo[ActorRef]
-            cd <- (dbAct ? message).mapTo[Event]
-          } yield cd
-
-          onComplete(event) {
-            case Success(response) => complete(ToResponseMarshallable(StatusCodes.Accepted -> response.toString))
-            case Failure(error) => completeWithInternalError(uri, error)
-          }
-        }
-      }
-    }
-
-  def institutionAdminRoutes(supervisor: ActorRef, querySupervisor: ActorRef) = encodeResponse { institutionsWritePath(supervisor) ~ institutionsSchemaPath(querySupervisor) }
+  def institutionAdminRoutes(supervisor: ActorRef, querySupervisor: ActorRef) = encodeResponse { institutionsWritePath(supervisor) }
 }

api/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala

@@ -27,7 +27,6 @@ import hmda.persistence.processing.ProcessingMessages.{ CompleteUpload, Persiste
 import hmda.persistence.processing.SubmissionManager
 import hmda.persistence.processing.SubmissionManager.AddFileName
 import hmda.query.HmdaQuerySupervisor.FindHmdaFilingView
-import hmda.query.projections.filing.HmdaFilingDBProjection.{ CreateSchema, DeleteLars }
 
 import scala.util.{ Failure, Success }
 
@@ -61,10 +60,6 @@ trait UploadPaths extends InstitutionProtocol with ApiErrorProtocol with Submiss
 
         onComplete(fUploadSubmission) {
           case Success((submission, true, processingActor)) =>
-            //TODO: remove this when removing PostgreSQL from project
-            val queryProjector = system.actorSelection(s"/user/query-supervisor/HmdaFilingView-$period/queryProjector")
-            queryProjector ! CreateSchema
-            queryProjector ! DeleteLars(institutionId)
             uploadFile(processingActor, uploadTimestamp, uri.path, submission)
           case Success((_, false, _)) =>
             val errorResponse = Failed(s"Submission $seqNr not available for upload")

api/src/main/scala/hmda/api/http/institutions/UploadPaths.scala

@@ -12,7 +12,6 @@ import hmda.api.model.{ Irs, IrsResponse }
 import hmda.api.protocol.processing.MsaProtocol
 import hmda.census.model.Msa
 import hmda.model.fi.SubmissionId
-import hmda.query.repository.filing.FilingComponent
 import hmda.validation.ValidationStats.FindIrsStats
 
 import scala.concurrent.{ ExecutionContext, Future }
@@ -21,8 +20,7 @@ import scala.util.{ Failure, Success }
 trait SubmissionIrsPaths
     extends HmdaCustomDirectives
     with RequestVerificationUtils
-    with MsaProtocol
-    with FilingComponent {
+    with MsaProtocol {
 
   implicit val system: ActorSystem
   implicit val materializer: ActorMaterializer

api/src/main/scala/hmda/api/http/institutions/submissions/SubmissionIrsPaths.scala

@@ -1,5 +1,7 @@
 package hmda.api.http.institutions.submissions
 
+import java.time.{ ZoneOffset, ZonedDateTime }
+
 import akka.actor.{ ActorRef, ActorSystem }
 import akka.event.LoggingAdapter
 import akka.pattern.ask
@@ -22,6 +24,15 @@ import spray.json.{ JsBoolean, JsFalse, JsObject, JsTrue }
 
 import scala.util.{ Failure, Success }
 import scala.concurrent.{ ExecutionContext, Future }
+import javax.mail._
+import javax.mail.internet.{ InternetAddress, MimeMessage }
+
+import com.typesafe.config.ConfigFactory
+import hmda.model.institution.Institution
+import hmda.persistence.model.HmdaSupervisorActor.FindActorByName
+import hmda.query.repository.KeyCloakRepository
+import hmda.query.view.institutions.InstitutionView
+import hmda.query.view.institutions.InstitutionView.GetInstitutionById
 
 trait SubmissionSignPaths
     extends InstitutionProtocol
@@ -30,7 +41,8 @@ trait SubmissionSignPaths
     with EditResultsProtocol
     with HmdaCustomDirectives
     with RequestVerificationUtils
-    with ValidationErrorConverter {
+    with ValidationErrorConverter
+    with KeyCloakRepository {
 
   implicit val system: ActorSystem
   implicit val materializer: ActorMaterializer
@@ -44,7 +56,7 @@ trait SubmissionSignPaths
       val submissionId = SubmissionId(institutionId, period, id)
       timedGet { uri =>
         completeVerified(supervisor, querySupervisor, institutionId, period, id, uri) {
-          completeWithSubmissionReceipt(supervisor, submissionId, uri)
+          completeWithSubmissionReceipt(supervisor, submissionId, uri, signed = false)
         }
       } ~
         timedPost { uri =>
@@ -59,7 +71,7 @@ trait SubmissionSignPaths
                     s <- actor ? hmda.persistence.processing.ProcessingMessages.Signed
                   } yield s
                   onComplete(fSign) {
-                    case Success(Some(_)) => completeWithSubmissionReceipt(supervisor, submissionId, uri)
+                    case Success(Some(_)) => completeWithSubmissionReceipt(supervisor, submissionId, uri, signed = true)
                     case Success(_) =>
                       val errorResponse = ErrorResponse(400, "Illegal State: Submission must be Validated or ValidatedWithErrors to sign", uri.path)
                       complete(ToResponseMarshallable(StatusCodes.BadRequest -> errorResponse))
@@ -76,7 +88,7 @@ trait SubmissionSignPaths
         }
     }
 
-  private def completeWithSubmissionReceipt(supervisor: ActorRef, subId: SubmissionId, uri: Uri)(implicit ec: ExecutionContext) = {
+  private def completeWithSubmissionReceipt(supervisor: ActorRef, subId: SubmissionId, uri: Uri, signed: Boolean)(implicit ec: ExecutionContext) = {
     val fSubmissionsActor = (supervisor ? FindSubmissions(SubmissionPersistence.name, subId.institutionId, subId.period)).mapTo[ActorRef]
     val fSubmission = for {
       a <- fSubmissionsActor
@@ -85,8 +97,69 @@ trait SubmissionSignPaths
 
     onComplete(fSubmission) {
       case Success(sub) =>
+        if (signed) {
+          emailSignature(supervisor, sub)
+        }
         complete(ToResponseMarshallable(Receipt(sub.end, sub.receipt, sub.status)))
       case Failure(error) => completeWithInternalError(uri, error)
     }
   }
+
+  private def emailSignature(supervisor: ActorRef, submission: Submission)(implicit ec: ExecutionContext) = {
+    val emails = findEmailsById(submission.id.institutionId)
+    val querySupervisor = system.actorSelection("/user/query-supervisor/singleton")
+    val fInstitutionsActor = (querySupervisor ? FindActorByName(InstitutionView.name)).mapTo[ActorRef]
+    val fName = for {
+      a <- fInstitutionsActor
+      i <- (a ? GetInstitutionById(submission.id.institutionId)).mapTo[Institution]
+      e <- emails
+    } yield (i.respondent.name, e)
+
+    fName.onComplete({
+      case Success((instName, emailSeq)) =>
+        emailSeq.foreach(t => {
+          val username = t._1 + " " + t._2
+          sendMail(t._3, username, submission, instName)
+        })
+      case Failure(error) => log.error(error, s"An error has occured retrieving the institution name for ID ${submission.id.institutionId}")
+    })
+  }
+
+  private def sendMail(address: String, username: String, submission: Submission, instName: String) = {
+    val config = ConfigFactory.load()
+    val host = config.getString("hmda.mail.host")
+    val port = config.getString("hmda.mail.port")
+    val senderAddress = config.getString("hmda.mail.senderAddress")
+
+    val properties = System.getProperties
+    properties.put("mail.smtp.host", host)
+    properties.put("mail.smtp.port", port)
+
+    val session = Session.getDefaultInstance(properties)
+    val message = new MimeMessage(session)
+
+    val date = getFormattedDate
+
+    val text = s"$username,\n\nCongratulations, you've completed filing your HMDA data for $instName for filing period ${submission.id.period}.\n" +
+      s"We received your filing on: $date\n" +
+      s"Your receipt is: ${submission.receipt}"
+    message.setFrom(new InternetAddress(senderAddress))
+    message.setRecipients(Message.RecipientType.TO, address)
+    message.setSubject("HMDA Filing Successful")
+    message.setText(text)
+
+    log.info(s"Sending message to $address with the message \n$text")
+    Transport.send(message)
+  }
+
+  private def getFormattedDate: String = {
+    val offset = ZoneOffset.ofHours(-5)
+    val zonedTime = ZonedDateTime.now(offset)
+
+    val day = zonedTime.getDayOfMonth
+    val month = zonedTime.getMonthValue
+    val year = zonedTime.getYear
+
+    s"$month/$day/$year"
+  }
 }