Remove postgresql code from dependent projects

Author: jmarin

api/src/main/scala/hmda/api/http/admin/InstitutionAdminHttpApi.scala

@@ -18,14 +18,9 @@ import hmda.persistence.HmdaSupervisor.FindFilings
 import hmda.persistence.institutions.FilingPersistence.CreateFiling
 import hmda.persistence.institutions.{ FilingPersistence, InstitutionPersistence }
 import hmda.persistence.institutions.InstitutionPersistence.{ CreateInstitution, ModifyInstitution }
-import hmda.persistence.messages.CommonMessages.Event
 import hmda.persistence.model.HmdaSupervisorActor.FindActorByName
-import hmda.query.projections.institutions.InstitutionDBProjection.{ CreateSchema, DeleteSchema }
-import hmda.query.view.institutions.InstitutionView
-import hmda.query.view.messages.CommonViewMessages.GetProjectionActorRef
 
 import scala.concurrent.ExecutionContext
-import scala.util.matching.Regex
 import scala.util.{ Failure, Success }
 
 trait InstitutionAdminHttpApi
@@ -85,31 +80,5 @@ trait InstitutionAdminHttpApi
       }
     }
 
-  private val cdRegex = new Regex("create|delete")
-
-  def institutionsSchemaPath(querySupervisor: ActorRef) =
-    path("institutions" / cdRegex) { command =>
-      extractExecutionContext { executor =>
-        implicit val ec: ExecutionContext = executor
-        val fInstitutionsActor = (querySupervisor ? FindActorByName(InstitutionView.name)).mapTo[ActorRef]
-        val message = command match {
-          case "create" => CreateSchema
-          case "delete" => DeleteSchema
-        }
-        timedGet { uri =>
-          val event = for {
-            instAct <- fInstitutionsActor
-            dbAct <- (instAct ? GetProjectionActorRef).mapTo[ActorRef]
-            cd <- (dbAct ? message).mapTo[Event]
-          } yield cd
-
-          onComplete(event) {
-            case Success(response) => complete(ToResponseMarshallable(StatusCodes.Accepted -> response.toString))
-            case Failure(error) => completeWithInternalError(uri, error)
-          }
-        }
-      }
-    }
-
-  def institutionAdminRoutes(supervisor: ActorRef, querySupervisor: ActorRef) = encodeResponse { institutionsWritePath(supervisor) ~ institutionsSchemaPath(querySupervisor) }
+  def institutionAdminRoutes(supervisor: ActorRef, querySupervisor: ActorRef) = encodeResponse { institutionsWritePath(supervisor) }
 }

api/src/main/scala/hmda/api/http/institutions/UploadPaths.scala

@@ -27,7 +27,6 @@ import hmda.persistence.processing.ProcessingMessages.{ CompleteUpload, Persiste
 import hmda.persistence.processing.SubmissionManager
 import hmda.persistence.processing.SubmissionManager.AddFileName
 import hmda.query.HmdaQuerySupervisor.FindHmdaFilingView
-import hmda.query.projections.filing.HmdaFilingDBProjection.{ CreateSchema, DeleteLars }
 
 import scala.util.{ Failure, Success }
 
@@ -61,10 +60,6 @@ trait UploadPaths extends InstitutionProtocol with ApiErrorProtocol with Submiss
 
         onComplete(fUploadSubmission) {
           case Success((submission, true, processingActor)) =>
-            //TODO: remove this when removing PostgreSQL from project
-            val queryProjector = system.actorSelection(s"/user/query-supervisor/HmdaFilingView-$period/queryProjector")
-            queryProjector ! CreateSchema
-            queryProjector ! DeleteLars(institutionId)
             uploadFile(processingActor, uploadTimestamp, uri.path, submission)
           case Success((_, false, _)) =>
             val errorResponse = Failed(s"Submission $seqNr not available for upload")

api/src/test/scala/hmda/api/http/admin/InstitutionAdminHttpApiSpec.scala

@@ -10,13 +10,11 @@ import hmda.api.model.{ ModelGenerators, Status }
 import org.scalatest.{ BeforeAndAfterAll, MustMatchers, WordSpec }
 import akka.http.scaladsl.marshallers.sprayjson.SprayJsonSupport._
 import akka.http.scaladsl.server.MalformedRequestContentRejection
-import akka.testkit.TestProbe
 import hmda.api.http.BaseHttpApi
 import hmda.model.institution.Institution
 import hmda.model.institution.InstitutionGenerators._
 import hmda.persistence.HmdaSupervisor
 import hmda.query.HmdaQuerySupervisor
-import hmda.query.projections.institutions.InstitutionDBProjection._
 import hmda.validation.ValidationStats
 
 import scala.concurrent.duration._
@@ -39,12 +37,6 @@ class InstitutionAdminHttpApiSpec
 
   val querySupervisor = HmdaQuerySupervisor.createQuerySupervisor(system)
 
-  override def beforeAll(): Unit = {
-    val probe = TestProbe()
-    val db = createInstitutionDBProjection(system)
-    probe.send(db, CreateSchema)
-  }
-
   override def afterAll(): Unit = {
     super.afterAll()
     system.terminate()
@@ -115,18 +107,6 @@ class InstitutionAdminHttpApiSpec
         status mustBe StatusCodes.NotFound
       }
     }
-    "delete institution schema" in {
-      HttpRequest(HttpMethods.GET, uri = "/institutions/delete") ~> institutionAdminRoutes(supervisor, querySupervisor) ~> check {
-        status mustBe StatusCodes.Accepted
-        responseAs[String] mustBe "InstitutionSchemaDeleted()"
-      }
-    }
-    "create institution schema" in {
-      HttpRequest(HttpMethods.GET, uri = "/institutions/create") ~> institutionAdminRoutes(supervisor, querySupervisor) ~> check {
-        status mustBe StatusCodes.Accepted
-        responseAs[String] mustBe "InstitutionSchemaCreated()"
-      }
-    }
   }
 
   private def createRequest(jsonRequest: ByteString, method: HttpMethod): HttpRequest = {

api/src/test/scala/hmda/api/http/institutions/UploadPathsSpec.scala

@@ -11,12 +11,8 @@ import hmda.model.fi._
 import hmda.persistence.HmdaSupervisor.FindSubmissions
 import hmda.persistence.institutions.SubmissionPersistence
 import hmda.persistence.institutions.SubmissionPersistence.UpdateSubmissionStatus
-import hmda.query.DbConfiguration._
-import hmda.query.model.filing.ModifiedLoanApplicationRegister
-import scala.concurrent.{ Await, Future }
 
 class UploadPathsSpec extends InstitutionHttpApiAsyncSpec with SubmissionProtocol with UploadPaths {
-  import config.profile.api._
 
   val csv = "1|0123456789|9|201301171330|2013|99-9999999|900|MIKES SMALL BANK   XXXXXXXXXXX|1234 Main St       XXXXXXXXXXXXXXXXXXXXX|Sacramento         XXXXXX|CA|99999-9999|MIKES SMALL INC    XXXXXXXXXXX|1234 Kearney St    XXXXXXXXXXXXXXXXXXXXX|San Francisco      XXXXXX|CA|99999-1234|Mrs. Krabappel     XXXXXXXXXXX|916-999-9999|999-753-9999|krabappel@gmail.comXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\n" +
     "2|0123456789|9|ABC|20130117|4|3|2|1|10000|1|5|20130119|06920|06|034|0100.01|4|5|7|4|3|2|1|8|7|6|5|4|1|2|9000|0|9|8|7|01.05|2|4\n" +
@@ -34,30 +30,9 @@ class UploadPathsSpec extends InstitutionHttpApiAsyncSpec with SubmissionProtoco
 
   val id = SubmissionId("0", "2017", 1)
 
-  val repository = new LarRepository(config)
-  val modifiedLarRepository = new ModifiedLarRepository(config)
-
-  private def dropAllObjects: Future[Int] = {
-    val db = repository.config.db
-    val dropAll = sqlu"""DROP ALL OBJECTS"""
-    db.run(dropAll)
-  }
-
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    val setup = for {
-      _ <- dropAllObjects
-      _ <- repository.createSchema()
-      c <- modifiedLarRepository.createSchema()
-    } yield c
-    Await.result(setup, duration)
-  }
-
   override def afterAll(): Unit = {
     super.afterAll()
-    dropAllObjects.onComplete {
-      case _ => system.terminate()
-    }
+    system.terminate()
   }
 
   "Upload Paths" must {
@@ -78,12 +53,6 @@ class UploadPathsSpec extends InstitutionHttpApiAsyncSpec with SubmissionProtoco
         val submission = responseAs[Submission]
         submission.status mustBe Uploaded
         submission.start must be < System.currentTimeMillis()
-        Thread.sleep(4000) //Needs to go through validation process
-        modifiedLarRepository.findByInstitutionId("0").map {
-          case xs: Seq[ModifiedLoanApplicationRegister] =>
-            xs.length mustBe 3
-            xs.head.agencyCode mustBe 8
-        }
       }
     }
 

api/src/test/scala/hmda/api/http/public/PublicHttpApiSpec.scala

@@ -1,63 +1,36 @@
 package hmda.api.http.public
 
 import akka.event.{ LoggingAdapter, NoLogging }
-import akka.http.scaladsl.model.{ ContentTypes, StatusCodes }
+import akka.http.scaladsl.model.StatusCodes
 import akka.http.scaladsl.testkit.ScalatestRouteTest
 import akka.util.Timeout
 import hmda.api.RequestHeaderUtils
 import hmda.model.fi.lar.LarGenerators
 import hmda.query.repository.filing.LarConverter._
-import hmda.query.DbConfiguration._
-
 import org.scalatest.{ BeforeAndAfterAll, MustMatchers, WordSpec }
+
 import scala.concurrent.duration._
-import scala.concurrent.{ Await, Future }
 
 class PublicHttpApiSpec extends WordSpec with MustMatchers with BeforeAndAfterAll
     with ScalatestRouteTest with RequestHeaderUtils with PublicHttpApi with LarGenerators {
 
   override val log: LoggingAdapter = NoLogging
   implicit val ec = system.dispatcher
-  val repository = new LarRepository(config)
 
   val duration = 10.seconds
   override implicit val timeout = Timeout(duration)
 
-  import repository.config.profile.api._
-
   val p = "2017"
   val l1 = toLoanApplicationRegisterQuery(sampleLar).copy(period = p, institutionId = "0")
   val l2 = toLoanApplicationRegisterQuery(sampleLar).copy(period = p, institutionId = "0")
 
-  override def beforeAll(): Unit = {
-    super.beforeAll()
-    val setup = for {
-      a <- dropAllObjects
-      b <- repository.createSchema()
-      c <- modifiedLarRepository.createSchema()
-      d <- repository.insertOrUpdate(l1)
-      e <- repository.insertOrUpdate(l2)
-    } yield (d, e)
-    Await.result(setup, duration)
-  }
-
-  override def afterAll(): Unit = {
-    super.afterAll()
-    dropAllObjects
-  }
-
-  private def dropAllObjects: Future[Int] = {
-    val db = repository.config.db
-    val dropAll = sqlu"""DROP ALL OBJECTS"""
-    db.run(dropAll)
-  }
-
   "Modified LAR Http API" must {
     "return list of modified LARs in proper format" in {
       Get("/institutions/0/filings/2017/lar") ~> publicHttpRoutes ~> check {
         status mustBe StatusCodes.OK
-        contentType mustBe ContentTypes.`text/csv(UTF-8)`
-        responseAs[String].split("\n") must have size 2
+        //TODO: update when modified lar is reimplemented
+        //contentType mustBe ContentTypes.`text/csv(UTF-8)`
+        //responseAs[String].split("\n") must have size 2
       }
     }
   }

cluster/src/main/scala/hmda/cluster/HmdaPlatform.scala

@@ -19,17 +19,11 @@ import hmda.publication.HmdaPublication
 import hmda.query.{ HmdaProjectionQuery, HmdaQuerySupervisor }
 import hmda.query.view.institutions.InstitutionView
 import hmda.validation.ValidationStats
-import hmda.query.DbConfiguration._
-import hmda.query.projections.institutions.InstitutionDBProjection.{ CreateSchema, _ }
 import hmda.cluster.HmdaConfig._
 import hmda.persistence.demo.DemoData
-import hmda.persistence.messages.events.institutions.InstitutionEvents.InstitutionSchemaCreated
 import hmda.persistence.messages.CommonMessages._
 
 import scala.concurrent.duration._
-import hmda.query.projections.filing.HmdaFilingDBProjection._
-import hmda.query.view.messages.CommonViewMessages.GetProjectionActorRef
-import hmda.util.FutureRetry.retry
 
 object HmdaPlatform extends App {
 
@@ -147,19 +141,14 @@ object HmdaPlatform extends App {
     if (isDemo) {
       implicit val ec = system.dispatcher
       cleanup()
-      implicit val scheduler = system.scheduler
-      val retries = List(200.millis, 200.millis, 500.millis, 1.seconds, 2.seconds)
       log.info("*** LOADING DEMO DATA ***")
 
       val institutionCreatedF = for {
-        q <- retry((institutionView ? GetProjectionActorRef).mapTo[ActorRef], retries, 10, 300.millis)
-        s <- (q ? CreateSchema).mapTo[InstitutionSchemaCreated]
         i <- (supervisor ? FindActorByName(InstitutionPersistence.name)).mapTo[ActorRef]
-      } yield (s, i)
+      } yield i
 
       institutionCreatedF.map {
-        case (s, i) =>
-          log.info(s.toString)
+        case i =>
           DemoData.loadDemoData(system, i)
       }
     }
@@ -172,12 +161,6 @@ object HmdaPlatform extends App {
       log.info("CLEANING JOURNAL")
       file.listFiles.foreach(f => f.delete())
     }
-
-    val larRepository = new LarRepository(config)
-    val institutionRepository = new InstitutionRepository(config)
-
-    larRepository.dropSchema()
-    institutionRepository.dropSchema()
   }
 
 }

persistence-model/src/main/scala/hmda/persistence/messages/events/institutions/InstitutionEvents.scala

@@ -5,8 +5,6 @@ import hmda.persistence.messages.CommonMessages._
 
 object InstitutionEvents {
   trait InstitutionEvent extends Event
-  case class InstitutionSchemaCreated() extends InstitutionEvent
-  case class InstitutionSchemaDeleted() extends InstitutionEvent
   case class InstitutionCreated(institution: Institution) extends InstitutionEvent
   case class InstitutionModified(institution: Institution) extends InstitutionEvent
 }

query/src/main/resources/application.conf

@@ -24,16 +24,17 @@ hmda {
   }
 }
 
-db {
-  driver = "slick.driver.PostgresDriver$"
-
-  db {
-    url = "jdbc:postgresql://192.168.99.100:54321/hmda?user=postgres&password=postgres"
-    url = ${?JDBC_URL}
-    driver = org.postgresql.Driver
-    numThreads = 2
-  }
-}
+//TODO: configure database access for Keycloak
+//db {
+//  driver = "slick.driver.PostgresDriver$"
+//
+//  db {
+//    url = "jdbc:postgresql://192.168.99.100:54321/hmda?user=postgres&password=postgres"
+//    url = ${?JDBC_URL}
+//    driver = org.postgresql.Driver
+//    numThreads = 2
+//  }
+//}
 
 cassandra {
   host = "192.168.99.100"