Update KPL and KCL libraries (#44)

* upgraded KCL

* upgrade KPL

* updated readme

Author: markglh

README.md

@@ -35,6 +35,7 @@ It's worth familiarising yourself with [Sequence numbers and Sub sequence number
         * [Valid Release Tag Examples:](#contributor-guide-tag-requirements-valid-release-tag-examples)
         * [Invalid Release Tag Examples:](#contributor-guide-tag-requirements-invalid-release-tag-examples)
 * [Contribution policy](#contribution-policy)
+* [Changelog](#changelog)
 * [License](#license)
 
 
@@ -596,6 +597,10 @@ explicitly, by submitting any copyrighted material via pull request, email, or o
 agree to license the material under the project's open source license and warrant that you have the
 legal authority to do so.
 
+<a name="changelog"></a>
+# Changelog
+See the releases tab: https://github.com/WW-Digital/reactive-kinesis/releases
+
 <a name="license"></a>
 # License
 

build.sbt

@@ -40,11 +40,11 @@ lazy val library =
       "com.fasterxml.uuid"         % "java-uuid-generator"           % "3.1.4"            % Compile)
 
     val amazon = Seq(
-      "com.amazonaws"              % "amazon-kinesis-client"         % "1.8.1"            % Compile
+      "com.amazonaws"              % "amazon-kinesis-client"         % "1.8.8"            % Compile
         excludeAll(
           ExclusionRule(organization = "com.fasterxml.jackson.core"),
           ExclusionRule(organization = "com.fasterxml.jackson.dataformat")),
-      "com.amazonaws"              % "amazon-kinesis-producer"       % "0.12.5"           % Compile
+      "com.amazonaws"              % "amazon-kinesis-producer"       % "0.12.8"           % Compile
         excludeAll(
           ExclusionRule(organization = "com.fasterxml.jackson.core"),
           ExclusionRule(organization = "com.fasterxml.jackson.dataformat"))

src/main/resources/reference.conf

@@ -652,6 +652,70 @@ kinesis {
          # Default: no timeout
          #timeoutInSeconds =
 
+         # The amount of milliseconds to wait before graceful shutdown forcefully terminates the process.
+         # Default: 5000
+         #shutdownGraceMillis = 5000
+
+         # Internally support timeouts and retries for GetRecords calls.
+         #
+         # If retryGetRecordsInSeconds is set
+         # And maxGetRecordsThreadPool is set
+         # Then getRecords will asynchronously retry internally using a CompletionService of
+         # max size "maxGetRecordsThreadPool" with "retryGetRecordsInSeconds" between each retry.
+         #
+         # NOTE: this enables the AsynchronousGetRecordsRetrievalStrategy for getRecords
+         #
+         # Time in seconds to wait before the worker retries to get a record
+         # Default: Optional value, default not set
+         #retryGetRecordsInSeconds = 1
+         #
+         # max number of threads in the getRecords thread pool (per shard)
+         # Default: Optional value, default not set
+         #maxGetRecordsThreadPool = 2
+
+
+         #######################
+         # Pre-fetching config #
+         #######################
+         # Pre-fetching will retrieve and queue additional records from Kinesis while the
+         # application is processing existing records.
+         # Pre-fetching can be enabled by setting dataFetchingStrategy to PREFETCH_CACHED. Once
+         # enabled an additional fetching thread will be started to retrieve records from Kinesis.
+         # Retrieved records will be held in a queue until the application is ready to process them.
+
+         # Which data fetching strategy to use (DEFAULT, PREFETCH_CACHED)
+         # Default: DEFAULT
+         dataFetchingStrategy = PREFETCH_CACHED
+
+         #
+         # Pre-fetching supports the following configuration values:
+         #
+
+         # The maximum number of process records input that can be queued
+         # Default: 3
+         #maxPendingProcessRecordsInput = 3
+
+         # The maximum number of bytes that can be queued
+         # Default 8388608 (8 * 1024 * 1024 / 8Mb)
+         #maxCacheByteSize = 8388608
+
+         # The maximum number of records that can be queued
+         # Default: 30000
+         #maxRecordsCount = 30000
+
+         # The amount of time to wait between calls to Kinesis
+         # Default: 1500
+         #idleMillisBetweenCalls = 1500
+
+         ##############################
+         # End of Pre-fetching config #
+         ##############################
+
+
+         # Milliseconds after which the logger will log a warning message for the long running task
+         # Default: not set
+         #logWarningForTaskAfterMillis = 100
+
       }
    }
 }

src/main/scala/com/weightwatchers/reactive/kinesis/producer/ProducerConf.scala

@@ -96,17 +96,6 @@ object ProducerConf {
       KinesisProducerConfiguration.fromProperties(kplProps)
     credentialsProvider.foreach(kplLibConfiguration.setCredentialsProvider)
 
-    //TODO, this should be part of the KPL. The KCL would handle enums and ints and let us use props directly.
-    //TODO can be removed once this is merged: https://github.com/awslabs/amazon-kinesis-producer/pull/134
-    if (kplConfig.hasPath("ThreadingModel")) {
-      kplLibConfiguration.setThreadingModel(
-        ThreadingModel.valueOf(kplConfig.getString("ThreadingModel"))
-      )
-    }
-    if (kplConfig.hasPath("ThreadPoolSize")) {
-      kplLibConfiguration.setThreadPoolSize(kplConfig.getInt("ThreadPoolSize"))
-    }
-
     kplLibConfiguration
   }