limit number of uploaded check-ins in order to keep the size small

Author: cbellone

backend/src/main/kotlin/alfio/pi/manager/CheckInManager.kt

@@ -340,12 +340,12 @@ open class CheckInDataManager(@Qualifier("masterConnectionConfiguration") privat
         })
     }
 
-    @Scheduled(fixedDelay = 15000L)
+    @Scheduled(fixedDelay = 1500L)
     open fun processPendingEntries() {
         if(kvStore.isLeader()) {
-            val failures = kvStore.findRemoteFailures()
-            logger.trace("found ${failures.size} pending scan to upload")
-            failures
+            val pending = kvStore.findCheckInToUpload()
+            logger.trace("found ${pending.size} pending scan to upload")
+            pending
                 .groupBy { it.eventKey }
                 .mapKeys { eventRepository.loadSingle(it.key) }
                 .filter { it.key.isPresent }
@@ -354,18 +354,18 @@ open class CheckInDataManager(@Qualifier("masterConnectionConfiguration") privat
     }
 
     private fun uploadEntriesForEvent(entry: Map.Entry<Optional<Event>, List<ScanLog>>) {
-        logger.info("******** uploading check-in **********")
+        logger.info("******** uploading check-in for event ${entry.key.get().key} (${entry.value.size}) **********")
         tryOrDefault<Unit>().invoke({
             val event = entry.key.get()
-            val scanLogEntries = entry.value.filter { it.ticket != null }
-            val ticketIdToScanLogId = scanLogEntries.associate { it.ticket!!.uuid to it.id }
-            val response = remoteCheckInExecutor.remoteBulkCheckIn(event.key, scanLogEntries) { list -> list.map { mapOf("identifier" to it.ticket!!.uuid, "code" to "${it.ticket.uuid}/${it.ticket.hmac}")}}
-
-            response.forEach {
-                if(logger.isTraceEnabled) {
-                    logger.trace("upload entries response is ${it.key} ${gson.toJson(it.value)}")
+            entry.value.filter { it.ticket != null }.chunked(200).forEach { scanLogEntries ->
+                val ticketIdToScanLogId = scanLogEntries.associate { it.ticket!!.uuid to it.id }
+                val response = remoteCheckInExecutor.remoteBulkCheckIn(event.key, scanLogEntries) { list -> list.map { mapOf("identifier" to it.ticket!!.uuid, "code" to "${it.ticket.uuid}/${it.ticket.hmac}")}}
+                response.forEach {
+                    if(logger.isTraceEnabled) {
+                        logger.trace("upload entries response is ${it.key} ${gson.toJson(it.value)}")
+                    }
+                    kvStore.updateRemoteResult(it.value.result.status, ticketIdToScanLogId[it.key] ?: error("Unexpected error during check-in upload"))
                 }
-                kvStore.updateRemoteResult(it.value.result.status, ticketIdToScanLogId[it.key] ?: error("Unexpected error during check-in upload"))
             }
         }, { logger.error("unable to upload pending check-in", it)})
         logger.info("******** upload completed **********")

backend/src/main/kotlin/alfio/pi/manager/KVStore.kt

@@ -202,7 +202,7 @@ open class KVStore(private val gson: Gson) {
         val key = scanLogId()
         val scanLogWithKey = ScanLogToPersist(key, timestamp.toInstant().toEpochMilli(), timestamp.zone.id, eventKey, uuid, userId,
             localResult, remoteResult, badgePrinted, jsonPayload)
-        putScanLong(scanLogWithKey)
+        putScanLog(scanLogWithKey)
     }
 
     open fun insertBadgeScan(eventKey: String, badgeScan: BadgeScan) =
@@ -224,7 +224,7 @@ open class KVStore(private val gson: Gson) {
         .limit(100)
         .collect(Collectors.toList())
 
-    private fun putScanLong(scanLog: ScanLogToPersist) {
+    private fun putScanLog(scanLog: ScanLogToPersist) {
         scanLogTable.put(scanLog.id, scanLog)
         scanLogForEventTicket.put(scanLog.eventKey + "_" + scanLog.ticketUuid, scanLog.id)
     }
@@ -255,7 +255,7 @@ open class KVStore(private val gson: Gson) {
         return findOptionalById(key).filter { scanLog -> scanLog.eventKey == eventKey }
     }
 
-    open fun findRemoteFailures(): List<ScanLog> {
+    open fun findCheckInToUpload(): List<ScanLog> {
         return scanLogTable.stream()
             .filter{ it.value.remoteResult == CheckInStatus.RETRY}
             .map { it.value.toScanLog() }
@@ -273,7 +273,7 @@ open class KVStore(private val gson: Gson) {
         findOptionalById(key).ifPresent { scanLog ->
             val updatedScanLog = ScanLogToPersist(scanLog.id, scanLog.timestamp.toInstant().toEpochMilli(), scanLog.timestamp.zone.id, scanLog.eventKey, scanLog.ticketUuid,
                 scanLog.userId, scanLog.localResult, remoteResult, scanLog.badgePrinted, scanLog.ticketData)
-            putScanLong(updatedScanLog)
+            putScanLog(updatedScanLog)
         }
     }
 

backend/src/main/kotlin/alfio/pi/manager/RemoteCheckInExecutor.kt

@@ -57,7 +57,7 @@ open class RemoteCheckInExecutor(private val gson: Gson,
                 .url(url)
                 .build()
             logger.debug("Will call remote url {}", url)
-            httpClientWithCustomTimeout(100L to TimeUnit.MILLISECONDS, 1L to TimeUnit.SECONDS)
+            httpClientWithCustomTimeout(1L to TimeUnit.SECONDS, 10L to TimeUnit.SECONDS)
                 .invoke(httpClient)
                 .newCall(request)
                 .execute()