Initiates renewal of certificates during last third of lifetime (#15)

Also
* Upgraded kubernetes-client
* Configurable solver role to locate
* Option to dry-run
* Development option to override issuer
* Improved concurrent access of account retrieval

Author: itzg

build.gradle

@@ -2,7 +2,7 @@ plugins {
     id 'org.springframework.boot' version '2.7.3'
     id 'io.spring.dependency-management' version '1.0.13.RELEASE'
     id 'java'
-    id "io.github.itzg.simple-boot-image" version "0.5.0"
+    id 'io.github.itzg.simple-boot-image' version '0.5.1'
     // https://github.com/qoomon/gradle-git-versioning-plugin
     id 'me.qoomon.git-versioning' version '6.3.0'
 }
@@ -41,8 +41,7 @@ dependencies {
     implementation 'org.springframework.boot:spring-boot-starter-actuator'
     implementation 'org.springframework.boot:spring-boot-starter-webflux'
     implementation 'org.springframework.boot:spring-boot-starter-validation'
-    implementation 'com.google.code.findbugs:jsr305:3.0.2'
-    implementation 'io.fabric8:kubernetes-client:5.12.2'
+    implementation 'io.fabric8:kubernetes-client:6.0.0'
     implementation 'com.nimbusds:nimbus-jose-jwt:9.24.2'
     implementation 'org.bouncycastle:bcpkix-jdk18on:1.71.1'
 

k8s/deployment.yml

@@ -22,6 +22,8 @@ spec:
           env:
             - name: LOGGING_LEVEL_APP
               value: DEBUG
+            - name: KITA_SOLVER_ROLE
+              value: solver-dev
           volumeMounts:
             - mountPath: /application/config
               name: configs

k8s/service.yml

@@ -4,7 +4,7 @@ metadata:
   name: kita-dev
   labels:
     app: kita-dev
-    acme.itzg.github.io/role: solver
+    acme.itzg.github.io/role: solver-dev
 spec:
   selector:
     app: kita-dev

src/main/java/app/K8sIngressTlsAcmeApplication.java

@@ -1,16 +1,15 @@
 package app;
 
-import app.config.AppProperties;
 import java.security.Security;
 import org.springframework.boot.SpringApplication;
 import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.boot.context.properties.EnableConfigurationProperties;
+import org.springframework.boot.context.properties.ConfigurationPropertiesScan;
 import org.springframework.scheduling.annotation.EnableScheduling;
-import org.springframework.web.reactive.config.EnableWebFlux;
 
 @SpringBootApplication
 @EnableScheduling
-@EnableConfigurationProperties(AppProperties.class)
+@ConfigurationPropertiesScan
+//@EnableConfigurationProperties(AppProperties.class)
 public class K8sIngressTlsAcmeApplication {
 
     public static void main(String[] args) {

src/main/java/app/config/AppProperties.java

@@ -4,11 +4,11 @@
 import java.util.Map;
 import javax.validation.Valid;
 import javax.validation.constraints.Min;
+import javax.validation.constraints.NotBlank;
 import javax.validation.constraints.NotEmpty;
 import javax.validation.constraints.NotNull;
 import org.springframework.boot.context.properties.ConfigurationProperties;
 import org.springframework.boot.context.properties.bind.DefaultValue;
-import org.springframework.stereotype.Component;
 import org.springframework.validation.annotation.Validated;
 
 @ConfigurationProperties("kita")
@@ -27,7 +27,14 @@ public record AppProperties(
     long maxAuthFinalizeAttempts,
 
     @DefaultValue("2s") @NotNull
-    Duration authFinalizeRetryDelay
+    Duration authFinalizeRetryDelay,
+
+    boolean dryRun,
+
+    @DefaultValue("solver") @NotBlank
+    String solverRole,
+
+    String overrideIssuer
 ) {
 
 }

src/main/java/app/services/AcmeAccountService.java

@@ -2,36 +2,29 @@
 
 import app.config.Issuer;
 import app.messages.AccountRequest;
-import app.model.AcmeAccount;
 import app.messages.AccountResponse;
+import app.model.AcmeAccount;
 import com.nimbusds.jose.JOSEException;
 import com.nimbusds.jose.JWSAlgorithm;
 import com.nimbusds.jose.jwk.KeyUse;
 import com.nimbusds.jose.jwk.RSAKey;
 import com.nimbusds.jose.jwk.gen.RSAKeyGenerator;
 import java.net.URI;
-import java.nio.charset.StandardCharsets;
-import java.security.MessageDigest;
-import java.security.NoSuchAlgorithmException;
-import java.util.Base64;
-import java.util.Base64.Encoder;
-import java.util.Collections;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Objects;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import lombok.extern.slf4j.Slf4j;
 import org.springframework.stereotype.Service;
-import org.springframework.web.reactive.function.client.WebClient;
 import reactor.core.publisher.Mono;
 
 @Service
 @Slf4j
 public class AcmeAccountService {
 
-    private AcmeBaseRequestService baseRequestService;
+    private final AcmeBaseRequestService baseRequestService;
     private final AcmeDirectoryService directoryService;
-    private final Map<String, AcmeAccount> accounts = Collections.synchronizedMap(new HashMap<>());
+    private final Map<String, Mono<AcmeAccount>> accounts = new ConcurrentHashMap<>();
 
     public AcmeAccountService(
         AcmeDirectoryService directoryService,
@@ -54,22 +47,17 @@ private static RSAKey generateJwk() {
     }
 
     public Mono<AcmeAccount> accountForIssuer(String issuerId) {
-        synchronized (accounts) {
-            final AcmeAccount acmeAccount = accounts.get(issuerId);
-            if (acmeAccount != null) {
-                return Mono.just(acmeAccount);
-            }
-
-            log.debug("Retrieving account for issuerId={}", issuerId);
+        return accounts.computeIfAbsent(issuerId, key -> {
+            final Issuer issuer = directoryService.issuerFor(key);
 
-            final Issuer issuer = directoryService.issuerFor(issuerId);
-
-            return retrieveAccount(issuerId, issuer)
-                .doOnNext(account -> accounts.put(issuerId, account));
-        }
+            return retrieveAccount(key, issuer)
+                .cache();
+        });
     }
 
     private Mono<AcmeAccount> retrieveAccount(String issuerId, Issuer issuer) {
+        log.debug("Retrieving account for issuerId={}", issuerId);
+
         final RSAKey jwk = generateJwk();
 
         final URI newAccountUrl = directoryService.directoryFor(issuerId).newAccount();

src/main/java/app/services/AcmeBaseRequestService.java

@@ -4,17 +4,14 @@
 import app.model.SignableValue;
 import com.nimbusds.jose.jwk.RSAKey;
 import java.net.URI;
-import java.security.cert.X509Certificate;
 import lombok.extern.slf4j.Slf4j;
-import org.jetbrains.annotations.NotNull;
 import org.springframework.http.HttpStatus;
-import org.springframework.http.MediaType;
 import org.springframework.http.ResponseEntity;
+import org.springframework.lang.NonNull;
 import org.springframework.lang.Nullable;
 import org.springframework.stereotype.Service;
 import org.springframework.web.reactive.function.client.WebClient;
 import org.springframework.web.reactive.function.client.WebClient.ResponseSpec;
-import reactor.core.publisher.Flux;
 import reactor.core.publisher.Mono;
 
 @Service
@@ -39,18 +36,16 @@ public <T> Mono<ResponseEntity<T>> request(String issuerId, RSAKey jwk, @Nullabl
     ) {
         log.debug("Creating POST for issuerId={} to url={} payload={}", issuerId, requestUrl, payload);
 
-        return preEntityRequest(issuerId, jwk, kid, requestUrl, payload, responseClass)
+        return preEntityRequest(issuerId, jwk, kid, requestUrl, payload)
             .toEntity(responseClass)
             .doOnNext(directoryService.latchNonce(issuerId))
             .doOnNext(entity -> log.debug("Response status={} from url={} for issuerId={} body={}",
                 entity.getStatusCode(), requestUrl, issuerId, entity.getBody()
             ));
     }
 
-    @NotNull
-    private <T> ResponseSpec preEntityRequest(String issuerId, RSAKey jwk, String kid, URI requestUrl, Object payload,
-        Class<T> responseClass
-    ) {
+    @NonNull
+    private ResponseSpec preEntityRequest(String issuerId, RSAKey jwk, String kid, URI requestUrl, Object payload) {
         return webClient.post()
             .uri(requestUrl)
             .contentType(JwsMessageWriter.JOSE_JSON)

src/main/java/app/services/ApplicationIngressesService.java

@@ -1,5 +1,6 @@
 package app.services;
 
+import app.config.AppProperties;
 import io.fabric8.kubernetes.api.model.Secret;
 import io.fabric8.kubernetes.api.model.networking.v1.Ingress;
 import io.fabric8.kubernetes.api.model.networking.v1.IngressList;
@@ -8,14 +9,26 @@
 import io.fabric8.kubernetes.client.Watch;
 import io.fabric8.kubernetes.client.Watcher;
 import io.fabric8.kubernetes.client.WatcherException;
+import java.io.ByteArrayInputStream;
 import java.io.Closeable;
 import java.io.IOException;
+import java.io.StringReader;
+import java.nio.charset.StandardCharsets;
+import java.security.cert.CertificateException;
+import java.security.cert.CertificateFactory;
+import java.security.cert.X509Certificate;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Base64;
+import java.util.Base64.Decoder;
 import java.util.Collections;
 import java.util.HashSet;
 import java.util.Map;
 import java.util.Objects;
 import java.util.Set;
 import lombok.extern.slf4j.Slf4j;
+import org.bouncycastle.util.io.pem.PemObject;
+import org.bouncycastle.util.io.pem.PemReader;
 import org.springframework.lang.NonNull;
 import org.springframework.scheduling.annotation.Scheduled;
 import org.springframework.stereotype.Service;
@@ -26,13 +39,18 @@ public class ApplicationIngressesService implements Closeable {
 
     private final KubernetesClient k8s;
     private final CertificateProcessingService certificateProcessingService;
+    private final AppProperties appProperties;
     private final Watch ingressWatches;
     private final Watch tlsSecretWatches;
     private final Set<String/*ingress name*/> activeReconciles = Collections.synchronizedSet(new HashSet<>());
 
-    public ApplicationIngressesService(KubernetesClient k8s, CertificateProcessingService certificateProcessingService) {
+    public ApplicationIngressesService(KubernetesClient k8s,
+        CertificateProcessingService certificateProcessingService,
+        AppProperties appProperties
+    ) {
         this.k8s = k8s;
         this.certificateProcessingService = certificateProcessingService;
+        this.appProperties = appProperties;
 
         this.ingressWatches = setupIngressWatch();
         this.tlsSecretWatches = setupTlsSecretWatch();
@@ -74,7 +92,11 @@ public void onClose(WatcherException cause) {
             });
     }
 
-    @Scheduled(fixedDelayString = "#{@'kita-app.config.AppProperties'.certRenewalCheckInterval}")
+    @Scheduled(
+        // initial ingress listing will handle reconciling at startup, so delay for given interval
+        initialDelayString = "#{@'kita-app.config.AppProperties'.certRenewalCheckInterval}",
+        fixedDelayString = "#{@'kita-app.config.AppProperties'.certRenewalCheckInterval}"
+    )
     public void checkCertRenewals() {
         final IngressList ingresses = k8s.network().v1().ingresses()
             .withLabel(Metadata.ISSUER_LABEL)
@@ -99,33 +121,80 @@ private void reconcileIngressTls(Ingress ingress) {
                 .withName(tls.getSecretName())
                 .get();
 
-            final String requestedIssuerId = ingress.getMetadata().getLabels().get(Metadata.ISSUER_LABEL);
+            final String requestedIssuerId =
+                appProperties.overrideIssuer() != null ?
+                    appProperties.overrideIssuer()
+                    : ingress.getMetadata().getLabels().get(Metadata.ISSUER_LABEL);
+
             if (tlsSecret == null) {
-                initiateCertCreation(ingress, name, tls, requestedIssuerId);
+                initiateCertCreation(ingress, tls, requestedIssuerId);
             } else {
                 final String tlsSecretIssuer = nullSafe(tlsSecret.getMetadata().getLabels()).get(Metadata.ISSUER_LABEL);
-                if (!Objects.equals(tlsSecretIssuer, requestedIssuerId)) {
-                    initiateCertCreation(ingress, name, tls, requestedIssuerId);
+                if (!Objects.equals(tlsSecretIssuer, requestedIssuerId)
+                    || needsRenewal(tlsSecret)) {
+                    initiateCertCreation(ingress, tls, requestedIssuerId);
                 } else {
-                    // TODO is cert needing refresh
                     activeReconciles.remove(name);
                 }
             }
         }
 
     }
 
-    private void initiateCertCreation(Ingress ingress, String name, IngressTLS tls, String requestedIssuerId) {
+    private boolean needsRenewal(Secret tlsSecret) {
+        final String certContentEncoded = tlsSecret.getData().get("tls.crt");
+        if (certContentEncoded != null) {
+            final Decoder decoder = Base64.getDecoder();
+
+            try (PemReader pemReader = new PemReader(new StringReader(
+                new String(decoder.decode(certContentEncoded), StandardCharsets.UTF_8)
+            ))) {
+                final PemObject pemObject = pemReader.readPemObject();
+
+                CertificateFactory cf = CertificateFactory.getInstance("X.509");
+                final X509Certificate cert = (X509Certificate) cf.generateCertificate(
+                    new ByteArrayInputStream(pemObject.getContent()));
+                final Instant notAfter = cert.getNotAfter().toInstant();
+                final Instant notBefore = cert.getNotBefore().toInstant();
+                final Duration lifetime = Duration.between(notBefore,
+                    // since it sets expiration just before and between's argument is exclusive
+                    notAfter.plusSeconds(1)
+                );
+                // LetsEncrypt recommends renewing when there is a 3rd of lifetime left
+                // https://letsencrypt.org/docs/integration-guide/#when-to-renew
+                if (Instant.now().isAfter(notAfter.minus(lifetime.dividedBy(3)))) {
+                    log.info("TLS secret {} is due to be renewed since its lifetime is {} days and expires at {}",
+                        tlsSecret.getMetadata().getName(), lifetime.toDays(), notAfter
+                    );
+                    return true;
+                }
+            } catch (IOException e) {
+                log.error("Failed to read/close PEM reader", e);
+            } catch (CertificateException e) {
+                log.error("Failed to get X.509 cert factory", e);
+            }
+        } else {
+            log.error("TLS secret {} is missing tls.crt data", tlsSecret.getMetadata().getName());
+        }
+        return false;
+    }
+
+    private void initiateCertCreation(Ingress ingress, IngressTLS tls, String requestedIssuerId) {
+        final String ingressName = ingress.getMetadata().getName();
+        if (appProperties.dryRun()) {
+            log.info("Skipping cert creation of {} for ingress {} since dry-run is enabled",
+                tls.getSecretName(), ingressName
+            );
+            return;
+        }
+
         certificateProcessingService.initiateCertCreation(ingress, tls, requestedIssuerId)
-            .subscribe(secret -> {
+            .subscribe(secret ->
                     log.info("Cert creation complete for tls entry with secret={} hosts={} in ingress={}",
-                        secret.getMetadata().getName(), tls.getHosts(), name
-                    );
-                },
-                throwable -> {
-                    log.warn("Problem while processing cert creation");
-                },
-                () -> activeReconciles.remove(name)
+                        secret.getMetadata().getName(), tls.getHosts(), ingressName
+                    ),
+                throwable -> log.warn("Problem while processing cert creation"),
+                () -> activeReconciles.remove(ingressName)
             );
     }
 
@@ -135,7 +204,7 @@ private Map<String, String> nullSafe(Map<String, String> value) {
     }
 
     @Override
-    public void close() throws IOException {
+    public void close() {
         ingressWatches.close();
         tlsSecretWatches.close();
     }

src/main/java/app/services/CertificateProcessingService.java

@@ -143,7 +143,8 @@ private Secret storeSecret(String issuerId, List<String> hosts, String certChain
         log.debug("Stored secret={}", secret.getMetadata().getName());
 
         return k8s.secrets()
-            .createOrReplace(secret);
+            .resource(secret)
+            .createOrReplace();
     }
 
     private CertAndKey buildCertAndKey(String certChain, PrivateKey privateKey) {