From f1b3d7125f339f22ab4dc278adf26d0bf03c47cd Mon Sep 17 00:00:00 2001 From: ppawlowski Date: Mon, 6 Oct 2025 14:34:44 +0200 Subject: [PATCH 1/6] Add helper functions for cert manager annotations --- helm/flowfuse/templates/_helpers.tpl | 76 ++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/helm/flowfuse/templates/_helpers.tpl b/helm/flowfuse/templates/_helpers.tpl index 21541c44..3e4a6826 100644 --- a/helm/flowfuse/templates/_helpers.tpl +++ b/helm/flowfuse/templates/_helpers.tpl @@ -222,3 +222,79 @@ Generate NPM registry admin password if not provided {{- sha256sum $seed | trunc 25 }} {{- end }} {{- end -}} + +{{/* +Check if cert-manager is enabled by detecting cert-manager annotations in ingress.annotations +Usage: {{ if include "forge.certManagerEnabled" . }} +*/}} +{{- define "forge.certManagerEnabled" -}} +{{- $certManagerDetected := false -}} +{{- if .Values.ingress.certManagerIssuer -}} + {{- $certManagerDetected = true -}} +{{- else if .Values.ingress.annotations -}} + {{- range $key, $value := .Values.ingress.annotations -}} + {{- if hasPrefix "cert-manager.io/" $key -}} + {{- $certManagerDetected = true -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- if $certManagerDetected -}} +true +{{- end -}} +{{- end -}} + +{{/* +Check if cert-manager is enabled by detecting cert-manager annotations in broker ingress annotations +Usage: {{ if include "forge.brokerCertManagerEnabled" . }} +*/}} +{{- define "forge.brokerCertManagerEnabled" -}} +{{- $certManagerDetected := false -}} +{{- if and .Values.forge.broker.enabled .Values.ingress.certManagerIssuer -}} + {{- $certManagerDetected = true -}} +{{- else if and .Values.forge.broker.enabled ((.Values.forge.broker).ingress).annotations -}} + {{- range $key, $value := .Values.forge.broker.ingress.annotations -}} + {{- if hasPrefix "cert-manager.io/" $key -}} + {{- $certManagerDetected = true -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- if $certManagerDetected -}} +true +{{- end -}} +{{- end -}} + +{{/* +Filter ingress annotations to remove cert-manager.io annotations when certManagerIssuer is set +Usage: {{ include "forge.filteredIngressAnnotations" . }} +*/}} +{{- define "forge.filteredIngressAnnotations" -}} +{{- $filtered := dict -}} +{{- if .Values.ingress.annotations -}} + {{- range $key, $value := .Values.ingress.annotations -}} + {{- if not (and $.Values.ingress.certManagerIssuer (hasPrefix "cert-manager.io/" $key)) -}} + {{- $_ := set $filtered $key $value -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- if $filtered -}} +{{- toYaml $filtered -}} +{{- end -}} +{{- end -}} + +{{/* +Filter broker ingress annotations to remove cert-manager.io annotations when certManagerIssuer is set +Usage: {{ include "forge.filteredBrokerIngressAnnotations" . }} +*/}} +{{- define "forge.filteredBrokerIngressAnnotations" -}} +{{- $filtered := dict -}} +{{- if and .Values.forge.broker.enabled ((.Values.forge.broker).ingress).annotations -}} + {{- range $key, $value := (.Values.forge.broker).ingress.annotations -}} + {{- if not (and $.Values.ingress.certManagerIssuer (hasPrefix "cert-manager.io/" $key)) -}} + {{- $_ := set $filtered $key $value -}} + {{- end -}} + {{- end -}} +{{- end -}} +{{- if $filtered -}} +{{- toYaml $filtered -}} +{{- end -}} +{{- end -}} \ No newline at end of file From 4a5c38a1af047826476c86b42ad088a2185c7759 Mon Sep 17 00:00:00 2001 From: ppawlowski Date: Mon, 6 Oct 2025 14:35:54 +0200 Subject: [PATCH 2/6] Handle annotations for all ingress objects --- helm/flowfuse/templates/broker-ingress.yaml | 9 +++++---- helm/flowfuse/templates/emqx.yaml | 11 ++++++----- helm/flowfuse/templates/service-ingress.yaml | 20 ++++++++++++-------- 3 files changed, 23 insertions(+), 17 deletions(-) diff --git a/helm/flowfuse/templates/broker-ingress.yaml b/helm/flowfuse/templates/broker-ingress.yaml index d86e9c99..5015871e 100644 --- a/helm/flowfuse/templates/broker-ingress.yaml +++ b/helm/flowfuse/templates/broker-ingress.yaml @@ -8,10 +8,11 @@ metadata: {{- include "forge.brokerSelectorLabels" . | nindent 4 }} annotations: {{- if .Values.ingress.certManagerIssuer }} - cert-manager.io/cluster-issuer: {{ $.Values.ingress.certManagerIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.ingress.certManagerIssuer }} {{- end }} - {{- if and .Values.forge.broker.enabled .Values.forge.broker.ingress (hasKey .Values.forge.broker.ingress "annotations") }} -{{ toYaml .Values.forge.broker.ingress.annotations | replace "{{ instanceHost }}" "{{ include forge.brokerDomain . }}" | replace "{{ serviceName }}" "flowforge-broker" | indent 4 }} + {{- $filteredAnnotations := include "forge.filteredBrokerIngressAnnotations" . | replace "{{ instanceHost }}" "{{ include forge.brokerDomain . }}" | replace "{{ serviceName }}" "flowforge-broker" }} + {{- if $filteredAnnotations }} +{{ $filteredAnnotations | indent 4 }} {{- end }} spec: {{- if $.Values.ingress.className }} @@ -28,7 +29,7 @@ spec: name: flowforge-broker port: number: 1884 - {{- if .Values.ingress.certManagerIssuer }} + {{- if include "forge.brokerCertManagerEnabled" . }} tls: - hosts: - {{ include "forge.brokerDomain" . }} diff --git a/helm/flowfuse/templates/emqx.yaml b/helm/flowfuse/templates/emqx.yaml index 2590a121..658c845f 100644 --- a/helm/flowfuse/templates/emqx.yaml +++ b/helm/flowfuse/templates/emqx.yaml @@ -102,11 +102,12 @@ metadata: {{- include "forge.brokerSelectorLabels" . | nindent 4 }} annotations: {{- if .Values.ingress.certManagerIssuer }} - cert-manager.io/cluster-issuer: {{ $.Values.ingress.certManagerIssuer }} - {{- end }} - {{- if and .Values.forge.broker.enabled .Values.forge.broker.ingress (hasKey .Values.forge.broker.ingress "annotations") }} -{{ toYaml .Values.forge.broker.ingress.annotations | replace "{{ instanceHost }}" "{{ include forge.brokerDomain . }}" | replace "{{ serviceName }}" "flowforge-broker" | indent 4 }} + cert-manager.io/cluster-issuer: {{ .Values.ingress.certManagerIssuer }} {{- end }} + {{- $filteredAnnotations := include "forge.filteredBrokerIngressAnnotations" . | replace "{{ instanceHost }}" "{{ include forge.brokerDomain . }}" | replace "{{ serviceName }}" "flowforge-broker" }} + {{- if $filteredAnnotations }} +{{ $filteredAnnotations | indent 4 }} + {{- end }} spec: {{- if $.Values.ingress.className }} ingressClassName: {{ $.Values.ingress.className }} @@ -122,7 +123,7 @@ spec: name: emqx-listeners port: number: 8080 - {{- if .Values.ingress.certManagerIssuer }} + {{- if include "forge.brokerCertManagerEnabled" . }} tls: - hosts: - {{ include "forge.brokerDomain" . }} diff --git a/helm/flowfuse/templates/service-ingress.yaml b/helm/flowfuse/templates/service-ingress.yaml index e8af5677..813fba9e 100644 --- a/helm/flowfuse/templates/service-ingress.yaml +++ b/helm/flowfuse/templates/service-ingress.yaml @@ -34,10 +34,11 @@ metadata: {{- include "forge.labels" . | nindent 4 }} annotations: {{- if .Values.ingress.certManagerIssuer }} - cert-manager.io/cluster-issuer: {{ $.Values.ingress.certManagerIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.ingress.certManagerIssuer }} {{- end }} - {{- if .Values.ingress.annotations }} -{{ toYaml .Values.ingress.annotations | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge" | indent 4 }} + {{- $filteredAnnotations := include "forge.filteredIngressAnnotations" . | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge" }} + {{- if $filteredAnnotations }} +{{ $filteredAnnotations | indent 4 }} {{- end }} spec: {{- if and $.Values.ingress.className (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} @@ -54,13 +55,13 @@ spec: name: forge port: number: 80 - {{- if .Values.ingress.certManagerIssuer }} + {{- if include "forge.certManagerEnabled" . }} tls: - hosts: - {{ $forgeHostname }} secretName: {{ $forgeHostname }} {{- end }} -{{- if gt (int .Values.forge.replicas) 1 -}} +{{- if gt (int .Values.forge.replicas) 1 }} --- apiVersion: networking.k8s.io/v1 kind: Ingress @@ -71,14 +72,17 @@ metadata: {{- include "forge.labels" . | nindent 4 }} annotations: {{- if .Values.ingress.certManagerIssuer }} - cert-manager.io/cluster-issuer: {{ $.Values.ingress.certManagerIssuer }} + cert-manager.io/cluster-issuer: {{ .Values.ingress.certManagerIssuer }} {{- end }} {{- if .Values.ingress.annotations }} nginx.ingress.kubernetes.io/affinity: cookie nginx.ingress.kubernetes.io/affinity-mode: persistent nginx.ingress.kubernetes.io/session-cookie-name: FFSESSION nginx.ingress.kubernetes.io/session-cookie-samesite: Strict -{{ toYaml .Values.ingress.annotations | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge" | indent 4 }} + {{- end }} + {{- $filteredAnnotations := include "forge.filteredIngressAnnotations" . | replace "{{ instanceHost }}" $forgeHostname | replace "{{ serviceName }}" "forge" }} + {{- if $filteredAnnotations }} +{{ $filteredAnnotations | indent 4 }} {{- end }} spec: {{- if and $.Values.ingress.className (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} @@ -95,7 +99,7 @@ spec: name: forge port: number: 80 - {{- if .Values.ingress.certManagerIssuer }} + {{- if include "forge.certManagerEnabled" . }} tls: - hosts: - {{ $forgeHostname }} From 1fc04beeb8a0ee69bb782aac42e7b0414dbf9078 Mon Sep 17 00:00:00 2001 From: ppawlowski Date: Mon, 6 Oct 2025 14:36:39 +0200 Subject: [PATCH 3/6] Add driver new k8s config options `projectIngressAnnotations` and `customHostname.ingressAnnotations` --- helm/flowfuse/templates/configmap.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/helm/flowfuse/templates/configmap.yaml b/helm/flowfuse/templates/configmap.yaml index 49e6f613..715f8046 100644 --- a/helm/flowfuse/templates/configmap.yaml +++ b/helm/flowfuse/templates/configmap.yaml @@ -56,6 +56,12 @@ data: {{ $key }}: {{ $value }} {{- end}} {{- end }} + {{- if .Values.forge.projectIngressAnnotations }} + projectIngressAnnotations: + {{- range $key, $value := .Values.forge.projectIngressAnnotations }} + {{ $key }}: {{ $value }} + {{- end}} + {{- end }} {{ if .Values.forge.projectServiceType }} service: type: {{ .Values.forge.projectServiceType }} @@ -84,6 +90,10 @@ data: {{- if .Values.forge.customHostname.ingressClass }} ingressClass: {{ .Values.forge.customHostname.ingressClass }} {{- end }} + {{- if .Values.forge.customHostname.ingressAnnotations }} + ingressAnnotations: +{{ toYaml .Values.forge.customHostname.ingressAnnotations | indent 12 }} + {{- end }} {{- end }} {{- if .Values.forge.persistentStorage }} storage: From 72043de7dab1af281d5a6063fc53061e5de0f4a6 Mon Sep 17 00:00:00 2001 From: ppawlowski Date: Mon, 6 Oct 2025 14:36:52 +0200 Subject: [PATCH 4/6] Add certManager-related unit tests --- helm/flowfuse/tests/ingress_test.yaml | 217 +++++++++++++++++++++++++- 1 file changed, 216 insertions(+), 1 deletion(-) diff --git a/helm/flowfuse/tests/ingress_test.yaml b/helm/flowfuse/tests/ingress_test.yaml index 3e310304..24f8fce6 100644 --- a/helm/flowfuse/tests/ingress_test.yaml +++ b/helm/flowfuse/tests/ingress_test.yaml @@ -104,4 +104,219 @@ tests: template: service-ingress.yaml documentSelector: path: metadata.name - value: flowforge-ingress \ No newline at end of file + value: flowforge-ingress + + - it: should create forge ingress with TLS when certManagerIssuer is set + template: service-ingress.yaml + documentSelector: + path: metadata.name + value: flowforge-ingress + set: + ingress.certManagerIssuer: "test-issuer" + asserts: + - equal: + path: metadata.annotations["cert-manager.io/cluster-issuer"] + value: "test-issuer" + - isNotNullOrEmpty: + path: spec.tls + - equal: + path: spec.tls[0].hosts[0] + value: "forge.chart-unit-tests.com" + - equal: + path: spec.tls[0].secretName + value: "forge.chart-unit-tests.com" + + - it: should create forge ingress with TLS section when cert-manager annotations are detected + template: service-ingress.yaml + documentSelector: + path: metadata.name + value: flowforge-ingress + set: + ingress.annotations: + cert-manager.io/issuer: "annotation-issuer" + nginx.ingress.kubernetes.io/ssl-redirect: "true" + asserts: + - equal: + path: metadata.annotations["cert-manager.io/issuer"] + value: "annotation-issuer" + - equal: + path: metadata.annotations["nginx.ingress.kubernetes.io/ssl-redirect"] + value: "true" + - isNotNullOrEmpty: + path: spec.tls + - equal: + path: spec.tls[0].hosts[0] + value: "forge.chart-unit-tests.com" + - equal: + path: spec.tls[0].secretName + value: "forge.chart-unit-tests.com" + + - it: should add TLS section when multiple cert-manager annotations are present + template: service-ingress.yaml + documentSelector: + path: metadata.name + value: flowforge-ingress + set: + ingress: + annotations: + cert-manager.io/issuer: "letsencrypt-prod" + cert-manager.io/acme-challenge-type: "http01" + cert-manager.io/acme-http01-edit-in-place: "true" + kubernetes.io/ingress.class: "nginx" + asserts: + - equal: + path: metadata.annotations["cert-manager.io/issuer"] + value: "letsencrypt-prod" + - equal: + path: metadata.annotations["cert-manager.io/acme-challenge-type"] + value: "http01" + - equal: + path: metadata.annotations["cert-manager.io/acme-http01-edit-in-place"] + value: "true" + - equal: + path: metadata.annotations["kubernetes.io/ingress.class"] + value: "nginx" + - isNotNullOrEmpty: + path: spec.tls + + - it: should NOT add TLS section when no cert-manager annotations are present + template: service-ingress.yaml + documentSelector: + path: metadata.name + value: flowforge-ingress + set: + ingress: + annotations: + kubernetes.io/ingress.class: "nginx" + nginx.ingress.kubernetes.io/ssl-redirect: "false" + asserts: + - equal: + path: metadata.annotations["kubernetes.io/ingress.class"] + value: "nginx" + - equal: + path: metadata.annotations["nginx.ingress.kubernetes.io/ssl-redirect"] + value: "false" + - notExists: + path: spec.tls + + - it: should NOT add TLS section when no annotations are set + template: service-ingress.yaml + documentSelector: + path: metadata.name + value: flowforge-ingress + asserts: + - isNullOrEmpty: + path: metadata.annotations + - notExists: + path: spec.tls + + - it: should use certManagerIssuer when both certManagerIssuer and cert-manager annotations are set + template: service-ingress.yaml + documentSelector: + path: metadata.name + value: flowforge-ingress + set: + ingress: + certManagerIssuer: "priority-issuer" + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + cert-manager.io/issuer: "secondary-issuer" + asserts: + - equal: + path: metadata.annotations["cert-manager.io/cluster-issuer"] + value: "priority-issuer" + - equal: + path: metadata.annotations["nginx.ingress.kubernetes.io/ssl-redirect"] + value: "true" + - notExists: + path: metadata.annotations["cert-manager.io/issuer"] + - isNotNullOrEmpty: + path: spec.tls + + - it: should create broker ingress with TLS when certManagerIssuer is set + template: broker-ingress.yaml + set: + forge: + broker: + enabled: true + teamBroker: + enabled: false + ingress.certManagerIssuer: "test-issuer" + asserts: + - equal: + path: metadata.annotations["cert-manager.io/cluster-issuer"] + value: "test-issuer" + - isNotNullOrEmpty: + path: spec.tls + - equal: + path: spec.tls[0].hosts[0] + value: "mqtt.chart-unit-tests.com" + - equal: + path: spec.tls[0].secretName + value: "mqtt.chart-unit-tests.com" + + - it: should add TLS section to broker ingress when cert-manager annotations are detected + template: broker-ingress.yaml + set: + forge: + broker: + enabled: true + ingress: + annotations: + cert-manager.io/cluster-issuer: "letsencrypt-prod" + teamBroker: + enabled: false + asserts: + - equal: + path: metadata.annotations["cert-manager.io/cluster-issuer"] + value: "letsencrypt-prod" + - isNotNullOrEmpty: + path: spec.tls + - equal: + path: spec.tls[0].hosts[0] + value: "mqtt.chart-unit-tests.com" + - equal: + path: spec.tls[0].secretName + value: "mqtt.chart-unit-tests.com" + + # Test broker ingress without cert-manager annotations + - it: should NOT add TLS section to broker ingress when no cert-manager annotations + template: broker-ingress.yaml + set: + forge: + broker: + enabled: true + ingress: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "false" + teamBroker: + enabled: false + asserts: + - notExists: + path: spec.tls + + - it: should use certManagerIssuer when both certManagerIssuer and cert-manager annotations are set for broker ingress + template: broker-ingress.yaml + set: + forge: + broker: + enabled: true + ingress: + annotations: + nginx.ingress.kubernetes.io/ssl-redirect: "true" + cert-manager.io/issuer: "letsencrypt-prod" + teamBroker: + enabled: false + ingress: + certManagerIssuer: "priority-issuer" + asserts: + - equal: + path: metadata.annotations["cert-manager.io/cluster-issuer"] + value: "priority-issuer" + - equal: + path: metadata.annotations["nginx.ingress.kubernetes.io/ssl-redirect"] + value: "true" + - notExists: + path: metadata.annotations["cert-manager.io/issuer"] + - isNotNullOrEmpty: + path: spec.tls From 4c7aa45673a09f579a7b20e3f5eacd8202d12d59 Mon Sep 17 00:00:00 2001 From: ppawlowski Date: Mon, 6 Oct 2025 14:37:31 +0200 Subject: [PATCH 5/6] Update values and values schema --- helm/flowfuse/values.schema.json | 5 ++++- helm/flowfuse/values.yaml | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/helm/flowfuse/values.schema.json b/helm/flowfuse/values.schema.json index aac3028a..8a8877ed 100644 --- a/helm/flowfuse/values.schema.json +++ b/helm/flowfuse/values.schema.json @@ -900,7 +900,7 @@ "logPassthrough": { "type": "boolean" }, - "customeHostname": { + "customHostname": { "type": "object", "properties": { "enabled": { @@ -914,6 +914,9 @@ }, "cnameTarget": { "type": "string" + }, + "ingressAnnotations": { + "type": "object" } } }, diff --git a/helm/flowfuse/values.yaml b/helm/flowfuse/values.yaml index 33d869ba..a7a2da2a 100644 --- a/helm/flowfuse/values.yaml +++ b/helm/flowfuse/values.yaml @@ -152,6 +152,7 @@ forge: logPassthrough: false customHostname: enabled: false + ingressAnnotations: {} assistant: {} From e73621653ab28eb5ea094ad89259428beeb19c05 Mon Sep 17 00:00:00 2001 From: ppawlowski Date: Mon, 6 Oct 2025 14:37:50 +0200 Subject: [PATCH 6/6] Update chart documentation --- helm/flowfuse/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/helm/flowfuse/README.md b/helm/flowfuse/README.md index 8e979089..14556ca9 100644 --- a/helm/flowfuse/README.md +++ b/helm/flowfuse/README.md @@ -227,6 +227,7 @@ One of either `storageClass` or `storageClassEFSTag` needs to be set. - `forge.customHostname.cnameTarget` the hostname of the ingress loadbalancer that custom hostnames must point to. Required (default not set) - `forge.customHostname.certManagerIssuer` name of CertManager ClusterIssuer to use to request HTTPS certificates for custom hostnames (default is not set) - `forge.customHostname.ingressClass` name of the IngressClass to use for exposing the custom hostname (default is not set) + - `forge.customHostname.ingressAnnotations` ingress annotations for custom hostname ingress (default is `{}`) ### Rate Limiting