From a8d26cd8bb12a12fb292459f2fa8264cbc4866b8 Mon Sep 17 00:00:00 2001 From: Panagiotatos Date: Tue, 23 Sep 2025 17:08:24 -0400 Subject: [PATCH 1/2] HPCC-34819 Document SSL/TLS setup for Containerized ESPs Signed-off-by: Panagiotatos --- .../ContainerizedMods/ConfigureValues.xml | 283 +++++++++++++++++- .../Inst-Mods/ssl-esp.xml | 33 +- 2 files changed, 308 insertions(+), 8 deletions(-) diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml index 11bdeec0cdc..e020e1a0e96 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml @@ -1195,8 +1195,287 @@ thor: Security Values - This section will look at the values.yaml - sections dealing with the system security components. + This section outlines the contents of the + values.yaml file pertaining to system security + components. + + + Certificates + + The certificates section can be used to enable the cert-manager + to generate TLS certificates for each component in the HPCC Systems + deployment. + + certificates: + enabled: false + issuers: + local: + name: hpcc-local-issuer + + In the delivered YAML file certificates are not enabled, as + illustrated above. You must first install the cert-manager to use this + feature. + + + + SSL/TLS Configuration for ESP + + Configuring SSL/TLS for ESP services in containerized + deployments differs significantly from bare-metal installations. In + Kubernetes environments, SSL/TLS can be implemented through multiple + approaches, each with different benefits and use cases. + + + SSL/TLS Implementation Approaches + + There are three primary approaches for implementing SSL/TLS in + containerized ESP deployments: + + + + Ingress Controller SSL + Termination - SSL is terminated at the ingress + controller level, with internal traffic remaining unencrypted. + This is the most common and recommended approach for most + deployments. + + + + ESP Pod-Level SSL - SSL + termination occurs within the ESP pod itself, requiring + certificates to be mounted into the container. This provides + end-to-end encryption but requires more complex certificate + management. + + + + Service Mesh SSL - + SSL/TLS is handled by a service mesh like Istio, providing + automatic certificate management and mTLS between + services. + + + + + + Ingress Controller SSL Termination + + The most straightforward approach is to configure SSL + termination at the ingress controller level. This approach leverages + Kubernetes ingress resources and integrates well with cert-manager + for automatic certificate provisioning. + + Example values.yaml configuration for ingress-based + SSL: + + esp: +- name: eclwatch + application: eclwatch + replicas: 1 + service: + port: 8010 + servicePort: 8010 + protocol: http + ingress: + enabled: true + annotations: + kubernetes.io/ingress.class: nginx + cert-manager.io/cluster-issuer: letsencrypt-prod + nginx.ingress.kubernetes.io/ssl-redirect: "true" + hosts: + - host: eclwatch.example.com + paths: + - path: / + pathType: Prefix + tls: + - secretName: eclwatch-tls + hosts: + - eclwatch.example.com + +certificates: + enabled: true + issuers: + letsencrypt-prod: + name: letsencrypt-prod + spec: + acme: + server: https://acme-v02.api.letsencrypt.org/directory + email: admin@example.com + privateKeySecretRef: + name: letsencrypt-prod + solvers: + - http01: + ingress: + class: nginx + + This configuration: + + + + Enables SSL termination at the NGINX ingress + controller + + + + Uses cert-manager with Let's Encrypt for automatic + certificate provisioning + + + + Redirects all HTTP traffic to HTTPS + + + + Stores the certificate in a Kubernetes secret named + 'eclwatch-tls' + + + + + + ESP Pod-Level SSL Configuration + + For scenarios requiring end-to-end encryption or when ingress + controllers cannot be used, SSL can be configured directly at the + ESP pod level. This approach requires mounting certificates into the + ESP containers and configuring ESP to use HTTPS. + + First, create a Kubernetes secret containing your SSL + certificates: + + kubectl create secret tls esp-tls-secret \ + --cert=path/to/certificate.crt \ + --key=path/to/private.key + + Then configure ESP to use SSL in your values.yaml: + + esp: +- name: eclwatch + application: eclwatch + replicas: 1 + service: + port: 18010 + servicePort: 18010 + protocol: https + https: + enabled: true + certificateSecretName: esp-tls-secret + # Optional: specify passphrase for encrypted private keys + # passphrase: "your-passphrase" + volumes: + - name: esp-certs + secret: + secretName: esp-tls-secret + volumeMounts: + - name: esp-certs + mountPath: /etc/ssl/esp + readOnly: true + + Important Security + Notes: + + + + Certificate files should be owned by the container's hpcc + user with appropriate permissions + + + + Private keys should have restrictive permissions (0400 or + 0600) + + + + Use Kubernetes secrets for certificate storage rather than + embedding in container images + + + + Consider using init containers to set proper file + permissions if needed + + + + + + Integration with cert-manager + + When using cert-manager for automatic certificate management, + you can configure ESP services to automatically receive and use + certificates. This works with both ingress-level and pod-level SSL + configurations. + + Example cert-manager Certificate resource for ESP: + + apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: esp-certificate + namespace: hpcc +spec: + secretName: esp-tls-secret + issuerRef: + name: letsencrypt-prod + kind: ClusterIssuer + dnsNames: + - eclwatch.example.com + - ws-ecl.example.com + + This Certificate resource will automatically provision and + renew SSL certificates, storing them in the specified Kubernetes + secret for use by ESP pods. + + + + SSL/TLS Troubleshooting + + Common issues and solutions when configuring SSL/TLS for ESP + in containerized deployments: + + + + Certificate Not Found + + + Verify the certificate secret exists in the correct + namespace and contains both 'tls.crt' and 'tls.key' keys. Use + kubectl describe secret <secret-name> + to inspect. + + + + + Permission Denied + + + Ensure certificate files are readable by the hpcc user + in the container. Consider using an init container to set + proper ownership and permissions. + + + + + cert-manager Issues + + + Check cert-manager logs and Certificate resource status. + Ensure DNS is properly configured for ACME challenges when + using Let's Encrypt. + + + + + Ingress SSL Redirect Loops + + + Verify ingress annotations for SSL redirect are + correctly configured. Check if the ingress controller is + properly handling SSL termination. + + + + + Secrets diff --git a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml index 67d3abd912d..194296e5452 100644 --- a/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml +++ b/docs/EN_US/Installing_and_RunningTheHPCCPlatform/Inst-Mods/ssl-esp.xml @@ -44,12 +44,7 @@ is encrypted and secure. The Public and Private Keys use 2048-bit RSA encryption. - These server keys are read at runtime by the ESP process. It is - important the installed keys have correct ownership and permissions. - Typically, it is the HPCC user and their public key (certificate.cer) with - read permissions such as 0444 (or 0644), along with the private key - (privatekey.cer) with more restrictive permissions of 0400 (or - 0600). + These server keys (whether they are self-signed or imported) are read at runtime by the ESP process. For security reasons, it is important the installed keys have correct ownership and permissions. The certificate (the public key) should be owned by user hpcc with permissions 0444 (or 0644); the private key should be owned by user hpcc with permissions 0400 (or 0600). <emphasis role="bold">Generate an RSA Private @@ -383,6 +378,32 @@ http. You should then repeat the process for <emphasis role="bold">all</emphasis> other service bindings.</para> </sect3> + + <sect3 id="ConvertingFromHTTPtoHTTPS"> + <title><emphasis role="bold">Converting an Existing Cluster from HTTP to HTTPS</emphasis> + + If your goal is to convert an existing unsecured cluster to a fully secured cluster (rather than just adding SSL support alongside existing HTTP services), you can simplify the process by converting existing service bindings instead of creating new ones. + + + + In the ESP Service Bindings tab, locate each existing HTTP service binding. + + + + For each service binding, change the protocol from http to https in the protocol drop-list. + + + + The port will automatically update to the default HTTPS port (e.g., 18002 for ws_ecl) if it hasn't been previously edited. + + + + Click the disk icon to save each change. + + + + This approach is more efficient than creating new HTTPS bindings and then deleting the HTTP ones, especially when converting multiple services in a production environment. + From 421317d714e8e9ee11e70db158df3c739ecc1450 Mon Sep 17 00:00:00 2001 From: Terrence Asselin Date: Tue, 28 Oct 2025 16:52:40 -0500 Subject: [PATCH 2/2] HPCC-34819 TLS Config Documentation Update Significant change from the Copilot-driven original, removing examples that didn't best capture the platform supported features and included examples of Helm template output rather than values.yaml configuration. - Restructured to focus on the three best-supported TLS configuration scenarios. - Includes much material from helm/examples and references some files in that location for CLI steps in examples. - Moved the 'rebased' Certificates section to be a child under the Security Values section. - Even though they were maybe out of scope for the ticket, kept the Secrets Vault and CORS sections. Made mimimal changes to improve the readability and helpfulness, but didn't give the overhaul they likely need. This proposed update will likely need changes for tone, style and readability. Signed-off-by: Terrence Asselin --- .../ContainerizedMods/ConfigureValues.xml | 1369 ++++++++++------- 1 file changed, 794 insertions(+), 575 deletions(-) diff --git a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml index e020e1a0e96..2a1389cc846 100644 --- a/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml +++ b/docs/EN_US/ContainerizedHPCC/ContainerizedMods/ConfigureValues.xml @@ -1199,231 +1199,430 @@ thor: values.yaml file pertaining to system security components. - - Certificates + + TLS Configuration for Platform Components - The certificates section can be used to enable the cert-manager - to generate TLS certificates for each component in the HPCC Systems - deployment. + Configuring TLS for secure communication between platform components in containerized + deployments differs significantly from bare-metal installations. + Kubernetes environments allow flexibility when configuring for TLS, + but this section addresses the methods most directly supported by the + HPCC Systems Helm chart: + - certificates: - enabled: false - issuers: - local: - name: hpcc-local-issuer + + + Cluster mTLS: Mutual TLS for trust between components within a cluster + + + Remote mTLS: Establishing a zone of trust with mTLS between components across clusters + + + TLS Using Public CA: Use a public Certificate Authority to establish trust between + external clients and a publically-exposed component such as ECL Watch + + - In the delivered YAML file certificates are not enabled, as - illustrated above. You must first install the cert-manager to use this - feature. - + cert-manager is used In all these configurations to manage the Kubernetes + certificate-related resources. Vault is used as the PKI (Public Key Infrastructure) + when establishing trust between components in separate HPCC clusters. + - - SSL/TLS Configuration for ESP + To enable TLS for component make these configuration changes: + + + + Set the certificates.enabled property to + true + + + Set the enabled property to true + for the specific issuer you want to use: local, public or remote + + + Set the component's visibility property to the value that + assigns it the issuer you want to use: `cluster` visibility for `local` issuer, `local` or `global` visibility + for the `public` issuer. + + + To use the `remote` issuer, set the component's `visibility` property to + `local` or `global`, and it's `trustClients` array to the list of remote + trusted clients. + + - Configuring SSL/TLS for ESP services in containerized - deployments differs significantly from bare-metal installations. In - Kubernetes environments, SSL/TLS can be implemented through multiple - approaches, each with different benefits and use cases. + + Although an effort was made to standardize on the term TLS + throughout this section, there may be references to the older SSL + standard it replaced. Any SSL references should be taken as + colloquial or legacy nomencalature that actually refers to a TLS + implementation. + - - SSL/TLS Implementation Approaches + The following examples are to exhibit basic functionality in a + testing environment- production deployments may require different + settings. The instructions reference paths that are relative to the + root of the HPCC-Platform repository- adjust as needed for your + environment. + + + A prerequisite to all these examples is installing cert-manager to your + Kubernetes cluster: + + + + Add Jetstack helm repo: + + ```bash + helm repo add jetstack https://charts.jetstack.io + ``` + + Install cert-manager. + + ```bash + helm install cert-manager jetstack/cert-manager --set crds.enabled=true --namespace cert-manager --create-namespace + ``` + + + + Cluster mTLS + + This approach requires creating a root certificate and private key for + a local cluster certificate authority, then mounting them as Kubernetes secrets + for cert-manager to use. + + Create a root certificate and private key for our local cluster certificate authority with + a single openssl call. This call uses a sample openssl config file found in the repo. + + + ```bash + openssl req -x509 -newkey rsa:2048 -nodes -keyout ca.key -sha256 -days 1825 -out ca.crt -config helm/examples/certmanager/ca-req.cfg + ``` + + The root certificate needs to be added as a kubernetes secret in order to be + accessible to cert-manager. The secret name matches the default name used in the + local issuer configuration in values.yaml. + + ```bash + kubectl create secret tls hpcc-local-issuer-key-pair --cert=ca.crt --key=ca.key + ``` + + Repeat the root certificate creation process to create the CA secret for the code + signing issuer. This is needed to run the ecl testing script at the end to show the + example is working. + + ```bash + openssl req -x509 -newkey rsa:2048 -nodes -keyout signingca.key -sha256 -days 1825 -out signingca.crt -config examples/certmanager/ca-req.cfg + ``` + + Create a Kubernetes TLS secret from the generated signing root certificate and privatekey + + ```bash + kubectl create secret tls hpcc-signing-issuer-key-pair --cert=signingca.crt --key=signingca.key + ``` + + Install the HPCC helm chart with the "--set certificates.enabled" option set to true. + + ```bash + helm install myhpcc hpcc/hpcc --set certificates.enabled=true + ``` + + The cluster ESPs are now using TLS both locally and publicly. Verify it is working + by running an ecl job that requires using mutual TLS (using local client certificate): + + ``` + ecl run --ssl hthor helm/examples/certmanager/localhttpcall.ecl + ``` + + Note that for the HTTPCALL in our ecl example the url now starts with "mtls:" + this tells HTTPCALL/SOAPCALL to use mutual TLS, using the local client certificate, + and to verify the server using the local certificate authority certificate. + + You should see a result similar to this, possibly with some additional headers: + + ```xml + + + + GET + /WsSmc/HttpEcho + name=doe,joe&number=1 + +
Accept-Encoding: gzip, deflate
+
Accept: text/xml
+ + + + + ``` + - There are three primary approaches for implementing SSL/TLS in - containerized ESP deployments: + + Remote Mutual TLS - - - Ingress Controller SSL - Termination - SSL is terminated at the ingress - controller level, with internal traffic remaining unencrypted. - This is the most common and recommended approach for most - deployments. - + This walkthrough demonstrates using a single Hashicorp Vault + PKI Certificate authority to establish a zone of trust between two separate + HPCC environments. cert-manager continues to handle the backing Kubernetes + resources. For this example each HPCC environment is in a separate kubernetes + namespace: hpcc1 and hpcc2, but in production they could be in different + Kubernetes clusters, or even in different cloud subscriptions. - - ESP Pod-Level SSL - SSL - termination occurs within the ESP pod itself, requiring - certificates to be mounted into the container. This provides - end-to-end encryption but requires more complex certificate - management. - + + Here we have hpcc2.roxie making a call to hpcc1.roxie in another + environment, and we want them to trust each other. There are two main configuration + customizations needed to set up this zone + of trust. Each of the two HPCC environments have these changes. - - Service Mesh SSL - - SSL/TLS is handled by a service mesh like Istio, providing - automatic certificate management and mTLS between - services. - - - + First, the remote issuer is enabled and its spec set to use the Vault. + Both HPCC environments share the same Vault server, which allows them to use + the same underlying certificates to establish trush. Note alseo that the + ca property must be set to null to override the default. - - Ingress Controller SSL Termination + + spec: + ca: null + vault: + path: pki/sign/hpccremote + server: http://vault.vaultns:8200 + auth: + tokenSecretRef: + name: cert-manager-vault-token + key: token + - The most straightforward approach is to configure SSL - termination at the ingress controller level. This approach leverages - Kubernetes ingress resources and integrates well with cert-manager - for automatic certificate provisioning. + Second, each roxie must be told to trust the other. In the hpcc2 values file the + roxie service has its trustClients array set to include hpcc1.roxie, and likewise the + hpcc1 values file has its roxie service trustClients array set to include hpcc2.roxie. + These files can be found in the repo at helm/examples/vault-pki-remote/values-hpcc2.yaml + and helm/examples/vault-pki-remote/values-hpcc1.yaml. + Here are the relevant parts of the hpcc2 values file: + - Example values.yaml configuration for ingress-based - SSL: + + roxie: + - name: roxie2 + disabled: false + services: + - name: roxie2 + servicePort: 29876 + visibility: local + trustClients: + - commonName: roxie1.hpcc1 + - esp: -- name: eclwatch - application: eclwatch - replicas: 1 - service: - port: 8010 - servicePort: 8010 - protocol: http - ingress: - enabled: true - annotations: - kubernetes.io/ingress.class: nginx - cert-manager.io/cluster-issuer: letsencrypt-prod - nginx.ingress.kubernetes.io/ssl-redirect: "true" - hosts: - - host: eclwatch.example.com - paths: - - path: / - pathType: Prefix - tls: - - secretName: eclwatch-tls - hosts: - - eclwatch.example.com + + Install Hashicorp vault service in dev mode -certificates: - enabled: true - issuers: - letsencrypt-prod: - name: letsencrypt-prod - spec: - acme: - server: https://acme-v02.api.letsencrypt.org/directory - email: admin@example.com - privateKeySecretRef: - name: letsencrypt-prod - solvers: - - http01: - ingress: - class: nginx - - This configuration: - - - - Enables SSL termination at the NGINX ingress - controller - + This is for development only, never deploy this way in + production. Deploying in dev mode sets up an in memory kv store + that won't persist secret values across restart, and the vault will + automatically be unsealed. - - Uses cert-manager with Let's Encrypt for automatic - certificate provisioning - + In dev mode the default root token is simply the string + "root". - - Redirects all HTTP traffic to HTTPS - + Add Hashicorp helm repo: - - Stores the certificate in a Kubernetes secret named - 'eclwatch-tls' - - - + helm repo add hashicorp https://helm.releases.hashicorp.com - - ESP Pod-Level SSL Configuration + Update Helm repos. - For scenarios requiring end-to-end encryption or when ingress - controllers cannot be used, SSL can be configured directly at the - ESP pod level. This approach requires mounting certificates into the - ESP containers and configuring ESP to use HTTPS. + helm repo update - First, create a Kubernetes secret containing your SSL - certificates: + Install vault server. - kubectl create secret tls esp-tls-secret \ - --cert=path/to/certificate.crt \ - --key=path/to/private.key + When using developer mode vault you have to set the + VAULT_DEV_LISTEN_ADDRESS environment variable as shown in order to + access the vault service from an external pod. - Then configure ESP to use SSL in your values.yaml: + helm install vault hashicorp/vault --set "injector.enabled=false" --set "server.dev.enabled=true" --set 'server.extraEnvironmentVars.VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200' --namespace vaultns --create-namespace - esp: -- name: eclwatch - application: eclwatch - replicas: 1 - service: - port: 18010 - servicePort: 18010 - protocol: https - https: - enabled: true - certificateSecretName: esp-tls-secret - # Optional: specify passphrase for encrypted private keys - # passphrase: "your-passphrase" - volumes: - - name: esp-certs - secret: - secretName: esp-tls-secret - volumeMounts: - - name: esp-certs - mountPath: /etc/ssl/esp - readOnly: true - - Important Security - Notes: - - - - Certificate files should be owned by the container's hpcc - user with appropriate permissions - + Check the pods: - - Private keys should have restrictive permissions (0400 or - 0600) - + kubectl get pods -n vaultns - - Use Kubernetes secrets for certificate storage rather than - embedding in container images - + Vault pods should now be running and ready. + - - Consider using init containers to set proper file - permissions if needed - - - + + Setting up vault + + Tell the vault command line application the server location + (dev mode is http, default location is https) + + export VAULT_ADDR=http://127.0.0.1:8200 + + Export an environment variable for the vault CLI to + authenticate with the Vault server. As mentioned before since we + installed dev mode, the vault token is the string 'root'. + + export VAULT_TOKEN=root + + In a separate terminal window start vault port + forwarding. + + kubectl port-forward vault-0 8200:8200 -n vaultns + + Login to the vault command line using the vault root token + (development mode defaults to "root"): + + vault login root + + Enable the PKI secrets engine at its default path. + + vault secrets enable pki + + Configure the max lease time-to-live (TTL) to 8760h. + + vault secrets tune -max-lease-ttl=87600h pki + + Generate the HPCC remote issuer CA, giving it the name + hpcc-remote-issuer This name will be used to identify it + in the platform configuration. + + vault write -field=certificate pki/root/generate/internal common_name="hpcc-issuer" issuer_name="hpcc-remote-issuer" ttl=87600h + + Configure the PKI secrets engine certificate issuing and + certificate revocation list (CRL) endpoints to use the Vault + service in the "vaultns" namespace. + + vault write pki/config/urls issuing_certificates="http://vault.vaultns:8200/v1/pki/ca" crl_distribution_points="http://vault.vaultns:8200/v1/pki/crl" + + For our local mTLS certificates we will use our Kubernetes + namespace as our domain name. This will allow us to recognize where + these components reside. For our public TLS certificates for this + demo we will use myhpcc.com as our domain. + + Configure a role named hpccremote that enables the creation + of certificates in the hpcc1 and hpcc2 + domains with any subdomains. + + vault write pki/roles/hpccremote key_type=any allowed_domains="hpcc1,hpcc2" allow_subdomains=true allowed_uri_sans="spiffe://*" max_ttl=72 + + Create a policy named pki that enables read access to the PKI + secrets engine paths. + + vault policy write hpcc-remote-pki - <<EOF +path "pki*" { capabilities = ["read", "list"] } +path "pki/roles/hpccremote" { capabilities = ["create", "update"] } +path "pki/sign/hpccremote" { capabilities = ["create", "update"] } +path "pki/issue/hpccremote" { capabilities = ["create"] } +EOF + + + + Install the first Platform cluster + + Create the hpcc1 namespace. + + kubectl create namespace hpcc1 + + The local and signing issuers are isolated and won't be using + Vault. Create the unique secrets for both issuers in the hpcc1 namespace. + + + openssl req -x509 -newkey rsa:2048 -nodes -keyout hpcc1local.key -sha256 -days 1825 -out hpcc1local.crt -config helm/examples/vault-pki-remote/local-ca-req.cfg +kubectl create secret tls hpcc-local-issuer-key-pair --cert=hpcc1local.crt --key=hpcc1local.key -n hpcc1 + +openssl req -x509 -newkey rsa:2048 -nodes -keyout hpcc1signing.key -sha256 -days 1825 -out hpcc1signing.crt -config helm/examples/vault-pki-remote/signing-ca-req.cfg +kubectl create secret tls hpcc-signing-issuer-key-pair --cert=hpcc1signing.crt --key=hpcc1signing.key -n hpcc1 + + The remote issuer does use Vault. Create the secret the remote + issuer that hpcc1 will use to access the Vault pki engine + + kubectl create secret generic cert-manager-vault-token --from-literal=token=root -n hpcc1 + + helm install myhpcc hpcc/hpcc --values helm/examples/vault-pki-remote/values-hpcc1.yaml -n hpcc1 + + Use kubectl to check the status of the deployed pods. Wait + until all pods are running before continuing. + + kubectl get pods -n hpcc1 + + Check and see if the certificate issuers have been + successfully created: + + kubectl get issuers -o wide + + + + Install the second Platform cluster + + Create the hpcc2 namespace + + kubectl create namespace hpcc2 + + The local and signing issuers are isolated and won't be using + vault. Create the unique secrets for both issuers in the hpcc2 namespace. + + + openssl req -x509 -newkey rsa:2048 -nodes -keyout hpcc2local.key -sha256 -days 1825 -out hpcc2local.crt -config helm/examples/vault-pki-remote/local-ca-req.cfg +kubectl create secret tls hpcc-local-issuer-key-pair --cert=hpcc2local.crt --key=hpcc2local.key -n hpcc2 + +openssl req -x509 -newkey rsa:2048 -nodes -keyout hpcc2signing.key -sha256 -days 1825 -out hpcc2signing.crt -config helm/examples/vault-pki-remote/signing-ca-req.cfg +kubectl create secret tls hpcc-signing-issuer-key-pair --cert=hpcc2signing.crt --key=hpcc2signing.key -n hpcc2 + + The remote issuer does use vault. Create the secret the remote + issuer that hpcc1 will use to access the vault pki engine + + kubectl create secret generic cert-manager-vault-token --from-literal=token=root -n hpcc2 + + helm install myhpcc hpcc/hpcc --values helm/examples/vault-pki-remote/values-hpcc2.yaml -n hpcc2 + + Use kubectl to check the status of the deployed pods. Wait + until all pods are running before continuing. + + kubectl get pods -n hpcc2 + + Check and see if the certificate issuers have been + successfully created: + + kubectl get issuers -o wide + + + + ECL example demonstrating trust + + Now we can ECL in each environment that will communicate + securely to the other. - - Integration with cert-manager - - When using cert-manager for automatic certificate management, - you can configure ESP services to automatically receive and use - certificates. This works with both ingress-level and pod-level SSL - configurations. - - Example cert-manager Certificate resource for ESP: - - apiVersion: cert-manager.io/v1 -kind: Certificate -metadata: - name: esp-certificate - namespace: hpcc -spec: - secretName: esp-tls-secret - issuerRef: - name: letsencrypt-prod - kind: ClusterIssuer - dnsNames: - - eclwatch.example.com - - ws-ecl.example.com - - This Certificate resource will automatically provision and - renew SSL certificates, storing them in the specified Kubernetes - secret for use by ESP pods. + roxie_echo.ecl which returns a dataset passed into it. + remote_echo.ecl which calls roxie_echo.ecl. + + For this example we will: + + + + publish roxie_echo.ecl to the hpcc1 namespace. It returns + the dataset passed into it. + + + + Publish remote_echo.ecl to the hpcc2 namespace. It calls + roxie_echo.ecl in the hpcc1 namespace using remote-mtls. This + is accomplished by using the `remote-mtls:` prefix in the url + to signal that the call should be made using the certificate + associated with the `remote` issuer. + + + + Use hpcc2::remote_echo.ecl to call + hpcc1::roxie_echo.ecl. + + + + Publish the queries: + + ecl publish roxie1 --ssl --port 18010 roxie_echo.ecl +ecl publish roxie2 --ssl --port 28010 remote_echo.ecl + + Call the query and demonstrate trust + + You can navigate to EclQueries/WsEcl on port 28002 in your + browser and run the `remote_echo` query from there. You should see + results similar to the following: + + {"remote_echoResponse": {"sequence": 0, "Results": {"remoteResult": {"Row": [{"Dataset": {"Row": [{"name": {"first": "Joeseph", "last": "Johnson"}, "address": {"city": "Fresno", "state": "CA", "zipcode": "11111"}}, {"name": {"first": "Joeseph", "last": "Johnson"}, "address": {"city": "Fresno", "state": "CA", "zipcode": "22222"}}, {"name": {"first": "Joeseph", "last": "Johnson"}, "address": {"city": "Fresno", "state": "CA", "zipcode": "33333"}}, {"name": {"first": "Joeseph", "last": "Johnson"}, "address": {"city": "Fresno", "state": "CA", "zipcode": "44444"}}, {"name": {"first": "Joeseph", "last": "Johnson"}, "address": {"city": "Fresno", "state": "CA", "zipcode": "55555"}}]}, "Exception": {"Code": "0"}}]}}}}} + @@ -1463,174 +1662,129 @@ spec: using Let's Encrypt. - - - Ingress SSL Redirect Loops - - - Verify ingress annotations for SSL redirect are - correctly configured. Check if the ingress controller is - properly handling SSL termination. - - - - - - Secrets - - The Secrets section contains a set of categories, each of which - contain a list of secrets. The Secrets section is where to get info - into the system if you don't want it in the source. Such as code with - embedded code, you can have that defined in the code sign sections. If - you have information that you don't want public but need to run it you - could use secrets. There is a category named "eclUser" which is where - you would put secrets that you want to be readable directly from ECL - code. Other secret categories, including the "ecl" category, are read - internally by system components and not exposed directly to ECL - code. - - - - Vaults - Vaults is another way to do Secrets. The vaults section mirrors - the secret section but leverages HashiCorp Vault - for the storage of secrets. There is a category for vaults named - "eclUser". The intent of the eclUser vault category is to be readable - directly from ECL code. Only add vault configurations to the "eclUser" - category that you want ECL users to be able to access. Other vault - categories, including the "ecl" category, are read internally by - system components and not exposed directly to ECL code. - + + TLS Using Public CA - - Cross Origin Resource Handling + The HPCC Systems Helm chart templates support using a Public CA issuer + such as Let's Encrypt or ZeroSSL through cert-manager, but most of the work + is external to configuring the platform itself and is out of scope for this + document. This section outlines the configuration changes needed in the + platform to make use of a Public CA issuer setup outside the platform + deployment process. - Cross-origin resource sharing (CORS) is a mechanism for - integrating applications in different domains. CORS defines how client - web applications in one domain can interact with resources in another - domain. You can configure CORS support settings in the ESP section of - the values.yaml file as illustrated below: + Assume you're using Let's Encrypt as your Public CA issuer, and you've + named it letsencrypt-prod. Then set the platform's + public issuer to use that name: + + + certificates: + issuers: + public: + name: letsencrypt-prod + kind: ClusterIssuer + - esp: -- name: eclwatch - application: eclwatch - auth: ldap - replicas: 1 - # The following 'corsAllowed' section is used to configure CORS support - # origin - the origin to support CORS requests from - # headers - the headers to allow for the given origin via CORS - # methods - the HTTP methods to allow for the given origin via CORS - # - corsAllowed: - # origin starting with https will only allow https CORS - - origin: https://*.example2.com - headers: - - "X-Custom-Header" - methods: - - "GET" - # origin starting with http will allow http or https CORS - - origin: http://www.example.com - headers: - - "*" - methods: - - "GET" - - "POST" + + ClusterIssuer is usually preferred for Public CA + configurations to centralize certificate management across all namespaces + in a cluster and help avoid hitting Public CA usage limits. + + - - - Certificates + + Certificates - HPCC Systems containerized deployments support comprehensive - certificate management through Helm chart configuration. This includes - support for multiple certificate issuers, domain management, and - advanced certificate features for secure communication between - components. + HPCC Systems containerized deployments support comprehensive + certificate management through Helm chart configuration. This includes + support for multiple certificate issuers, domain management, and + advanced certificate features for secure communication between + components. - The certificate system integrates with cert-manager to provide - automated certificate generation and renewal for both internal (local, - i.e., communication between HPCC components within the same cluster or - namespace) and external (public/remote) communications. + The certificate system integrates with cert-manager to provide + automated certificate generation and renewal for both internal (local, + i.e., communication between HPCC components within the same cluster or + namespace) and external (public/remote) communications. - - Enable Certificates + + Enable Certificates - Use the certificates section to enable cert-manager to generate - TLS certificates for each component in the HPCC Systems - deployment. + Use the certificates section to enable cert-manager to generate + TLS certificates for each component in the HPCC Systems + deployment. - certificates: + certificates: enabled: false issuers: local: name: hpcc-local-issuer - In the delivered YAML file certificates are not enabled, as - illustrated above. You must first install the cert-manager to use this - feature. - + In the delivered YAML file certificates are not enabled, as + illustrated above. You must first install the cert-manager to use this + feature. + - - Certificate Issuers Configuration + + Certificate Issuers Configuration - Certificate issuers are configured in the - certificates.issuers section of the Helm values. - The system supports multiple issuer types: + Certificate issuers are configured in the + certificates.issuers section of the Helm values. + The system supports multiple issuer types: - - - local: For internal - component communication - + + + local: For internal + component communication + - - public: For external-facing - services - + + public: For external-facing + services + - - remote: For remote client - connections - + + remote: For remote client + connections + - - signing: For code signing - certificates - - + + signing: For code signing + certificates + + - - Remote Issuer Configuration + + Remote Issuer Configuration - The remote issuer is specifically designed - for generating certificates that can be used by remote clients to - securely connect to HPCC services. This issuer supports advanced - domain configuration options. - + The remote issuer is specifically designed + for generating certificates that can be used by remote clients to + securely connect to HPCC services. This issuer supports advanced + domain configuration options. + - - Alternative Domains Support + + Alternative Domains Support - The certificates.issuers.remote section - supports an alternativeDomains array - configuration option. This feature allows certificates to be valid - for multiple domains in addition to the primary domain specified in - the issuer configuration. + The certificates.issuers.remote section + supports an alternativeDomains array + configuration option. This feature allows certificates to be valid + for multiple domains in addition to the primary domain specified in + the issuer configuration. - Configuration Syntax + Configuration Syntax - Each entry in the alternativeDomains array - represents an additional domain that will be included in the - dnsNames property of the generated Certificate - resource manifest. + Each entry in the alternativeDomains array + represents an additional domain that will be included in the + dnsNames property of the generated Certificate + resource manifest. - - Remote Issuer with Alternative Domains + + Remote Issuer with Alternative Domains - + certificates: enabled: true issuers: @@ -1643,189 +1797,189 @@ certificates: - subdomain2.example.com - api.example.org - secure.mycompany.net - - + + - In this configuration: + In this configuration: - + + + The primary domain is + example.com + + + + Certificates will also be valid for all domains listed in + alternativeDomains + + + + The generated Certificate resource will include all + domains in its dnsNames section + + + + + + + Certificate Generation Process + + When the Helm chart is deployed with certificate generation + enabled, the system automatically creates Certificate resources based + on the issuer configuration. For remote issuers with alternative + domains: + + - The primary domain is - example.com + Domain Collection: The + system collects the primary domain and all alternative domains + from the issuer configuration - Certificates will also be valid for all domains listed in - alternativeDomains + DNS Names Generation: + Service names are combined with each domain to create + comprehensive DNS name lists - The generated Certificate resource will include all - domains in its dnsNames section + Certificate Creation: + cert-manager generates certificates valid for all specified domain + combinations - - - - - Certificate Generation Process + + Secret Storage: Generated + certificates are stored in Kubernetes secrets for use by HPCC + components + + - When the Helm chart is deployed with certificate generation - enabled, the system automatically creates Certificate resources based - on the issuer configuration. For remote issuers with alternative - domains: + + Generated Certificate Properties - - - Domain Collection: The - system collects the primary domain and all alternative domains - from the issuer configuration - + Certificates generated with alternative domains will + contain: - - DNS Names Generation: - Service names are combined with each domain to create - comprehensive DNS name lists - + + + Subject Alternative Names (SAN) - - Certificate Creation: - cert-manager generates certificates valid for all specified domain - combinations - + + All combinations of service names with the primary + domain and alternative domains + + - - Secret Storage: Generated - certificates are stored in Kubernetes secrets for use by HPCC - components - - + + Common Name - - Generated Certificate Properties + + Typically set to the service name combined with the + primary domain + + - Certificates generated with alternative domains will - contain: + + Validity Period - - - Subject Alternative Names (SAN) + + Default 90-day validity with automatic renewal before + expiration + + - - All combinations of service names with the primary - domain and alternative domains - - + + Usage - - Common Name + + Configured for both server authentication and client + authentication + + + + + - - Typically set to the service name combined with the - primary domain - - + + Use Cases and Best Practices - - Validity Period + + Multi-Domain Certificate Management + The alternativeDomains feature is the + preferred method for configuring certificates that need to be valid + across multiple domains and subdomains. Common use cases + include: + + - Default 90-day validity with automatic renewal before - expiration + Multi-tenant Deployments: + Supporting multiple customer domains from a single HPCC + deployment - - - - Usage - Configured for both server authentication and client - authentication + Load Balancer + Integration: Certificates valid for both direct + service access and load balancer endpoints - - - - - - - Use Cases and Best Practices - - - Multi-Domain Certificate Management - - The alternativeDomains feature is the - preferred method for configuring certificates that need to be valid - across multiple domains and subdomains. Common use cases - include: - - - Multi-tenant Deployments: - Supporting multiple customer domains from a single HPCC - deployment - - - - Load Balancer - Integration: Certificates valid for both direct - service access and load balancer endpoints - - - - Development and - Production: Single certificates valid across multiple - environment domains - + + Development and + Production: Single certificates valid across multiple + environment domains + - - Legacy Migration: - Supporting both old and new domain names during migration - periods - - - + + Legacy Migration: + Supporting both old and new domain names during migration + periods + + + - - Configuration Best Practices + + Configuration Best Practices - - - Domain Validation: Ensure - all domains in alternativeDomains are - properly configured in DNS before certificate generation - + + + Domain Validation: Ensure + all domains in alternativeDomains are + properly configured in DNS before certificate generation + - - Certificate Authority - Limits: Be aware of rate limits imposed by - certificate authorities when using multiple domains - + + Certificate Authority + Limits: Be aware of rate limits imposed by + certificate authorities when using multiple domains + - - Security Considerations: - Only include domains that should legitimately be covered by the - same certificate - + + Security Considerations: + Only include domains that should legitimately be covered by the + same certificate + - - Renewal Planning: Monitor - certificate renewal processes when using external certificate - authorities - - + + Renewal Planning: Monitor + certificate renewal processes when using external certificate + authorities + + + - - - Complete Configuration Example + + Complete Configuration Example - Below is a comprehensive example showing how to configure - multiple issuers with alternative domain support: + Below is a comprehensive example showing how to configure + multiple issuers with alternative domain support: - - Complete Certificate Configuration + + Complete Certificate Configuration - + certificates: enabled: true issuers: @@ -1853,142 +2007,207 @@ certificates: name: code-signing-issuer kind: Issuer domain: signing.internal - - - - This configuration enables: - - - - Local inter-component communication with internal - certificates - - - - Public-facing services with Let's Encrypt - certificates - - - - Remote client certificates valid across multiple - domains - - - - Code signing capabilities for ECL applications - - - - - - Troubleshooting Certificates - - - Certificate Generation Issues + + - If certificates are not generating correctly with alternative - domains: + This configuration enables: - + - Verify that cert-manager is properly installed and - running + Local inter-component communication with internal + certificates - Check that all domains resolve properly via DNS + Public-facing services with Let's Encrypt + certificates - Examine Certificate and CertificateRequest resources for - error messages + Remote client certificates valid across multiple + domains - Validate issuer configuration and permissions + Code signing capabilities for ECL applications - + - - Common Error Messages + + Troubleshooting Certificates - Watch for these common issues when using alternative - domains: + + Certificate Generation Issues - - - DNS validation failures + If certificates are not generating correctly with alternative + domains: + - Ensure all alternative domains have proper DNS records - and are accessible from the certificate authority + Verify that cert-manager is properly installed and + running - - - - Rate limit exceeded - Some certificate authorities impose limits on the number - of domains per certificate or certificates per - timeframe + Check that all domains resolve properly via DNS - - - Invalid domain format + + Examine Certificate and CertificateRequest resources for + error messages + - Verify that all domains in alternativeDomains follow - proper DNS naming conventions + Validate issuer configuration and permissions - - + + + + + Common Error Messages + + Watch for these common issues when using alternative + domains: + + + + DNS validation failures + + + Ensure all alternative domains have proper DNS records + and are accessible from the certificate authority + + + + + Rate limit exceeded + + + Some certificate authorities impose limits on the number + of domains per certificate or certificates per + timeframe + + + + + Invalid domain format + + + Verify that all domains in alternativeDomains follow + proper DNS naming conventions + + + + - + + Migration from Legacy Configuration - - Migration from Legacy Configuration + If you are currently using individual domain configurations or + multiple certificate resources, consider migrating to the + alternativeDomains approach for simplified + management: - If you are currently using individual domain configurations or - multiple certificate resources, consider migrating to the - alternativeDomains approach for simplified - management: + + + Identify all domains currently covered by separate + certificates + - - - Identify all domains currently covered by separate - certificates - + + Update the remote issuer configuration to include all + domains in the alternativeDomains array + - - Update the remote issuer configuration to include all - domains in the alternativeDomains array - + + Deploy the updated configuration + - - Deploy the updated configuration - + + Verify that the new certificate covers all required + domains + - - Verify that the new certificate covers all required - domains - + + Remove old certificate configurations once the new setup is + validated + + - - Remove old certificate configurations once the new setup is - validated - - + + The alternativeDomains configuration is now + the preferred method for multi-domain certificate management in HPCC + Systems containerized deployments. This approach simplifies + certificate management and reduces the overhead of maintaining + multiple certificate resources. + + + - - The alternativeDomains configuration is now - the preferred method for multi-domain certificate management in HPCC - Systems containerized deployments. This approach simplifies - certificate management and reduces the overhead of maintaining - multiple certificate resources. - + + Secrets + + The Secrets section makes sensitive information published as Kubernetes secrets + available to the platform in a structured way without exposing it in + code or configuration files. At the top level is a set of categories, + where the category determines what the secret is and which components can + use it. Under each category is a list of secrets of the format: + platform-name: k8s-published-name where + platform-name is the name of the secret as it is referred + to in the platform, and k8s-published-name is the name of + the secret as it is published to the kubernetes cluster. + + + For example, the category named "eclUser" + is for placing secrets that you want to be readable directly from ECL + code. Other secret categories, including the "ecl" category, are read + internally by system components and not exposed directly to ECL + code. + + + + Vaults + + The Vaults section is an alternative to the Secrets section. It serves + the same purpose and shares the same top-level categories, but uses a Hashicorp + Vault server to store the secrets instead of Kubernetes. + + + + + Cross Origin Resource Handling + + Cross-origin resource sharing (CORS) is a mechanism for + integrating applications in different domains. CORS defines how client + web applications in one domain can interact with resources in another + domain. You can configure CORS support settings in the ESP section of + the values.yaml file as illustrated below: + + esp: +- name: eclwatch + application: eclwatch + auth: ldap + replicas: 1 + # The following 'corsAllowed' section is used to configure CORS support + # origin - the origin to support CORS requests from + # headers - the headers to allow for the given origin via CORS + # methods - the HTTP methods to allow for the given origin via CORS + # + corsAllowed: + # origin starting with https will only allow https CORS + - origin: https://*.example2.com + headers: + - "X-Custom-Header" + methods: + - "GET" + # origin starting with http will allow http or https CORS + - origin: http://www.example.com + headers: + - "*" + methods: + - "GET" + - "POST"