chore: Update to operator-rs 0.77.1 and use new S3 structs

Cargo.toml

@@ -24,11 +24,12 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.74.0" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.76.0" }
 product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.7.0" }
 strum = { version = "0.26", features = ["derive"] }
-tokio = { version = "1.39", features = ["full"] }
+tokio = { version = "1.40", features = ["full"] }
 tracing = "0.1"
 
-# [patch."https://github.com/stackabletech/operator-rs.git"]
-# stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "main" }
+[patch."https://github.com/stackabletech/operator-rs.git"]
+stackable-operator = { git = "https://github.com/stackabletech//operator-rs.git", branch = "fix/clashing-volumes-and-mounts-2" }
+# stackable-operator = { path = "../operator-rs/crates/stackable-operator" }

rust/crd/src/catalog/delta_lake.rs

@@ -1,7 +1,7 @@
 use super::commons::{HdfsConnection, MetastoreConnection};
 use serde::{Deserialize, Serialize};
 use stackable_operator::{
-    commons::s3::S3ConnectionDef,
+    commons::s3::S3ConnectionInlineOrReference,
     schemars::{self, JsonSchema},
 };
 
@@ -12,10 +12,12 @@ use stackable_operator::{
 pub struct DeltaLakeConnector {
     /// Mandatory connection to a Hive Metastore, which will be used as a storage for metadata.
     pub metastore: MetastoreConnection,
+
     /// Connection to an S3 store.
     /// Please make sure that the underlying Hive metastore also has access to the S3 store.
     /// Learn more about S3 configuration in the [S3 concept docs](DOCS_BASE_URL_PLACEHOLDER/concepts/s3).
-    pub s3: Option<S3ConnectionDef>,
+    pub s3: Option<S3ConnectionInlineOrReference>,
+
     /// Connection to an HDFS cluster.
     /// Please make sure that the underlying Hive metastore also has access to the HDFS.
     pub hdfs: Option<HdfsConnection>,

rust/crd/src/catalog/generic.rs

@@ -84,7 +84,7 @@ mod tests {
                         Property::ValueFromSecret {
                             secret_key_selector: SecretKeySelector {
                                 key: "user".to_string(),
-                                name: Some("my-postgresql-credentials-secret".to_string()),
+                                name: "my-postgresql-credentials-secret".to_string(),
                                 optional: None,
                             }
                         }
@@ -94,7 +94,7 @@ mod tests {
                         Property::ValueFromSecret {
                             secret_key_selector: SecretKeySelector {
                                 key: "password".to_string(),
-                                name: Some("my-postgresql-credentials-secret".to_string()),
+                                name: "my-postgresql-credentials-secret".to_string(),
                                 optional: None,
                             }
                         }

rust/crd/src/catalog/hive.rs

@@ -1,7 +1,7 @@
 use super::commons::{HdfsConnection, MetastoreConnection};
 use serde::{Deserialize, Serialize};
 use stackable_operator::{
-    commons::s3::S3ConnectionDef,
+    commons::s3::S3ConnectionInlineOrReference,
     schemars::{self, JsonSchema},
 };
 
@@ -10,10 +10,12 @@ use stackable_operator::{
 pub struct HiveConnector {
     /// Mandatory connection to a Hive Metastore, which will be used as a storage for metadata.
     pub metastore: MetastoreConnection,
+
     /// Connection to an S3 store.
     /// Please make sure that the underlying Hive metastore also has access to the S3 store.
     /// Learn more about S3 configuration in the [S3 concept docs](DOCS_BASE_URL_PLACEHOLDER/concepts/s3).
-    pub s3: Option<S3ConnectionDef>,
+    pub s3: Option<S3ConnectionInlineOrReference>,
+
     /// Connection to an HDFS cluster.
     /// Please make sure that the underlying Hive metastore also has access to the HDFS.
     pub hdfs: Option<HdfsConnection>,

rust/crd/src/catalog/iceberg.rs

@@ -1,7 +1,7 @@
 use super::commons::{HdfsConnection, MetastoreConnection};
 use serde::{Deserialize, Serialize};
 use stackable_operator::{
-    commons::s3::S3ConnectionDef,
+    commons::s3::S3ConnectionInlineOrReference,
     schemars::{self, JsonSchema},
 };
 
@@ -12,10 +12,12 @@ use stackable_operator::{
 pub struct IcebergConnector {
     /// Mandatory connection to a Hive Metastore, which will be used as a storage for metadata.
     pub metastore: MetastoreConnection,
+
     /// Connection to an S3 store.
     /// Please make sure that the underlying Hive metastore also has access to the S3 store.
     /// Learn more about S3 configuration in the [S3 concept docs](DOCS_BASE_URL_PLACEHOLDER/concepts/s3).
-    pub s3: Option<S3ConnectionDef>,
+    pub s3: Option<S3ConnectionInlineOrReference>,
+
     /// Connection to an HDFS cluster.
     /// Please make sure that the underlying Hive metastore also has access to the HDFS.
     pub hdfs: Option<HdfsConnection>,

rust/crd/src/lib.rs

@@ -92,12 +92,10 @@ pub const METRICS_PORT_PROPERTY: &str = "metricsPort";
 pub const CONFIG_DIR_NAME: &str = "/stackable/config";
 pub const RW_CONFIG_DIR_NAME: &str = "/stackable/rwconfig";
 pub const DATA_DIR_NAME: &str = "/stackable/data";
-pub const S3_SECRET_DIR_NAME: &str = "/stackable/secrets";
 pub const STACKABLE_SERVER_TLS_DIR: &str = "/stackable/server_tls";
 pub const STACKABLE_CLIENT_TLS_DIR: &str = "/stackable/client_tls";
 pub const STACKABLE_INTERNAL_TLS_DIR: &str = "/stackable/internal_tls";
 pub const STACKABLE_MOUNT_SERVER_TLS_DIR: &str = "/stackable/mount_server_tls";
-pub const STACKABLE_MOUNT_CLIENT_TLS_DIR: &str = "/stackable/mount_client_tls";
 pub const STACKABLE_MOUNT_INTERNAL_TLS_DIR: &str = "/stackable/mount_internal_tls";
 pub const SYSTEM_TRUST_STORE: &str = "/etc/pki/java/cacerts";
 // store pws

rust/operator-binary/src/authentication/mod.rs

@@ -11,7 +11,10 @@ use std::collections::{BTreeMap, HashMap};
 
 use snafu::{OptionExt, ResultExt, Snafu};
 use stackable_operator::{
-    builder::pod::{container::ContainerBuilder, PodBuilder},
+    builder::{
+        self,
+        pod::{container::ContainerBuilder, PodBuilder},
+    },
     commons::{
         authentication::{AuthenticationClass, AuthenticationClassProvider},
         product_image_selection::ResolvedProductImage,
@@ -58,6 +61,14 @@ pub enum Error {
 
     #[snafu(display("OIDC authentication details not specified. The AuthenticationClass {auth_class_name:?} uses an OIDC provider, you need to specify OIDC authentication details (such as client credentials) as well"))]
     OidcAuthenticationDetailsNotSpecified { auth_class_name: String },
+
+    #[snafu(display("failed to add needed volume"))]
+    AddVolume { source: builder::pod::Error },
+
+    #[snafu(display("failed to add needed volumeMount"))]
+    AddVolumeMount {
+        source: builder::pod::container::Error,
+    },
 }
 
 type Result<T, E = Error> = std::result::Result<T, E>;
@@ -140,9 +151,11 @@ impl TrinoAuthenticationConfig {
         pod_builder: &mut PodBuilder,
         prepare_builder: &mut ContainerBuilder,
         trino_builder: &mut ContainerBuilder,
-    ) {
+    ) -> Result<()> {
         // volumes
-        pod_builder.add_volumes(self.volumes());
+        pod_builder
+            .add_volumes(self.volumes())
+            .context(AddVolumeSnafu)?;
 
         let affected_containers = vec![
             stackable_trino_crd::Container::Prepare,
@@ -154,17 +167,23 @@ impl TrinoAuthenticationConfig {
 
             match container {
                 stackable_trino_crd::Container::Prepare => {
-                    prepare_builder.add_volume_mounts(volume_mounts);
+                    prepare_builder
+                        .add_volume_mounts(volume_mounts)
+                        .context(AddVolumeMountSnafu)?;
                 }
                 stackable_trino_crd::Container::Trino => {
-                    trino_builder.add_volume_mounts(volume_mounts);
+                    trino_builder
+                        .add_volume_mounts(volume_mounts)
+                        .context(AddVolumeMountSnafu)?;
                 }
                 // handled internally
                 stackable_trino_crd::Container::PasswordFileUpdater => {}
                 // nothing to do here
                 stackable_trino_crd::Container::Vector => {}
             }
         }
+
+        Ok(())
     }
 
     /// Add required init / side car containers

rust/operator-binary/src/authentication/oidc/mod.rs

@@ -4,7 +4,7 @@
 use crate::authentication::TrinoAuthenticationConfig;
 use crate::command;
 use snafu::{ResultExt, Snafu};
-use stackable_operator::commons::authentication::oidc;
+use stackable_operator::commons::{authentication::oidc, tls_verification::TlsClientDetailsError};
 use stackable_trino_crd::{TrinoRole, STACKABLE_CLIENT_TLS_DIR};
 
 // Trino properties
@@ -45,9 +45,7 @@ pub enum Error {
     UnverifiedOidcTlsConnectionNotSupported,
 
     #[snafu(display("Failed to create OIDC Volumes and VolumeMounts"))]
-    FailedToCreateOidcVolumeAndVolumeMounts {
-        source: stackable_operator::commons::authentication::tls::TlsClientDetailsError,
-    },
+    FailedToCreateOidcVolumeAndVolumeMounts { source: TlsClientDetailsError },
 }
 
 #[derive(Clone, Debug, Default)]

rust/operator-binary/src/authentication/password/file.rs

@@ -1,11 +1,15 @@
 use crate::authentication::password::PASSWORD_AUTHENTICATOR_NAME;
 use crate::controller::STACKABLE_LOG_DIR;
 
+use snafu::{ResultExt, Snafu};
 use stackable_operator::{
-    builder::pod::{
-        container::ContainerBuilder,
-        resources::ResourceRequirementsBuilder,
-        volume::{VolumeBuilder, VolumeMountBuilder},
+    builder::{
+        self,
+        pod::{
+            container::ContainerBuilder,
+            resources::ResourceRequirementsBuilder,
+            volume::{VolumeBuilder, VolumeMountBuilder},
+        },
     },
     commons::{authentication::static_, product_image_selection::ResolvedProductImage},
     k8s_openapi::api::core::v1::{Container, Volume, VolumeMount},
@@ -22,6 +26,14 @@ pub const PASSWORD_AUTHENTICATOR_SECRET_MOUNT_PATH: &str = "/stackable/auth-secr
 const PASSWORD_AUTHENTICATOR_NAME_FILE: &str = "file";
 const FILE_PASSWORD_FILE: &str = "file.password-file";
 
+#[derive(Snafu, Debug)]
+pub enum Error {
+    #[snafu(display("failed to add needed volumeMounts"))]
+    AddVolumeMounts {
+        source: builder::pod::container::Error,
+    },
+}
+
 #[derive(Clone, Debug)]
 pub struct FileAuthenticator {
     name: String,
@@ -103,7 +115,7 @@ impl FileAuthenticator {
 pub fn build_password_file_update_container(
     resolved_product_image: &ResolvedProductImage,
     volume_mounts: Vec<VolumeMount>,
-) -> Container {
+) -> Result<Container, Error> {
     let mut cb_pw_file_updater =
         ContainerBuilder::new(&stackable_trino_crd::Container::PasswordFileUpdater.to_string())
             .expect(
@@ -164,12 +176,14 @@ wait_for_termination $!
         stackable_auth_secret_dir = PASSWORD_AUTHENTICATOR_SECRET_MOUNT_PATH,
     ));
 
-    cb_pw_file_updater
+    Ok(cb_pw_file_updater
         .image_from_product_image(resolved_product_image)
         // calculated mounts
         .add_volume_mounts(volume_mounts)
+        .context(AddVolumeMountsSnafu)?
         // fixed
         .add_volume_mount("log", STACKABLE_LOG_DIR)
+        .context(AddVolumeMountsSnafu)?
         .resources(
             ResourceRequirementsBuilder::new()
                 .with_cpu_request("100m")
@@ -186,7 +200,7 @@ wait_for_termination $!
             "-c".to_string(),
         ])
         .args(vec![commands.join("\n")])
-        .build()
+        .build())
 }
 
 #[cfg(test)]

rust/operator-binary/src/authentication/password/mod.rs

@@ -29,16 +29,19 @@ const PASSWORD_AUTHENTICATOR_NAME: &str = "password-authenticator.name";
 
 #[derive(Snafu, Debug)]
 pub enum Error {
-    #[snafu(display("Failed to configure LDAP password authentication"))]
+    #[snafu(display("failed to configure LDAP password authentication"))]
     InvalidLdapAuthenticationConfiguration { source: ldap::Error },
 
-    #[snafu(display("Failed to write password authentication config file"))]
-    FailedToWritePasswordAuthenticationFile {
+    #[snafu(display("failed to write password authentication config file"))]
+    WritePasswordAuthenticationFile {
         source: product_config::writer::PropertiesWriterError,
     },
 
-    #[snafu(display("Failed to create LDAP Volumes and VolumeMounts"))]
+    #[snafu(display("failed to create LDAP Volumes and VolumeMounts"))]
     LdapVolumeAndVolumeMounts { source: ldap::Error },
+
+    #[snafu(display("failed to create LDAP Volumes and VolumeMounts"))]
+    BuildPasswordFileUpdateContainer { source: file::Error },
 }
 
 #[derive(Clone, Debug, Default)]
@@ -96,7 +99,7 @@ impl TrinoPasswordAuthentication {
                                 .collect::<BTreeMap<String, Option<String>>>()
                                 .iter(),
                         )
-                        .context(FailedToWritePasswordAuthenticationFileSnafu)?,
+                        .context(WritePasswordAuthenticationFileSnafu)?,
                     );
                     // required volumes
                     password_authentication_config.add_volume(file_authenticator.secret_volume());
@@ -134,7 +137,7 @@ impl TrinoPasswordAuthentication {
                                 .collect::<BTreeMap<String, Option<String>>>()
                                 .iter(),
                         )
-                        .context(FailedToWritePasswordAuthenticationFileSnafu)?,
+                        .context(WritePasswordAuthenticationFileSnafu)?,
                     );
 
                     // extra commands
@@ -176,7 +179,8 @@ impl TrinoPasswordAuthentication {
                 file::build_password_file_update_container(
                     resolved_product_image,
                     pw_file_update_container_volume_mounts,
-                ),
+                )
+                .context(BuildPasswordFileUpdateContainerSnafu)?,
             );
         }