feat(stackable-operator)!: Reduce CRD size by omitting podOverride and affinities schemas (#821)

* feat!: Reduce CRD size by omitting podOverride and affinities schemas

* changelog

* changelog

* Add raw_object_list_schema

* Update crates/stackable-operator/CHANGELOG.md

Co-authored-by: Techassi <git@techassi.dev>

* fix: Add missing full stops in changelogs

* changelog

* newlines

---------

Co-authored-by: Techassi <git@techassi.dev>
Co-authored-by: Techassi <sascha.lautenschlaeger@stackable.tech>

Author: sbernauer

crates/k8s-version/CHANGELOG.md

@@ -14,7 +14,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
-- Bump rust-toolchain to 1.79.0 ([#822])
+- Bump rust-toolchain to 1.79.0 ([#822]).
 
 [#822]: https://github.com/stackabletech/operator-rs/pull/822
 

crates/stackable-certs/CHANGELOG.md

@@ -8,7 +8,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
-- Bump rust-toolchain to 1.79.0 ([#822])
+- Bump rust-toolchain to 1.79.0 ([#822]).
 
 [#822]: https://github.com/stackabletech/operator-rs/pull/822
 
@@ -34,7 +34,7 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - Implement `PartialEq` for most _Snafu_ Error enums ([#757]).
-- Update Rust to 1.77 ([#759])
+- Update Rust to 1.77 ([#759]).
 
 [#757]: https://github.com/stackabletech/operator-rs/pull/757
 [#759]: https://github.com/stackabletech/operator-rs/pull/759

crates/stackable-operator-derive/CHANGELOG.md

@@ -8,7 +8,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
-- Bump rust-toolchain to 1.79.0 ([#822])
+- Bump rust-toolchain to 1.79.0 ([#822]).
 
 [#822]: https://github.com/stackabletech/operator-rs/pull/822
 
@@ -24,7 +24,7 @@ All notable changes to this project will be documented in this file.
 
 ### Changed
 
-- Update Rust to 1.77 ([#759])
+- Update Rust to 1.77 ([#759]).
 - Bump GitHub workflow actions ([#772]).
 - Revert `zeroize` version bump ([#772]).
 

crates/stackable-operator/CHANGELOG.md

@@ -7,7 +7,16 @@ All notable changes to this project will be documented in this file.
 ### Changed
 
 - BREAKING: Replace `lazy_static` with `std::cell::LazyCell` ([#827]).
-
+- BREAKING: Convert `podOverrides` and `affinity` fields to take any arbitrary
+  YAML input, rather than using the underlying schema. With this change, one of
+  the larger CRDs, like the Druid CRD went down in size from `2.4MB` to `288K`
+  (a 88% reduction). One downside is that user input is not checked to be a
+  valid `PodTemplateSpec`, `PodAffinity`, `PodAntiAffinity` and `NodeAffinity`
+  any more. However, checks can be re-added by using validation webhooks if
+  needed. This change should not be breaking for the user and is a preparation
+  for CRD versioning. ([#821]).
+
+[#821]: https://github.com/stackabletech/operator-rs/pull/821
 [#827]: https://github.com/stackabletech/operator-rs/pull/827
 
 ## [0.71.0] - 2024-07-29
@@ -37,7 +46,7 @@ All notable changes to this project will be documented in this file.
   - [kube#1494](https://github.com/kube-rs/kube/pull/1494)
   - [kube#1504](https://github.com/kube-rs/kube/pull/1504)
 - Upgrade opentelemetry crates ([#811]).
-- Bump rust-toolchain to 1.79.0 ([#822])
+- Bump rust-toolchain to 1.79.0 ([#822]).
 
 ### Fixed
 

crates/stackable-operator/src/commons/affinity.rs

@@ -13,6 +13,7 @@ use stackable_operator_derive::Fragment;
 use crate::{
     config::merge::{Atomic, Merge},
     kvp::consts::{K8S_APP_COMPONENT_KEY, K8S_APP_INSTANCE_KEY, K8S_APP_NAME_KEY},
+    utils::crds::raw_object_schema,
 };
 
 pub const TOPOLOGY_KEY_HOSTNAME: &str = "kubernetes.io/hostname";
@@ -36,9 +37,19 @@ pub const TOPOLOGY_KEY_HOSTNAME: &str = "kubernetes.io/hostname";
     serde(rename_all = "camelCase")
 )]
 pub struct StackableAffinity {
+    /// Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node)
+    #[fragment_attrs(schemars(schema_with = "raw_object_schema"))]
     pub pod_affinity: Option<PodAffinity>,
+
+    /// Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node)
+    #[fragment_attrs(schemars(schema_with = "raw_object_schema"))]
     pub pod_anti_affinity: Option<PodAntiAffinity>,
+
+    /// Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node)
+    #[fragment_attrs(schemars(schema_with = "raw_object_schema"))]
     pub node_affinity: Option<NodeAffinity>,
+
+    // This schema isn't big, so it can stay
     pub node_selector: Option<StackableNodeSelector>,
 }
 
@@ -51,6 +62,7 @@ pub struct StackableAffinity {
 // FIXME: The generated JsonSchema will be wrong, so until https://github.com/GREsau/schemars/issues/259 is fixed, we
 // need to use `#[schemars(deny_unknown_fields)]`.
 // See https://github.com/stackabletech/operator-rs/pull/752#issuecomment-2017630433 for details.
+/// Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node)
 #[derive(Clone, Debug, Eq, Deserialize, JsonSchema, PartialEq, Serialize)]
 #[serde(rename_all = "camelCase")]
 #[schemars(deny_unknown_fields)]

crates/stackable-operator/src/commons/mod.rs

@@ -6,7 +6,6 @@ pub mod cluster_operation;
 pub mod listener;
 pub mod opa;
 pub mod pdb;
-pub mod pod_overrides;
 pub mod product_image_selection;
 pub mod rbac;
 pub mod resources;

crates/stackable-operator/src/role_utils.rs

@@ -86,12 +86,13 @@ use std::{
 };
 
 use crate::{
-    commons::{pdb::PdbConfig, pod_overrides::pod_overrides_schema},
+    commons::pdb::PdbConfig,
     config::{
         fragment::{self, FromFragment},
         merge::Merge,
     },
     product_config_utils::Configuration,
+    utils::crds::raw_object_schema,
 };
 use derivative::Derivative;
 use k8s_openapi::api::core::v1::PodTemplateSpec;
@@ -111,13 +112,15 @@ pub struct CommonConfiguration<T> {
     // does not support specifying custom bounds.
     #[schemars(default = "config_schema_default")]
     pub config: T,
+
     /// The `configOverrides` can be used to configure properties in product config files
     /// that are not exposed in the CRD. Read the
     /// [config overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#config-overrides)
     /// and consult the operator specific usage guide documentation for details on the
     /// available config files and settings for the specific product.
     #[serde(default)]
     pub config_overrides: HashMap<String, HashMap<String, String>>,
+
     /// `envOverrides` configure environment variables to be set in the Pods.
     /// It is a map from strings to strings - environment variables and the value to set.
     /// Read the
@@ -126,18 +129,20 @@ pub struct CommonConfiguration<T> {
     /// the product specific environment variables that are available.
     #[serde(default)]
     pub env_overrides: HashMap<String, String>,
+
     // BTreeMap to keep some order with the cli arguments.
     // TODO add documentation.
     #[serde(default)]
     pub cli_overrides: BTreeMap<String, String>,
+
     /// In the `podOverrides` property you can define a
     /// [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core)
     /// to override any property that can be set on a Kubernetes Pod.
     /// Read the
     /// [Pod overrides documentation](DOCS_BASE_URL_PLACEHOLDER/concepts/overrides#pod-overrides)
     /// for more information.
     #[serde(default)]
-    #[schemars(schema_with = "pod_overrides_schema")]
+    #[schemars(schema_with = "raw_object_schema")]
     pub pod_overrides: PodTemplateSpec,
 }
 

crates/stackable-operator/src/commons/pod_overrides.rs renamed to crates/stackable-operator/src/utils/crds.rs

@@ -1,56 +1,36 @@
-use k8s_openapi::api::core::v1::PodTemplateSpec;
-use schemars::{schema::Schema, visit::Visitor, JsonSchema};
-
-/// Simplified schema for PodTemplateSpec without mandatory fields (e.g. `containers`) or documentation.
-///
-/// The normal PodTemplateSpec requires you to specify `containers` as an `Vec<Container>`.
-/// Often times the user want's to overwrite/add stuff not related to a container
-/// (e.g. tolerations or a ServiceAccount), so it's annoying that he always needs to
-/// specify an empty array for `containers`.
-///
-/// Additionally all docs are removed, as the resulting Stackable CRD objects where to big for Kubernetes.
-/// E.g. the HdfsCluster CRD increased to ~3.2 MB (which is over the limit of 3MB), after stripping
-/// the docs it went down to ~1.3 MiB.
-pub fn pod_overrides_schema(gen: &mut schemars::gen::SchemaGenerator) -> Schema {
-    let mut schema = PodTemplateSpec::json_schema(gen);
-    SimplifyOverrideSchema.visit_schema(&mut schema);
-
-    if let Schema::Object(schema) = &mut schema {
-        let meta = schema.metadata.get_or_insert_with(Default::default);
-        meta.description = Some("See PodTemplateSpec (https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) for more details".to_string());
-    }
-
-    schema
+use schemars::schema::Schema;
+
+pub fn raw_object_schema(_: &mut schemars::gen::SchemaGenerator) -> Schema {
+    serde_json::from_value(serde_json::json!({
+        "type": "object",
+        "x-kubernetes-preserve-unknown-fields": true,
+    }))
+    .expect("Failed to parse JSON of custom raw object schema")
 }
 
-struct SimplifyOverrideSchema;
-impl schemars::visit::Visitor for SimplifyOverrideSchema {
-    fn visit_schema_object(&mut self, schema: &mut schemars::schema::SchemaObject) {
-        // Strip docs to make the schema more compact
-        if let Some(meta) = &mut schema.metadata {
-            meta.description = None;
-            meta.examples.clear();
+pub fn raw_object_list_schema(_: &mut schemars::gen::SchemaGenerator) -> Schema {
+    serde_json::from_value(serde_json::json!({
+        "type": "array",
+        "items": {
+            "type": "object",
+            "x-kubernetes-preserve-unknown-fields": true,
         }
-
-        // Make all options optional
-        if let Some(object) = &mut schema.object {
-            object.required.clear();
-        }
-
-        schemars::visit::visit_schema_object(self, schema);
-    }
+    }))
+    .expect("Failed to parse JSON of custom raw object list schema")
 }
 
 #[cfg(test)]
 mod tests {
+    use k8s_openapi::api::core::v1::PodTemplateSpec;
+    use schemars::JsonSchema;
     use serde::{Deserialize, Serialize};
 
     use super::*;
 
     #[derive(Clone, Debug, Default, Deserialize, JsonSchema, PartialEq, Serialize)]
     #[serde(rename_all = "camelCase")]
     struct Test {
-        #[schemars(schema_with = "pod_overrides_schema")]
+        #[schemars(schema_with = "raw_object_schema")]
         pub pod_overrides: PodTemplateSpec,
     }
 
@@ -130,7 +110,7 @@ mod tests {
                 - image: docker.stackable.tech/stackable/nifi:1.23.2-stackable23.11.0
         "#;
 
-        // FIXME: Ideally we would require the names of the containers to be set.  We had users using podOverrides
+        // FIXME: Ideally we would require the names of the containers to be set. We had users using podOverrides
         // without setting the name of the container and wondering why it didn't work.
         serde_yaml::from_str::<Test>(input).expect("Failed to parse valid podOverride");
     }

crates/stackable-operator/src/utils/mod.rs

@@ -1,4 +1,5 @@
 pub mod bash;
+pub mod crds;
 pub mod logging;
 mod option;
 mod url;

crates/stackable-telemetry/CHANGELOG.md

@@ -12,7 +12,7 @@ All notable changes to this project will be documented in this file.
 - Use OpenTelemetry Context in Axum instrumentation layer, adjust log and span level, simplify trace config ([#811]).
   - tracing: Upgrade opentelemetry crates, simplify trace config, fix shutdown conditions, use new way to shutdown LoggerProvider.
   - instrumentation/axum: demote event severity for errors easily caused by clients, replace parent span context if given in http header and link to previous trace contexts.
-- Bump rust-toolchain to 1.79.0 ([#822])
+- Bump rust-toolchain to 1.79.0 ([#822]).
 
 [#801]: https://github.com/stackabletech/operator-rs/pull/801
 [#811]: https://github.com/stackabletech/operator-rs/pull/811