From 5033d797ba5ae7c216a4e1f2f187a90b9487cfdd Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 3 Apr 2025 18:26:51 +0200 Subject: [PATCH 01/39] mount listener volumes for relevant roles, type depending on listener class --- deploy/helm/airflow-operator/crds/crds.yaml | 11 +- .../airflow-operator/templates/roles.yaml | 6 + .../operator-binary/src/airflow_controller.rs | 156 +++++++++--------- rust/operator-binary/src/crd/mod.rs | 37 ++--- tests/templates/kuttl/oidc/login.py | 35 ++-- .../kuttl/opa/41_check-authorization.py | 16 +- 6 files changed, 125 insertions(+), 136 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 781fd8af..9c813849 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -586,16 +586,7 @@ spec: type: boolean listenerClass: default: cluster-internal - description: |- - This field controls which type of Service the Operator creates for this AirflowCluster: - - * cluster-internal: Use a ClusterIP service - - * external-unstable: Use a NodePort service - - * external-stable: Use a LoadBalancer service - - This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. enum: - cluster-internal - external-unstable diff --git a/deploy/helm/airflow-operator/templates/roles.yaml b/deploy/helm/airflow-operator/templates/roles.yaml index ee6e9a46..119590e3 100644 --- a/deploy/helm/airflow-operator/templates/roles.yaml +++ b/deploy/helm/airflow-operator/templates/roles.yaml @@ -84,6 +84,12 @@ rules: - customresourcedefinitions verbs: - get + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index f85da00d..1a32c6c1 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -19,8 +19,14 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::VolumeBuilder, PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, VolumeBuilder, + }, + PodBuilder, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -35,8 +41,9 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, EmptyDirVolumeSource, EnvVar, PodTemplateSpec, Probe, Service, - ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, VolumeMount, + ConfigMap, EmptyDirVolumeSource, EnvVar, PersistentVolumeClaim, PodTemplateSpec, + Probe, Service, ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, + VolumeMount, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -84,8 +91,9 @@ use crate::{ git_sync::{GitSync, GIT_SYNC_CONTENT, GIT_SYNC_NAME, GIT_SYNC_ROOT}, v1alpha1, AirflowClusterStatus, AirflowConfig, AirflowConfigOptions, AirflowExecutor, AirflowRole, Container, ExecutorConfig, ExecutorConfigFragment, AIRFLOW_CONFIG_FILENAME, - AIRFLOW_UID, APP_NAME, CONFIG_PATH, LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, - TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + AIRFLOW_UID, APP_NAME, CONFIG_PATH, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, + LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, + TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, }, env_vars::{ self, build_airflow_template_envs, build_gitsync_statefulset_envs, build_gitsync_template, @@ -324,6 +332,16 @@ pub enum Error { #[snafu(display("failed to build Statefulset environmental variables"))] BuildStatefulsetEnvVars { source: env_vars::Error }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, } type Result = std::result::Result; @@ -466,16 +484,6 @@ pub async fn reconcile_airflow( role: role_name.to_string(), })?; - // some roles will only run "internally" and do not need to be created as services - if let Some(resolved_port) = role_port(role_name) { - let role_service = - build_role_service(airflow, &resolved_product_image, role_name, resolved_port)?; - cluster_resources - .add(client, role_service) - .await - .context(ApplyRoleServiceSnafu)?; - } - for (rolegroup_name, rolegroup_config) in role_config.iter() { let rolegroup = RoleGroupRef { cluster: ObjectRef::from_obj(airflow), @@ -487,7 +495,12 @@ pub async fn reconcile_airflow( .merged_config(&airflow_role, &rolegroup) .context(FailedToResolveConfigSnafu)?; - let rg_service = build_rolegroup_service(airflow, &resolved_product_image, &rolegroup)?; + let rg_service = build_rolegroup_service( + airflow, + &airflow_role, + &resolved_product_image, + &rolegroup, + )?; cluster_resources.add(client, rg_service).await.context( ApplyRoleGroupServiceSnafu { rolegroup: rolegroup.clone(), @@ -621,55 +634,6 @@ async fn build_executor_template( Ok(()) } -/// The server-role service is the primary endpoint that should be used by clients that do not perform internal load balancing, -/// including targets outside the cluster. -fn build_role_service( - airflow: &v1alpha1::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - role_name: &str, - port: u16, -) -> Result { - let role_svc_name = format!("{}-{}", airflow.name_any(), role_name); - let ports = role_ports(port); - - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .name(&role_svc_name) - .ownerreference_from_resource(airflow, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - role_name, - "global", - )) - .context(ObjectMetaSnafu)? - .build(); - - let service_selector_labels = - Labels::role_selector(airflow, APP_NAME, role_name).context(BuildLabelSnafu)?; - - let service_spec = ServiceSpec { - type_: Some( - airflow - .spec - .cluster_config - .listener_class - .k8s_service_type(), - ), - ports: Some(ports), - selector: Some(service_selector_labels.into()), - ..ServiceSpec::default() - }; - - Ok(Service { - metadata, - spec: Some(service_spec), - status: None, - }) -} - fn role_ports(port: u16) -> Vec { vec![ServicePort { name: Some("http".to_string()), @@ -679,10 +643,6 @@ fn role_ports(port: u16) -> Vec { }] } -fn role_port(role_name: &str) -> Option { - AirflowRole::from_str(role_name).unwrap().get_http_port() -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( @@ -799,6 +759,7 @@ fn build_rolegroup_config_map( /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. fn build_rolegroup_service( airflow: &v1alpha1::AirflowCluster, + airflow_role: &AirflowRole, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, ) -> Result { @@ -809,7 +770,7 @@ fn build_rolegroup_service( ..Default::default() }]; - if let Some(http_port) = role_port(&rolegroup.role) { + if let Some(http_port) = airflow_role.get_http_port() { ports.append(&mut role_ports(http_port)); } @@ -890,15 +851,18 @@ fn build_server_rolegroup_statefulset( let rolegroup = role.role_groups.get(&rolegroup_ref.role_group); let mut pb = PodBuilder::new(); + let recommended_object_labels = build_recommended_labels( + airflow, + AIRFLOW_CONTROLLER_NAME, + &resolved_product_image.app_version_label, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + let recommended_labels = + Labels::recommended(recommended_object_labels.clone()).context(LabelBuildSnafu)?; let pb_metadata = ObjectMetaBuilder::new() - .with_recommended_labels(build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels) .context(ObjectMetaSnafu)? .build(); @@ -973,10 +937,17 @@ fn build_server_rolegroup_statefulset( .context(AddVolumeMountSnafu)?; } - if let Some(resolved_port) = airflow_role.get_http_port() { + let mut pvcs: Option> = None; + + // for roles with an http endpoint + if let Some(http_port) = airflow_role.get_http_port() { + airflow_container + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + let probe = Probe { tcp_socket: Some(TCPSocketAction { - port: IntOrString::Int(resolved_port.into()), + port: IntOrString::Int(http_port.into()), ..TCPSocketAction::default() }), initial_delay_seconds: Some(60), @@ -986,7 +957,29 @@ fn build_server_rolegroup_statefulset( }; airflow_container.readiness_probe(probe.clone()); airflow_container.liveness_probe(probe); - airflow_container.add_container_port("http", resolved_port.into()); + airflow_container.add_container_port("http", http_port.into()); + + let listener_class = &airflow.spec.cluster_config.listener_class; + pvcs = if listener_class.discoverable() { + // externally-reachable listener endpoints should use a pvc volume... + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass(listener_class.to_string()), + &recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + Some(vec![pvc]) + } else { + // ...whereas others will use ephemeral volumes + pb.add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &listener_class.to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; + None + }; } pb.add_container(airflow_container.build()); @@ -1128,6 +1121,7 @@ fn build_server_rolegroup_statefulset( }, service_name: rolegroup_ref.object_name(), template: pod_template, + volume_claim_templates: pvcs, ..StatefulSetSpec::default() }; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 5ff5926d..488f5615 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -17,7 +17,7 @@ use stackable_operator::{ }, config::{ fragment::{self, Fragment, ValidationError}, - merge::Merge, + merge::{Atomic, Merge}, }, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, @@ -72,6 +72,9 @@ pub const TEMPLATE_CONFIGMAP_NAME: &str = "airflow-executor-pod-template"; pub const TEMPLATE_LOCATION: &str = "/templates"; pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; + const DEFAULT_AIRFLOW_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(2); const DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); @@ -230,19 +233,9 @@ pub mod versioned { #[serde(default)] pub load_examples: bool, - /// This field controls which type of Service the Operator creates for this AirflowCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// * external-stable: Use a LoadBalancer service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, + pub listener_class: SupportedListenerClasses, /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. @@ -435,27 +428,31 @@ pub struct AirflowOpaConfig { pub cache: UserInformationCache, } -// TODO: Temporary solution until listener-operator is finished #[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "PascalCase")] -pub enum CurrentlySupportedListenerClasses { +pub enum SupportedListenerClasses { #[default] #[serde(rename = "cluster-internal")] + #[strum(serialize = "cluster-internal")] ClusterInternal, #[serde(rename = "external-unstable")] + #[strum(serialize = "external-unstable")] ExternalUnstable, #[serde(rename = "external-stable")] + #[strum(serialize = "external-stable")] ExternalStable, } -impl CurrentlySupportedListenerClasses { - pub fn k8s_service_type(&self) -> String { +impl Atomic for SupportedListenerClasses {} + +impl SupportedListenerClasses { + pub fn discoverable(&self) -> bool { match self { - CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), - CurrentlySupportedListenerClasses::ExternalStable => "LoadBalancer".to_string(), + SupportedListenerClasses::ClusterInternal => false, + SupportedListenerClasses::ExternalUnstable => true, + SupportedListenerClasses::ExternalStable => true, } } } diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index e0e22a70..aa859803 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -10,9 +10,10 @@ ) session = requests.Session() +url = "http://airflow-webserver-default:8080" # Click on "Sign In with keycloak" in Airflow -login_page = session.get("http://airflow-webserver:8080/login/keycloak?next=") +login_page = session.get(f"{url}/login/keycloak?next=") assert login_page.ok, "Redirection from Airflow to Keycloak failed" assert login_page.url.startswith( @@ -27,32 +28,32 @@ ) assert welcome_page.ok, "Login failed" -assert ( - welcome_page.url == "http://airflow-webserver:8080/home" -), "Redirection to the Airflow home page expected" +assert welcome_page.url == f"{url}/home", ( + "Redirection to the Airflow home page expected" +) # Open the user information page in Airflow -userinfo_page = session.get("http://airflow-webserver:8080/users/userinfo/") +userinfo_page = session.get(f"{url}/users/userinfo/") assert userinfo_page.ok, "Retrieving user information failed" -assert ( - userinfo_page.url == "http://airflow-webserver:8080/users/userinfo/" -), "Redirection to the Airflow user info page expected" +assert userinfo_page.url == f"{url}/users/userinfo/", ( + "Redirection to the Airflow user info page expected" +) # Expect the user data provided by Keycloak in Airflow userinfo_page_html = BeautifulSoup(userinfo_page.text, "html.parser") table_rows = userinfo_page_html.find_all("tr") user_data = {tr.find("th").text: tr.find("td").text for tr in table_rows} -assert ( - user_data["First Name"] == "Jane" -), "The first name of the user in Airflow should match the one provided by Keycloak" -assert ( - user_data["Last Name"] == "Doe" -), "The last name of the user in Airflow should match the one provided by Keycloak" -assert ( - user_data["Email"] == "jane.doe@stackable.tech" -), "The email of the user in Airflow should match the one provided by Keycloak" +assert user_data["First Name"] == "Jane", ( + "The first name of the user in Airflow should match the one provided by Keycloak" +) +assert user_data["Last Name"] == "Doe", ( + "The last name of the user in Airflow should match the one provided by Keycloak" +) +assert user_data["Email"] == "jane.doe@stackable.tech", ( + "The email of the user in Airflow should match the one provided by Keycloak" +) # Later this can be extended to use different OIDC providers (currently only Keycloak is # supported) diff --git a/tests/templates/kuttl/opa/41_check-authorization.py b/tests/templates/kuttl/opa/41_check-authorization.py index d6cd11d4..d86502dc 100644 --- a/tests/templates/kuttl/opa/41_check-authorization.py +++ b/tests/templates/kuttl/opa/41_check-authorization.py @@ -26,10 +26,12 @@ "password": "NvfpU518", } +url = "http://airflow-webserver-default:8080" + def create_user(user): requests.post( - "http://airflow-webserver:8080/auth/fab/v1/users", + f"{url}/auth/fab/v1/users", auth=("airflow", "airflow"), json=user, ) @@ -38,7 +40,7 @@ def create_user(user): def check_api_authorization_for_user( user, expected_status_code, method, endpoint, data=None, api="api/v1" ): - api_url = f"http://airflow-webserver:8080/{api}" + api_url = f"{url}/{api}" auth = (user["username"], user["password"]) response = requests.request(method, f"{api_url}/{endpoint}", auth=auth, json=data) @@ -59,18 +61,16 @@ def check_website_authorization_for_user(user, expected_status_code): password = user["password"] with requests.Session() as session: login_response = session.post( - "http://airflow-webserver:8080/login/", + f"{url}/login/", data=f"username={username}&password={password}", allow_redirects=False, headers={"Content-Type": "application/x-www-form-urlencoded"}, ) assert login_response.ok, f"Login for {username} failed" - home_response = session.get( - "http://airflow-webserver:8080/home", allow_redirects=False + home_response = session.get(f"{url}/home", allow_redirects=False) + assert home_response.status_code == expected_status_code, ( + f"GET /home returned status code {home_response.status_code}, but {expected_status_code} was expected." ) - assert ( - home_response.status_code == expected_status_code - ), f"GET /home returned status code {home_response.status_code}, but {expected_status_code} was expected." def test_is_authorized_configuration(): From 7e8456af086c46add640fb54df557b02c834d8db Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 7 Apr 2025 09:55:46 +0200 Subject: [PATCH 02/39] add podrefs to write webserver endpoint to configmap --- .../operator-binary/src/airflow_controller.rs | 62 ++++++++++- rust/operator-binary/src/crd/mod.rs | 92 +++++++++++++++- rust/operator-binary/src/crd/utils.rs | 104 ++++++++++++++++++ rust/operator-binary/src/discovery.rs | 87 +++++++++++++++ rust/operator-binary/src/main.rs | 1 + 5 files changed, 340 insertions(+), 6 deletions(-) create mode 100644 rust/operator-binary/src/crd/utils.rs create mode 100644 rust/operator-binary/src/discovery.rs diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 2c807e26..259ef6bc 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -83,17 +83,19 @@ use crate::{ crd::{ self, AIRFLOW_CONFIG_FILENAME, AIRFLOW_UID, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowConfigOptions, AirflowExecutor, AirflowRole, CONFIG_PATH, Container, ExecutorConfig, - ExecutorConfigFragment, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, - OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, - TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + ExecutorConfigFragment, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, + LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, + TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, authorization::AirflowAuthorizationResolved, build_recommended_labels, git_sync::{GIT_SYNC_CONTENT, GIT_SYNC_NAME, GIT_SYNC_ROOT, GitSync}, + utils::PodRef, v1alpha1, }, + discovery::build_discovery_configmap, env_vars::{ self, build_airflow_template_envs, build_gitsync_statefulset_envs, build_gitsync_template, }, @@ -341,6 +343,17 @@ pub enum Error { LabelBuild { source: stackable_operator::kvp::LabelError, }, + + #[snafu(display("cannot collect discovery configuration"))] + CollectDiscoveryConfig { source: crate::crd::Error }, + + #[snafu(display("failed to apply discovery configmap"))] + ApplyDiscoveryConfigMap { + source: stackable_operator::cluster_resources::Error, + }, + + #[snafu(display("failed to build discovery configmap"))] + BuildDiscoveryConfigMap { source: super::discovery::Error }, } type Result = std::result::Result; @@ -477,6 +490,8 @@ pub async fn reconcile_airflow( .await?; } + let mut listener_refs: BTreeMap> = BTreeMap::new(); + for (role_name, role_config) in validated_role_config.iter() { let airflow_role = AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { @@ -545,6 +560,45 @@ pub async fn reconcile_airflow( .with_context(|_| ApplyRoleGroupConfigSnafu { rolegroup: rolegroup.clone(), })?; + + // if the replicas are changed at the same time as the reconciliation + // being paused, it may be possible to have listeners that are *expected* + // (according to their replica number) but which are not yet created, so + // deactivate this action in such cases. + if airflow.spec.cluster_operation.reconciliation_paused + || airflow.spec.cluster_operation.stopped + { + tracing::info!( + "Cluster is in a transitional state so do not attempt to collect listener information that will only be active once cluster has returned to a non-transitional state." + ); + } else { + listener_refs.insert( + airflow_role.to_string(), + airflow + .listener_refs( + client, + &airflow_role, + airflow.spec.cluster_config.listener_class.clone(), + ) + .await + .context(CollectDiscoveryConfigSnafu)?, + ); + } + } + + tracing::info!( + "Listener references prepared for the ConfigMap {:#?}", + listener_refs + ); + + if !listener_refs.is_empty() { + let endpoint_cm = + build_discovery_configmap(airflow, &resolved_product_image, &listener_refs) + .context(BuildDiscoveryConfigMapSnafu)?; + cluster_resources + .add(client, endpoint_cm) + .await + .context(ApplyDiscoveryConfigMapSnafu)?; } let role_config = airflow.role_config(&airflow_role); @@ -956,7 +1010,7 @@ fn build_server_rolegroup_statefulset( }; airflow_container.readiness_probe(probe.clone()); airflow_container.liveness_probe(probe); - airflow_container.add_container_port("http", http_port.into()); + airflow_container.add_container_port(HTTP_PORT_NAME, http_port.into()); let listener_class = &airflow.spec.cluster_config.listener_class; pvcs = if listener_class.discoverable() { diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 9a0ab099..6fa79b9b 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet}; +use std::collections::{BTreeMap, BTreeSet, HashMap}; use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}; use serde::{Deserialize, Serialize}; @@ -23,7 +23,7 @@ use stackable_operator::{ api::core::v1::{Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, }, - kube::{CustomResource, ResourceExt}, + kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, kvp::ObjectLabels, memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::{self, Configuration}, @@ -43,6 +43,7 @@ use stackable_operator::{ }; use stackable_versioned::versioned; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; +use utils::{PodRef, get_persisted_listener_podrefs}; use crate::crd::{ affinity::{get_affinity, get_executor_affinity}, @@ -57,6 +58,7 @@ pub mod affinity; pub mod authentication; pub mod authorization; pub mod git_sync; +pub mod utils; pub const AIRFLOW_UID: i64 = 1000; pub const APP_NAME: &str = "airflow"; @@ -75,6 +77,8 @@ pub const TEMPLATE_NAME: &str = "airflow_executor_pod_template.yaml"; pub const LISTENER_VOLUME_NAME: &str = "listener"; pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; +pub const HTTP_PORT_NAME: &str = "http"; + const DEFAULT_AIRFLOW_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(2); const DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); @@ -87,10 +91,18 @@ pub const MAX_LOG_FILES_SIZE: MemoryQuantity = MemoryQuantity { pub enum Error { #[snafu(display("Unknown Airflow role found {role}. Should be one of {roles:?}"))] UnknownAirflowRole { role: String, roles: Vec }, + #[snafu(display("fragment validation failure"))] FragmentValidationFailure { source: ValidationError }, + #[snafu(display("Configuration/Executor conflict!"))] NoRoleForExecutorFailure, + + #[snafu(display("object has no associated namespace"))] + NoNamespace, + + #[snafu(display("listener podrefs could not be resolved"))] + ListenerPodRef { source: utils::Error }, } #[derive(Display, EnumIter, EnumString)] @@ -410,6 +422,82 @@ impl v1alpha1::AirflowCluster { tracing::debug!("Merged executor config: {:?}", conf_executor); fragment::validate(conf_executor).context(FragmentValidationFailureSnafu) } + + pub fn rolegroup_ref( + &self, + role_name: impl Into, + group_name: impl Into, + ) -> RoleGroupRef { + RoleGroupRef { + cluster: ObjectRef::from_obj(self), + role: role_name.into(), + role_group: group_name.into(), + } + } + + pub fn rolegroup_ref_and_replicas( + &self, + role: &AirflowRole, + ) -> Vec<(RoleGroupRef, u16)> { + match role { + AirflowRole::Webserver => self + .spec + .webservers + .iter() + .flat_map(|role| &role.role_groups) + // Order rolegroups consistently, to avoid spurious downstream rewrites + .collect::>() + .into_iter() + .map(|(rolegroup_name, role_group)| { + ( + self.rolegroup_ref(AirflowRole::Webserver.to_string(), rolegroup_name), + role_group.replicas.unwrap_or_default(), + ) + }) + .collect(), + AirflowRole::Scheduler | AirflowRole::Worker => vec![], + } + } + + pub fn pod_refs(&self, role: &AirflowRole) -> Result, Error> { + if let Some(port) = role.get_http_port() { + let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; + let rolegroup_ref_and_replicas = self.rolegroup_ref_and_replicas(role); + + Ok(rolegroup_ref_and_replicas + .iter() + .flat_map(|(rolegroup_ref, replicas)| { + let ns = ns.clone(); + (0..*replicas).map(move |i| PodRef { + namespace: ns.clone(), + role_group_service_name: rolegroup_ref.object_name(), + pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), + ports: HashMap::from([("http".to_owned(), port)]), + fqdn_override: None, + }) + }) + .collect()) + } else { + Ok(vec![]) + } + } + + pub async fn listener_refs( + &self, + client: &stackable_operator::client::Client, + role: &AirflowRole, + listener_class: SupportedListenerClasses, + ) -> Result, Error> { + // only externally-reachable listeners are relevant + if listener_class.discoverable() { + let pod_refs = self.pod_refs(role)?; + get_persisted_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) + .await + .context(ListenerPodRefSnafu) + } else { + Ok(vec![]) + } + } } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] diff --git a/rust/operator-binary/src/crd/utils.rs b/rust/operator-binary/src/crd/utils.rs new file mode 100644 index 00000000..c4b7b585 --- /dev/null +++ b/rust/operator-binary/src/crd/utils.rs @@ -0,0 +1,104 @@ +use std::{borrow::Cow, collections::HashMap, num::TryFromIntError}; + +use futures::future::try_join_all; +use snafu::{OptionExt, ResultExt, Snafu}; +use stackable_operator::{ + commons::listener::Listener, k8s_openapi::api::core::v1::Pod, + kube::runtime::reflector::ObjectRef, utils::cluster_info::KubernetesClusterInfo, +}; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("unable to get {listener} (for {pod})"))] + GetPodListener { + source: stackable_operator::client::Error, + listener: ObjectRef, + pod: ObjectRef, + }, + + #[snafu(display("{listener} (for {pod}) has no address"))] + PodListenerHasNoAddress { + listener: ObjectRef, + pod: ObjectRef, + }, + + #[snafu(display("port {port} ({port_name:?}) is out of bounds, must be within {range:?}", range = 0..=u16::MAX))] + PortOutOfBounds { + source: TryFromIntError, + port_name: String, + port: i32, + }, +} + +/// Reference to a single `Pod` that is a component of the product cluster +/// +/// Used for service discovery. +#[derive(Debug)] +pub struct PodRef { + pub namespace: String, + pub role_group_service_name: String, + pub pod_name: String, + pub fqdn_override: Option, + pub ports: HashMap, +} + +impl PodRef { + pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Cow { + self.fqdn_override.as_deref().map_or_else( + || { + Cow::Owned(format!( + "{pod_name}.{role_group_service_name}.{namespace}.svc.{cluster_domain}", + pod_name = self.pod_name, + role_group_service_name = self.role_group_service_name, + namespace = self.namespace, + cluster_domain = cluster_info.cluster_domain, + )) + }, + Cow::Borrowed, + ) + } +} + +pub async fn get_persisted_listener_podrefs( + client: &stackable_operator::client::Client, + pod_refs: Vec, + listener_prefix: &str, +) -> Result, Error> { + try_join_all(pod_refs.into_iter().map(|pod_ref| async { + // N.B. use the naming convention for persistent listener volumes as we + // only want externally-reachable endpoints. + let listener_name = format!("{listener_prefix}-{}", pod_ref.pod_name); + let listener_ref = || ObjectRef::::new(&listener_name).within(&pod_ref.namespace); + let pod_obj_ref = || ObjectRef::::new(&pod_ref.pod_name).within(&pod_ref.namespace); + let listener = client + .get::(&listener_name, &pod_ref.namespace) + .await + .context(GetPodListenerSnafu { + listener: listener_ref(), + pod: pod_obj_ref(), + })?; + let listener_address = listener + .status + .and_then(|s| s.ingress_addresses?.into_iter().next()) + .context(PodListenerHasNoAddressSnafu { + listener: listener_ref(), + pod: pod_obj_ref(), + })?; + Ok(PodRef { + fqdn_override: Some(listener_address.address), + ports: listener_address + .ports + .into_iter() + .map(|(port_name, port)| { + let port = u16::try_from(port).context(PortOutOfBoundsSnafu { + port_name: &port_name, + port, + })?; + Ok((port_name, port)) + }) + .collect::>()?, + ..pod_ref + }) + })) + .await +} diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs new file mode 100644 index 00000000..5467d939 --- /dev/null +++ b/rust/operator-binary/src/discovery.rs @@ -0,0 +1,87 @@ +use std::collections::BTreeMap; + +use snafu::{ResultExt, Snafu}; +use stackable_operator::{ + builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, + commons::product_image_selection::ResolvedProductImage, + k8s_openapi::api::core::v1::ConfigMap, + kube::runtime::reflector::ObjectRef, +}; + +use crate::{ + airflow_controller::AIRFLOW_CONTROLLER_NAME, + crd::{AirflowRole, HTTP_PORT_NAME, build_recommended_labels, utils::PodRef, v1alpha1}, +}; + +type Result = std::result::Result; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("object {airflow} is missing metadata to build owner reference"))] + ObjectMissingMetadataForOwnerRef { + source: stackable_operator::builder::meta::Error, + airflow: ObjectRef, + }, + + #[snafu(display("failed to build ConfigMap"))] + BuildConfigMap { + source: stackable_operator::builder::configmap::Error, + }, + + #[snafu(display("failed to build object meta data"))] + ObjectMeta { + source: stackable_operator::builder::meta::Error, + }, +} + +/// Creates a discovery config map containing the webserver endpoint for clients. +pub fn build_discovery_configmap( + airflow: &v1alpha1::AirflowCluster, + resolved_product_image: &ResolvedProductImage, + role_podrefs: &BTreeMap>, +) -> Result { + let mut cm = ConfigMapBuilder::new(); + + let cmm = cm.metadata( + ObjectMetaBuilder::new() + .name_and_namespace(airflow) + .ownerreference_from_resource(airflow, None, Some(true)) + .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { + airflow: ObjectRef::from_obj(airflow), + })? + .with_recommended_labels(build_recommended_labels( + airflow, + AIRFLOW_CONTROLLER_NAME, + &resolved_product_image.app_version_label, + &AirflowRole::Webserver.to_string(), + "discovery", + )) + .context(ObjectMetaSnafu)? + .build(), + ); + + for role_podref in role_podrefs { + let role_name = role_podref.0; + // podrefs are written into the collection by replica index + // and can be retrieved in the same order + let mut i = 0; + for podref in role_podref.1 { + if let PodRef { + fqdn_override: Some(fqdn_override), + ports, + .. + } = podref + { + if let Some(ui_port) = ports.get(HTTP_PORT_NAME) { + cmm.add_data( + format!("airflow.{role_name}-{i}.http"), + format!("{fqdn_override}:{ui_port}"), + ); + i += 1; + } + } + } + } + + cm.build().context(BuildConfigMapSnafu) +} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index d08a0512..68d9c835 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -30,6 +30,7 @@ mod airflow_controller; mod config; mod controller_commons; mod crd; +mod discovery; mod env_vars; mod operations; mod product_logging; From 7d67a6ba5e24efc6a7a4a2678ec302561a0b15d2 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 8 Apr 2025 12:53:22 +0200 Subject: [PATCH 03/39] moved listener-class to webserver role --- deploy/helm/airflow-operator/crds/crds.yaml | 634 ++++++++++-------- .../operator-binary/src/airflow_controller.rs | 99 ++- rust/operator-binary/src/crd/mod.rs | 153 ++++- rust/operator-binary/src/discovery.rs | 8 +- 4 files changed, 516 insertions(+), 378 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 9c813849..8056e240 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -584,14 +584,6 @@ spec: default: false description: for internal use only - not for production use. type: boolean - listenerClass: - default: cluster-internal - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - enum: - - cluster-internal - - external-unstable - - external-stable - type: string loadExamples: default: false description: Whether to load example DAGs or not; defaults to false. The examples are used in the [getting started guide](https://docs.stackable.tech/home/nightly/airflow/getting_started/). @@ -1295,162 +1287,191 @@ spec: config: default: {} properties: - affinity: + airflowConfig: default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + affinity: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + gracefulShutdownTimeout: null + logging: + containers: {} + enableVectorAgent: null + resources: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. - nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container properties: - configMap: - description: ConfigMap containing the log configuration files + console: + description: Configuration for the console appender nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean - type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. nullable: true - type: string + type: boolean type: object - memory: + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: type: object - type: object - storage: type: object type: object + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string type: object configOverrides: additionalProperties: @@ -1512,162 +1533,191 @@ spec: config: default: {} properties: - affinity: + airflowConfig: default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + affinity: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + gracefulShutdownTimeout: null + logging: + containers: {} + enableVectorAgent: null + resources: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. - nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container properties: - configMap: - description: ConfigMap containing the log configuration files + console: + description: Configuration for the console appender nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean - type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. nullable: true - type: string + type: boolean type: object - memory: + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: type: object - type: object - storage: type: object type: object + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string type: object configOverrides: additionalProperties: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 259ef6bc..c84518f1 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -84,8 +84,8 @@ use crate::{ self, AIRFLOW_CONFIG_FILENAME, AIRFLOW_UID, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowConfigOptions, AirflowExecutor, AirflowRole, CONFIG_PATH, Container, ExecutorConfig, ExecutorConfigFragment, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, - LOG_CONFIG_DIR, OPERATOR_NAME, STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, - TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, + TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -113,9 +113,6 @@ pub const DOCKER_IMAGE_BASE_NAME: &str = "airflow"; pub const AIRFLOW_FULL_CONTROLLER_NAME: &str = concatcp!(AIRFLOW_CONTROLLER_NAME, '.', OPERATOR_NAME); -const METRICS_PORT_NAME: &str = "metrics"; -const METRICS_PORT: i32 = 9102; - pub struct Ctx { pub client: stackable_operator::client::Client, pub product_config: ProductConfigManager, @@ -490,8 +487,6 @@ pub async fn reconcile_airflow( .await?; } - let mut listener_refs: BTreeMap> = BTreeMap::new(); - for (role_name, role_config) in validated_role_config.iter() { let airflow_role = AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { @@ -560,30 +555,27 @@ pub async fn reconcile_airflow( .with_context(|_| ApplyRoleGroupConfigSnafu { rolegroup: rolegroup.clone(), })?; + } - // if the replicas are changed at the same time as the reconciliation - // being paused, it may be possible to have listeners that are *expected* - // (according to their replica number) but which are not yet created, so - // deactivate this action in such cases. - if airflow.spec.cluster_operation.reconciliation_paused - || airflow.spec.cluster_operation.stopped - { - tracing::info!( - "Cluster is in a transitional state so do not attempt to collect listener information that will only be active once cluster has returned to a non-transitional state." - ); - } else { - listener_refs.insert( - airflow_role.to_string(), - airflow - .listener_refs( - client, - &airflow_role, - airflow.spec.cluster_config.listener_class.clone(), - ) - .await - .context(CollectDiscoveryConfigSnafu)?, - ); - } + let mut listener_refs: BTreeMap> = BTreeMap::new(); + // if the replicas are changed at the same time as the reconciliation + // being paused, it may be possible to have listeners that are *expected* + // (according to their replica number) but which are not yet created, so + // deactivate this action in such cases. + if airflow.spec.cluster_operation.reconciliation_paused + || airflow.spec.cluster_operation.stopped + { + tracing::info!( + "Cluster is in a transitional state so do not attempt to collect listener information that will only be active once cluster has returned to a non-transitional state." + ); + } else { + listener_refs.insert( + airflow_role.to_string(), + airflow + .listener_refs(client, &airflow_role) + .await + .context(CollectDiscoveryConfigSnafu)?, + ); } tracing::info!( @@ -606,7 +598,7 @@ pub async fn reconcile_airflow( pod_disruption_budget: pdb, }) = role_config { - add_pdbs(pdb, airflow, &airflow_role, client, &mut cluster_resources) + add_pdbs(&pdb, airflow, &airflow_role, client, &mut cluster_resources) .await .context(FailedToCreatePdbSnafu)?; } @@ -689,7 +681,7 @@ async fn build_executor_template( fn role_ports(port: u16) -> Vec { vec![ServicePort { - name: Some("http".to_string()), + name: Some(HTTP_PORT_NAME.to_owned()), port: port.into(), protocol: Some("TCP".to_string()), ..ServicePort::default() @@ -818,7 +810,7 @@ fn build_rolegroup_service( ) -> Result { let mut ports = vec![ServicePort { name: Some(METRICS_PORT_NAME.into()), - port: METRICS_PORT, + port: METRICS_PORT.into(), protocol: Some("TCP".to_string()), ..Default::default() }]; @@ -1012,25 +1004,30 @@ fn build_server_rolegroup_statefulset( airflow_container.liveness_probe(probe); airflow_container.add_container_port(HTTP_PORT_NAME, http_port.into()); - let listener_class = &airflow.spec.cluster_config.listener_class; - pvcs = if listener_class.discoverable() { + pvcs = if let Some(listener_class) = + airflow.merged_listener_class(airflow_role, &rolegroup_ref.role_group) + { // externally-reachable listener endpoints should use a pvc volume... - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerClass(listener_class.to_string()), - &recommended_labels, - ) - .context(BuildListenerVolumeSnafu)? - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerVolumeSnafu)?; - Some(vec![pvc]) + if listener_class.discoverable() { + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass(listener_class.to_string()), + &recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + Some(vec![pvc]) + } else { + // ...whereas others will use ephemeral volumes + pb.add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &listener_class.to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; + None + } } else { - // ...whereas others will use ephemeral volumes - pb.add_listener_volume_by_listener_class( - LISTENER_VOLUME_NAME, - &listener_class.to_string(), - &recommended_labels, - ) - .context(AddVolumeSnafu)?; None }; } @@ -1056,7 +1053,7 @@ fn build_server_rolegroup_statefulset( ] .join("\n"), ]) - .add_container_port(METRICS_PORT_NAME, METRICS_PORT) + .add_container_port(METRICS_PORT_NAME, METRICS_PORT.into()) .resources( ResourceRequirementsBuilder::new() .with_cpu_request("100m") diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 6fa79b9b..4b514c98 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -34,7 +34,7 @@ use stackable_operator::{ }, role_utils::{ CommonConfiguration, GenericProductSpecificCommonConfig, GenericRoleConfig, Role, - RoleGroupRef, + RoleGroup, RoleGroupRef, }, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, @@ -78,6 +78,9 @@ pub const LISTENER_VOLUME_NAME: &str = "listener"; pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; pub const HTTP_PORT_NAME: &str = "http"; +pub const HTTP_PORT: u16 = 8080; +pub const METRICS_PORT_NAME: &str = "metrics"; +pub const METRICS_PORT: u16 = 9102; const DEFAULT_AIRFLOW_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(2); const DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_minutes_unchecked(5); @@ -202,7 +205,7 @@ pub mod versioned { /// The `webserver` role provides the main UI for user interaction. #[serde(default, skip_serializing_if = "Option::is_none")] - pub webservers: Option>, + pub webservers: Option>, /// The `scheduler` is responsible for triggering jobs and persisting their metadata to the backend database. /// Jobs are scheduled on the workers/executors. @@ -245,10 +248,6 @@ pub mod versioned { #[serde(default)] pub load_examples: bool, - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. - #[serde(default)] - pub listener_class: SupportedListenerClasses, - /// Name of the Vector aggregator [discovery ConfigMap](DOCS_BASE_URL_PLACEHOLDER/concepts/service_discovery). /// It must contain the key `ADDRESS` with the address of the Vector aggregator. /// Follow the [logging tutorial](DOCS_BASE_URL_PLACEHOLDER/tutorials/logging-vector-aggregator) @@ -287,13 +286,20 @@ impl HasStatusCondition for v1alpha1::AirflowCluster { impl v1alpha1::AirflowCluster { /// the worker role will not be returned if airflow provisions pods as needed (i.e. when /// the kubernetes executor is specified) - pub fn get_role(&self, role: &AirflowRole) -> Option<&Role> { + pub fn get_role(&self, role: &AirflowRole) -> Option> { match role { - AirflowRole::Webserver => self.spec.webservers.as_ref(), - AirflowRole::Scheduler => self.spec.schedulers.as_ref(), + AirflowRole::Webserver => { + if let Some(webserver_config) = self.spec.webservers.to_owned() { + let role = extract_role_from_webserver_config(webserver_config); + Some(role) + } else { + None + } + } + AirflowRole::Scheduler => self.spec.schedulers.to_owned(), AirflowRole::Worker => { if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { - Some(config) + Some(config.clone()) } else { None } @@ -301,8 +307,8 @@ impl v1alpha1::AirflowCluster { } } - pub fn role_config(&self, role: &AirflowRole) -> Option<&GenericRoleConfig> { - self.get_role(role).map(|r| &r.role_config) + pub fn role_config(&self, role: &AirflowRole) -> Option { + self.get_role(role).map(|r| r.role_config) } pub fn volumes(&self) -> &Vec { @@ -350,13 +356,12 @@ impl v1alpha1::AirflowCluster { let role = match role { AirflowRole::Webserver => { - self.spec - .webservers - .as_ref() - .context(UnknownAirflowRoleSnafu { + &extract_role_from_webserver_config(self.spec.webservers.to_owned().context( + UnknownAirflowRoleSnafu { role: role.to_string(), roles: AirflowRole::roles(), - })? + }, + )?) } AirflowRole::Worker => { if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { @@ -398,6 +403,34 @@ impl v1alpha1::AirflowCluster { fragment::validate(conf_rolegroup).context(FragmentValidationFailureSnafu) } + pub fn merged_listener_class( + &self, + role: &AirflowRole, + rolegroup_name: &String, + ) -> Option { + if role == &AirflowRole::Webserver { + if let Some(webservers) = self.spec.webservers.as_ref() { + let conf_defaults = Some(SupportedListenerClasses::ClusterInternal); + let mut conf_role = webservers.config.config.listener_class.to_owned(); + let mut conf_rolegroup = webservers + .role_groups + .get(rolegroup_name) + .map(|rg| rg.config.config.listener_class.clone()) + .unwrap_or_default(); + + conf_role.merge(&conf_defaults); + conf_rolegroup.merge(&conf_role); + + tracing::debug!("Merged listener-class: {:?}", conf_rolegroup); + conf_rolegroup + } else { + None + } + } else { + None + } + } + /// Retrieve and merge resource configs for the executor template pub fn merged_executor_config( &self, @@ -448,6 +481,14 @@ impl v1alpha1::AirflowCluster { // Order rolegroups consistently, to avoid spurious downstream rewrites .collect::>() .into_iter() + .filter(|(rolegroup_name, _)| { + let listener_class = self.merged_listener_class(role, rolegroup_name); + if let Some(listener_class) = listener_class { + listener_class.discoverable() + } else { + false + } + }) .map(|(rolegroup_name, role_group)| { ( self.rolegroup_ref(AirflowRole::Webserver.to_string(), rolegroup_name), @@ -472,7 +513,10 @@ impl v1alpha1::AirflowCluster { namespace: ns.clone(), role_group_service_name: rolegroup_ref.object_name(), pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), - ports: HashMap::from([("http".to_owned(), port)]), + ports: HashMap::from([ + (HTTP_PORT_NAME.to_owned(), port), + (METRICS_PORT_NAME.to_owned(), METRICS_PORT), + ]), fqdn_override: None, }) }) @@ -486,17 +530,44 @@ impl v1alpha1::AirflowCluster { &self, client: &stackable_operator::client::Client, role: &AirflowRole, - listener_class: SupportedListenerClasses, ) -> Result, Error> { - // only externally-reachable listeners are relevant - if listener_class.discoverable() { - let pod_refs = self.pod_refs(role)?; - get_persisted_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) - .await - .context(ListenerPodRefSnafu) - } else { - Ok(vec![]) - } + let pod_refs = self.pod_refs(role)?; + get_persisted_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) + .await + .context(ListenerPodRefSnafu) + } +} + +fn extract_role_from_webserver_config( + fragment: Role, +) -> Role { + Role { + config: CommonConfiguration { + config: fragment.config.config.airflow_config, + config_overrides: fragment.config.config_overrides, + env_overrides: fragment.config.env_overrides, + cli_overrides: fragment.config.cli_overrides, + pod_overrides: fragment.config.pod_overrides, + product_specific_common_config: fragment.config.product_specific_common_config, + }, + role_config: fragment.role_config, + role_groups: fragment + .role_groups + .into_iter() + .map(|(k, v)| { + (k, RoleGroup { + config: CommonConfiguration { + config: v.config.config.airflow_config, + config_overrides: v.config.config_overrides, + env_overrides: v.config.env_overrides, + cli_overrides: v.config.cli_overrides, + pod_overrides: v.config.pod_overrides, + product_specific_common_config: v.config.product_specific_common_config, + }, + replicas: v.replicas, + }) + }) + .collect(), } } @@ -695,7 +766,7 @@ impl AirflowRole { /// created as services. pub fn get_http_port(&self) -> Option { match &self { - AirflowRole::Webserver => Some(8080), + AirflowRole::Webserver => Some(HTTP_PORT), AirflowRole::Scheduler => None, AirflowRole::Worker => None, } @@ -797,6 +868,30 @@ pub struct ExecutorConfig { pub graceful_shutdown_timeout: Option, } +#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] +#[fragment_attrs( + derive( + Clone, + Debug, + Default, + Deserialize, + Merge, + JsonSchema, + PartialEq, + Serialize + ), + serde(rename_all = "camelCase") +)] +pub struct WebserverConfig { + #[fragment_attrs(serde(default))] + #[serde(flatten)] + pub airflow_config: AirflowConfig, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + #[serde(default)] + pub listener_class: SupportedListenerClasses, +} + #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] #[fragment_attrs( derive( diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 5467d939..33671ab6 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -61,23 +61,19 @@ pub fn build_discovery_configmap( ); for role_podref in role_podrefs { - let role_name = role_podref.0; - // podrefs are written into the collection by replica index - // and can be retrieved in the same order - let mut i = 0; for podref in role_podref.1 { if let PodRef { fqdn_override: Some(fqdn_override), ports, + pod_name, .. } = podref { if let Some(ui_port) = ports.get(HTTP_PORT_NAME) { cmm.add_data( - format!("airflow.{role_name}-{i}.http"), + format!("{pod_name}.http"), format!("{fqdn_override}:{ui_port}"), ); - i += 1; } } } From 6c2e2b3703baaba35bb4223306275ad8466163a7 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 9 Apr 2025 11:42:17 +0200 Subject: [PATCH 04/39] move config map creation out of role-loop --- .../operator-binary/src/airflow_controller.rs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index c84518f1..994595e9 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -487,6 +487,8 @@ pub async fn reconcile_airflow( .await?; } + let mut listener_refs: BTreeMap> = BTreeMap::new(); + for (role_name, role_config) in validated_role_config.iter() { let airflow_role = AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { @@ -557,7 +559,6 @@ pub async fn reconcile_airflow( })?; } - let mut listener_refs: BTreeMap> = BTreeMap::new(); // if the replicas are changed at the same time as the reconciliation // being paused, it may be possible to have listeners that are *expected* // (according to their replica number) but which are not yet created, so @@ -578,21 +579,6 @@ pub async fn reconcile_airflow( ); } - tracing::info!( - "Listener references prepared for the ConfigMap {:#?}", - listener_refs - ); - - if !listener_refs.is_empty() { - let endpoint_cm = - build_discovery_configmap(airflow, &resolved_product_image, &listener_refs) - .context(BuildDiscoveryConfigMapSnafu)?; - cluster_resources - .add(client, endpoint_cm) - .await - .context(ApplyDiscoveryConfigMapSnafu)?; - } - let role_config = airflow.role_config(&airflow_role); if let Some(GenericRoleConfig { pod_disruption_budget: pdb, @@ -604,6 +590,21 @@ pub async fn reconcile_airflow( } } + tracing::info!( + "Listener references prepared for the ConfigMap {:#?}", + listener_refs + ); + + if !listener_refs.is_empty() { + let endpoint_cm = + build_discovery_configmap(airflow, &resolved_product_image, &listener_refs) + .context(BuildDiscoveryConfigMapSnafu)?; + cluster_resources + .add(client, endpoint_cm) + .await + .context(ApplyDiscoveryConfigMapSnafu)?; + } + cluster_resources .delete_orphaned_resources(client) .await From f2500598517b8f6047b0c130225cc87a78b32247 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 9 Apr 2025 11:57:06 +0200 Subject: [PATCH 05/39] merge conflicts --- nix/sources.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nix/sources.json b/nix/sources.json index 78d7121b..5aee1c9d 100644 --- a/nix/sources.json +++ b/nix/sources.json @@ -29,10 +29,10 @@ "homepage": "", "owner": "NixOS", "repo": "nixpkgs", - "rev": "b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b", - "sha256": "1y8kwbb5b0r1m88nk871ai56qi2drygvibjgc2swp48jfyp5ya99", + "rev": "b2b0718004cc9a5bca610326de0a82e6ea75920b", + "sha256": "0aqrxx1w40aqicjhg2057bpyrrbsx6mnii5dp5klpm4labfg2iwi", "type": "tarball", - "url": "https://github.com/NixOS/nixpkgs/archive/b0b4b5f8f621bfe213b8b21694bab52ecfcbf30b.tar.gz", + "url": "https://github.com/NixOS/nixpkgs/archive/b2b0718004cc9a5bca610326de0a82e6ea75920b.tar.gz", "url_template": "https://github.com///archive/.tar.gz" } } From b6b969e883039cf18cc21f739c645369abcebfe5 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 9 Apr 2025 14:13:41 +0200 Subject: [PATCH 06/39] regenerate nix --- Cargo.nix | 1184 +++++++++++++++++------------------------------------ 1 file changed, 382 insertions(+), 802 deletions(-) diff --git a/Cargo.nix b/Cargo.nix index eb024b05..51cd3831 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -149,7 +149,7 @@ rec { } { name = "getrandom"; - packageId = "getrandom"; + packageId = "getrandom 0.2.15"; optional = true; } { @@ -466,9 +466,9 @@ rec { }; "async-trait" = rec { crateName = "async-trait"; - version = "0.1.87"; + version = "0.1.88"; edition = "2021"; - sha256 = "15swwmyl4nx7w03rq6ibb4x2c8rzbx9fpiag1kn4fhapb49yqmnm"; + sha256 = "1dgxvz7g75cmz6vqqz0mri4xazc6a8xfj1db6r9fxz29lzyd6fg5"; procMacro = true; libName = "async_trait"; authors = [ @@ -1221,9 +1221,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.16"; + version = "1.2.18"; edition = "2018"; - sha256 = "131bhgafc1i86vvjipkj0kwzz0hlpwrkl8mdbmzyq2g69calqwdy"; + sha256 = "0p6d2pfyrjgqpf2w399wzj4hmyffj6g0gyzg3pdy6xl3gmhlcl2j"; authors = [ "Alex Crichton " ]; @@ -1329,10 +1329,10 @@ rec { }; "clap" = rec { crateName = "clap"; - version = "4.5.32"; + version = "4.5.35"; edition = "2021"; crateBin = []; - sha256 = "10vg2fbcsy0dwxdqpdqihxl8b935310lax6dc29d221nijpg7230"; + sha256 = "0i1rnz7mwbhs5qf10r6vmrkplkzm3477khkwz189rha49f9qdanq"; dependencies = [ { name = "clap_builder"; @@ -1371,9 +1371,9 @@ rec { }; "clap_builder" = rec { crateName = "clap_builder"; - version = "4.5.32"; + version = "4.5.35"; edition = "2021"; - sha256 = "1j5cdwdry9anb8ljzqymb15byghz8jcpzafshbxysmb1cxzyz9r2"; + sha256 = "1nczcw6cc49ap99nn3v3n0vrv7j74zin34palq6ji586vnrdn514"; dependencies = [ { name = "anstream"; @@ -1697,9 +1697,9 @@ rec { }; "crossbeam-channel" = rec { crateName = "crossbeam-channel"; - version = "0.5.14"; + version = "0.5.15"; edition = "2021"; - sha256 = "0wa41qybq5w8s70anb472myh4fid4aw6v65vws6wn528w9l6vfh6"; + sha256 = "1cicd9ins0fkpfgvz9vhz3m9rpkh6n8d3437c3wnfsdkd3wgif42"; libName = "crossbeam_channel"; dependencies = [ { @@ -1754,9 +1754,9 @@ rec { }; "darling" = rec { crateName = "darling"; - version = "0.20.10"; + version = "0.20.11"; edition = "2021"; - sha256 = "1299h2z88qn71mizhh05j26yr3ik0wnqmw11ijds89l8i9nbhqvg"; + sha256 = "1vmlphlrlw4f50z16p4bc9p5qwdni1ba95qmxfrrmzs6dh8lczzw"; authors = [ "Ted Driggs " ]; @@ -1779,9 +1779,9 @@ rec { }; "darling_core" = rec { crateName = "darling_core"; - version = "0.20.10"; + version = "0.20.11"; edition = "2021"; - sha256 = "1rgr9nci61ahnim93yh3xy6fkfayh7sk4447hahawah3m1hkh4wm"; + sha256 = "0bj1af6xl4ablnqbgn827m43b8fiicgv180749f5cphqdmcvj00d"; authors = [ "Ted Driggs " ]; @@ -1821,9 +1821,9 @@ rec { }; "darling_macro" = rec { crateName = "darling_macro"; - version = "0.20.10"; + version = "0.20.11"; edition = "2021"; - sha256 = "01kq3ibbn47czijj39h3vxyw0c2ksd0jvc097smcrk7n2jjs4dnk"; + sha256 = "1bbfbc2px6sj1pqqq97bgqn6c8xdnb2fmz66f7f40nrqrcybjd7w"; procMacro = true; authors = [ "Ted Driggs " @@ -1846,9 +1846,9 @@ rec { }; "delegate" = rec { crateName = "delegate"; - version = "0.13.2"; + version = "0.13.3"; edition = "2018"; - sha256 = "0ig9x6wiwfqkqbk4wh9frbihl0jq50vsi4jpn5kd02pkiqqhcy19"; + sha256 = "088d919b991lz5bj5k989ab33dzjsi8pdx8whsbnzlmy5cy4idmr"; procMacro = true; authors = [ "Godfrey Chan " @@ -1873,9 +1873,9 @@ rec { }; "deranged" = rec { crateName = "deranged"; - version = "0.3.11"; + version = "0.4.0"; edition = "2021"; - sha256 = "1d1ibqqnr5qdrpw8rclwrf1myn3wf0dygl04idf4j2s49ah6yaxl"; + sha256 = "13h6skwk411wzhf1l9l7d3yz5y6vg9d7s3dwhhb4a942r88nm7lw"; authors = [ "Jacob Pratt " ]; @@ -1889,10 +1889,13 @@ rec { ]; features = { "default" = [ "std" ]; + "macros" = [ "dep:deranged-macros" ]; "num" = [ "dep:num-traits" ]; "powerfmt" = [ "dep:powerfmt" ]; "quickcheck" = [ "dep:quickcheck" "alloc" ]; - "rand" = [ "dep:rand" ]; + "rand" = [ "rand08" "rand09" ]; + "rand08" = [ "dep:rand08" ]; + "rand09" = [ "dep:rand09" ]; "serde" = [ "dep:serde" ]; "std" = [ "alloc" ]; }; @@ -2208,9 +2211,9 @@ rec { }; "event-listener-strategy" = rec { crateName = "event-listener-strategy"; - version = "0.5.3"; + version = "0.5.4"; edition = "2021"; - sha256 = "1ch5gf6knllyq12jkb5zdfag573dh44307q4pwwi2g37sc6lwgiw"; + sha256 = "14rv18av8s7n8yixg38bxp5vg2qs394rl1w052by5npzmbgz7scb"; libName = "event_listener_strategy"; authors = [ "John Nunley " @@ -2229,6 +2232,7 @@ rec { features = { "default" = [ "std" ]; "loom" = [ "event-listener/loom" ]; + "portable-atomic" = [ "event-listener/portable-atomic" ]; "std" = [ "event-listener/std" ]; }; resolvedDefaultFeatures = [ "default" "std" ]; @@ -2349,9 +2353,9 @@ rec { }; "foldhash" = rec { crateName = "foldhash"; - version = "0.1.4"; + version = "0.1.5"; edition = "2021"; - sha256 = "0vsxw2iwpgs7yy6l7pndm7b8nllaq5vdxwnmjn1qpm5kyzhzvlm0"; + sha256 = "1wisr1xlc2bj7hk4rgkcjkz3j2x4dhd1h9lwk7mj8p71qpdgbi6r"; authors = [ "Orson Peters " ]; @@ -2721,7 +2725,7 @@ rec { }; resolvedDefaultFeatures = [ "more_lengths" ]; }; - "getrandom" = rec { + "getrandom 0.2.15" = rec { crateName = "getrandom"; version = "0.2.15"; edition = "2018"; @@ -2742,7 +2746,7 @@ rec { } { name = "wasi"; - packageId = "wasi"; + packageId = "wasi 0.11.0+wasi-snapshot-preview1"; usesDefaultFeatures = false; target = { target, features }: ("wasi" == target."os" or null); } @@ -2757,6 +2761,86 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "getrandom 0.3.2" = rec { + crateName = "getrandom"; + version = "0.3.2"; + edition = "2021"; + sha256 = "1w2mlixa1989v7czr68iji7h67yra2pbg3s480wsqjza1r2sizkk"; + authors = [ + "The Rand Project Developers" + ]; + dependencies = [ + { + name = "cfg-if"; + packageId = "cfg-if"; + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ((("linux" == target."os" or null) || ("android" == target."os" or null)) && (!((("linux" == target."os" or null) && ("" == target."env" or null)) || ("custom" == target."getrandom_backend" or null) || ("linux_raw" == target."getrandom_backend" or null) || ("rdrand" == target."getrandom_backend" or null) || ("rndr" == target."getrandom_backend" or null)))); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("dragonfly" == target."os" or null) || ("freebsd" == target."os" or null) || ("hurd" == target."os" or null) || ("illumos" == target."os" or null) || ("cygwin" == target."os" or null) || (("horizon" == target."os" or null) && ("arm" == target."arch" or null))); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("haiku" == target."os" or null) || ("redox" == target."os" or null) || ("nto" == target."os" or null) || ("aix" == target."os" or null)); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("ios" == target."os" or null) || ("visionos" == target."os" or null) || ("watchos" == target."os" or null) || ("tvos" == target."os" or null)); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: (("macos" == target."os" or null) || ("openbsd" == target."os" or null) || ("vita" == target."os" or null) || ("emscripten" == target."os" or null)); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ("netbsd" == target."os" or null); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ("solaris" == target."os" or null); + } + { + name = "libc"; + packageId = "libc"; + usesDefaultFeatures = false; + target = { target, features }: ("vxworks" == target."os" or null); + } + { + name = "r-efi"; + packageId = "r-efi"; + usesDefaultFeatures = false; + target = { target, features }: (("uefi" == target."os" or null) && ("efi_rng" == target."getrandom_backend" or null)); + } + { + name = "wasi"; + packageId = "wasi 0.14.2+wasi-0.2.4"; + usesDefaultFeatures = false; + target = { target, features }: (("wasm32" == target."arch" or null) && ("wasi" == target."os" or null) && ("p2" == target."env" or null)); + } + ]; + features = { + "rustc-dep-of-std" = [ "dep:compiler_builtins" "dep:core" ]; + "wasm_js" = [ "dep:wasm-bindgen" "dep:js-sys" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "gimli" = rec { crateName = "gimli"; version = "0.31.1"; @@ -2776,9 +2860,9 @@ rec { }; "git2" = rec { crateName = "git2"; - version = "0.20.0"; + version = "0.20.1"; edition = "2018"; - sha256 = "1zwav0r76njd9chqxh7wj4r4zfn08nzsisrg05liyd6cjf4piniz"; + sha256 = "1fgf67h78yrw2gm1n8ghgr0jwsbkvmjfhnbng9zrm2n68jxbh82j"; authors = [ "Josh Triplett " "Alex Crichton " @@ -2900,7 +2984,7 @@ rec { } { name = "indexmap"; - packageId = "indexmap 2.8.0"; + packageId = "indexmap 2.9.0"; features = [ "std" ]; } { @@ -3085,13 +3169,9 @@ rec { }; "hostname" = rec { crateName = "hostname"; - version = "0.4.0"; + version = "0.4.1"; edition = "2021"; - sha256 = "1fpjr3vgi64ly1ci8phdqjbha4k22c65c94a9drriiqnmk4cgizr"; - authors = [ - "fengcen " - "svartalf " - ]; + sha256 = "0rbxryl68bwv8hkjdjd8f37kdb10fncgsqrqksv64qy7s4y20vx5"; dependencies = [ { name = "cfg-if"; @@ -3103,10 +3183,9 @@ rec { target = { target, features }: ((target."unix" or false) || ("redox" == target."os" or null)); } { - name = "windows"; - packageId = "windows"; + name = "windows-link"; + packageId = "windows-link"; target = { target, features }: ("windows" == target."os" or null); - features = [ "Win32_Foundation" "Win32_System_SystemInformation" ]; } ]; features = { @@ -3584,9 +3663,9 @@ rec { }; "hyper-util" = rec { crateName = "hyper-util"; - version = "0.1.10"; + version = "0.1.11"; edition = "2021"; - sha256 = "1d1iwrkysjhq63pg54zk3vfby1j7zmxzm9zzyfr4lwvp0szcybfz"; + sha256 = "1wj3svb1r6yv6kgk5fsz6wwajmngc4zxcw4wxpwlmpbgl8rvqys9"; libName = "hyper_util"; authors = [ "Sean McArthur " @@ -3618,6 +3697,11 @@ rec { name = "hyper"; packageId = "hyper"; } + { + name = "libc"; + packageId = "libc"; + optional = true; + } { name = "pin-project-lite"; packageId = "pin-project-lite"; @@ -3665,8 +3749,8 @@ rec { ]; features = { "client" = [ "hyper/client" "dep:tracing" "dep:futures-channel" "dep:tower-service" ]; - "client-legacy" = [ "client" "dep:socket2" "tokio/sync" ]; - "full" = [ "client" "client-legacy" "server" "server-auto" "server-graceful" "service" "http1" "http2" "tokio" ]; + "client-legacy" = [ "client" "dep:socket2" "tokio/sync" "dep:libc" ]; + "full" = [ "client" "client-legacy" "server" "server-auto" "server-graceful" "service" "http1" "http2" "tokio" "tracing" ]; "http1" = [ "hyper/http1" ]; "http2" = [ "hyper/http2" ]; "server" = [ "hyper/server" ]; @@ -3674,14 +3758,15 @@ rec { "server-graceful" = [ "server" "tokio/sync" "futures-util/alloc" ]; "service" = [ "dep:tower-service" ]; "tokio" = [ "dep:tokio" "tokio/net" "tokio/rt" "tokio/time" ]; + "tracing" = [ "dep:tracing" ]; }; resolvedDefaultFeatures = [ "client" "client-legacy" "default" "http1" "http2" "server" "server-auto" "service" "tokio" ]; }; "iana-time-zone" = rec { crateName = "iana-time-zone"; - version = "0.1.61"; - edition = "2018"; - sha256 = "085jjsls330yj1fnwykfzmb2f10zp6l7w4fhq81ng81574ghhpi3"; + version = "0.1.63"; + edition = "2021"; + sha256 = "1n171f5lbc7bryzmp1h30zw86zbvl5480aq02z92lcdwvvjikjdh"; libName = "iana_time_zone"; authors = [ "Andrew Straw " @@ -3697,7 +3782,7 @@ rec { { name = "core-foundation-sys"; packageId = "core-foundation-sys"; - target = { target, features }: (("macos" == target."os" or null) || ("ios" == target."os" or null)); + target = { target, features }: ("apple" == target."vendor" or null); } { name = "iana-time-zone-haiku"; @@ -3709,6 +3794,11 @@ rec { packageId = "js-sys"; target = { target, features }: (("wasm32" == target."arch" or null) && ("unknown" == target."os" or null)); } + { + name = "log"; + packageId = "log"; + target = { target, features }: (("wasm32" == target."arch" or null) && ("unknown" == target."os" or null)); + } { name = "wasm-bindgen"; packageId = "wasm-bindgen"; @@ -3881,9 +3971,9 @@ rec { }; "icu_locid_transform_data" = rec { crateName = "icu_locid_transform_data"; - version = "1.5.0"; + version = "1.5.1"; edition = "2021"; - sha256 = "0vkgjixm0wzp2n3v5mw4j89ly05bg3lx96jpdggbwlpqi0rzzj7x"; + sha256 = "07gignya9gzynnyds88bmra4blq9jxzgrcss43vzk2q9h7byc5bm"; authors = [ "The ICU4X Project Developers" ]; @@ -3971,9 +4061,9 @@ rec { }; "icu_normalizer_data" = rec { crateName = "icu_normalizer_data"; - version = "1.5.0"; + version = "1.5.1"; edition = "2021"; - sha256 = "05lmk0zf0q7nzjnj5kbmsigj3qgr0rwicnn5pqi9n7krmbvzpjpq"; + sha256 = "1dqcm86spcqcs4jnra81yqq3g5bpw6bvf5iz621spj5x52137s65"; authors = [ "The ICU4X Project Developers" ]; @@ -4042,9 +4132,9 @@ rec { }; "icu_properties_data" = rec { crateName = "icu_properties_data"; - version = "1.5.0"; + version = "1.5.1"; edition = "2021"; - sha256 = "0scms7pd5a7yxx9hfl167f5qdf44as6r3bd8myhlngnxqgxyza37"; + sha256 = "1qm5vf17nyiwb87s3g9x9fsj32gkv4a7q7d2sblawx9vfncqgyw5"; authors = [ "The ICU4X Project Developers" ]; @@ -4238,11 +4328,11 @@ rec { "serde-1" = [ "serde" ]; }; }; - "indexmap 2.8.0" = rec { + "indexmap 2.9.0" = rec { crateName = "indexmap"; - version = "2.8.0"; + version = "2.9.0"; edition = "2021"; - sha256 = "0n3hkpzch6q3wgzh8g8hiyac6kk3vgd8nfsxy8mi80jvw47xam1r"; + sha256 = "07m15a571yywmvqyb7ms717q9n42b46badbpsmx215jrg7dhv9yf"; dependencies = [ { name = "equivalent"; @@ -4363,13 +4453,19 @@ rec { }; "jobserver" = rec { crateName = "jobserver"; - version = "0.1.32"; + version = "0.1.33"; edition = "2021"; - sha256 = "1l2k50qmj84x9mn39ivjz76alqmx72jhm12rw33zx9xnpv5xpla8"; + sha256 = "12jkn3cxvfs7jsb6knmh9y2b41lwmrk3vdqywkmssx61jzq65wiq"; authors = [ "Alex Crichton " ]; dependencies = [ + { + name = "getrandom"; + packageId = "getrandom 0.3.2"; + target = { target, features }: (target."windows" or false); + features = [ "std" ]; + } { name = "libc"; packageId = "libc"; @@ -5196,10 +5292,10 @@ rec { }; "libgit2-sys" = rec { crateName = "libgit2-sys"; - version = "0.18.0+1.9.0"; + version = "0.18.1+1.9.0"; edition = "2018"; links = "git2"; - sha256 = "1v7zcw1kky338grxs70y7fwpy70g846bpa5yzvl9f5bybr31g8g1"; + sha256 = "03i98nb84aa99bn7sxja11pllq6fghsaw4d3qwjxikgzhh7v5p71"; libName = "libgit2_sys"; libPath = "lib.rs"; authors = [ @@ -5240,10 +5336,10 @@ rec { }; "libz-sys" = rec { crateName = "libz-sys"; - version = "1.1.21"; + version = "1.1.22"; edition = "2018"; links = "z"; - sha256 = "1ajfpf413j9m7kmf4fwvvgv5jxxm5s438f2pfbv2c2vf1vjni6yz"; + sha256 = "07b5wxh0ska996kc0g2hanjhmb4di7ksm6ndljhr4pi0vykyfw4b"; libName = "libz_sys"; authors = [ "Alex Crichton " @@ -5327,9 +5423,9 @@ rec { }; "log" = rec { crateName = "log"; - version = "0.4.26"; + version = "0.4.27"; edition = "2021"; - sha256 = "17mvchkvhnm2zxyfagh2g9p861f0qx2g1sg2v14sww9nvjry5g9h"; + sha256 = "150x589dqil307rv0rwj0jsgz5bjbwvl83gyl61jf873a7rjvp0k"; authors = [ "The Rust Project Developers" ]; @@ -5420,9 +5516,9 @@ rec { }; "miniz_oxide" = rec { crateName = "miniz_oxide"; - version = "0.8.5"; + version = "0.8.8"; edition = "2021"; - sha256 = "1r9whkc61xri7m1cn4rjrjlhr32ab29nvfxcbg0ri5mmpgg08glf"; + sha256 = "0al9iy33flfgxawj789w2c8xxwg1n2r5vv6m6p5hl2fvd2vlgriv"; authors = [ "Frommi " "oyvindln " @@ -5441,6 +5537,7 @@ rec { "core" = [ "dep:core" ]; "default" = [ "with-alloc" ]; "rustc-dep-of-std" = [ "core" "alloc" "compiler_builtins" "adler2/rustc-dep-of-std" ]; + "serde" = [ "dep:serde" ]; "simd" = [ "simd-adler32" ]; "simd-adler32" = [ "dep:simd-adler32" ]; }; @@ -5474,7 +5571,7 @@ rec { } { name = "wasi"; - packageId = "wasi"; + packageId = "wasi 0.11.0+wasi-snapshot-preview1"; target = { target, features }: ("wasi" == target."os" or null); } { @@ -5588,9 +5685,9 @@ rec { }; "once_cell" = rec { crateName = "once_cell"; - version = "1.21.1"; + version = "1.21.3"; edition = "2021"; - sha256 = "1g645fg3rk800ica72ypwajllgij38az3n831sm2rragrknhnnyp"; + sha256 = "0b9x77lb9f1j6nqgf5aka4s2qj0nly176bpbrv6f9iakk5ff3xa2"; authors = [ "Aleksey Kladov " ]; @@ -6213,9 +6310,9 @@ rec { }; "pest" = rec { crateName = "pest"; - version = "2.7.15"; + version = "2.8.0"; edition = "2021"; - sha256 = "1p4rq45xprw9cx0pb8mmbfa0ih49l0baablv3cpfdy3c1pkayz4b"; + sha256 = "1dp741bxqiracvvwl66mfvlr29byvvph28n4c6ip136m652vg38r"; authors = [ "Dragoș Tiselice " ]; @@ -6247,9 +6344,9 @@ rec { }; "pest_derive" = rec { crateName = "pest_derive"; - version = "2.7.15"; + version = "2.8.0"; edition = "2021"; - sha256 = "0zpmcd1jv1c53agad5b3jb66ylxlzyv43x1bssh8fs7w3i11hrc1"; + sha256 = "1icp5i01mgpbgwbkrcy4d0ykbxmns4wyz8j1jg6dr1wysz7xj9fp"; procMacro = true; authors = [ "Dragoș Tiselice " @@ -6276,9 +6373,9 @@ rec { }; "pest_generator" = rec { crateName = "pest_generator"; - version = "2.7.15"; + version = "2.8.0"; edition = "2021"; - sha256 = "0yrpk5ymc56pffv7gqr5rkv92p3dc6s73lb8hy1wf3w77byrc4vx"; + sha256 = "0hgqngsxfr8y5p47bgjvd038j55ix1x4dpzr6amndaz8ddr02zfv"; authors = [ "Dragoș Tiselice " ]; @@ -6315,9 +6412,9 @@ rec { }; "pest_meta" = rec { crateName = "pest_meta"; - version = "2.7.15"; + version = "2.8.0"; edition = "2021"; - sha256 = "1skx7gm932bp77if63f7d72jrk5gygj39d8zsfzigmr5xa4q1rg1"; + sha256 = "182w5fyiqm7zbn0p8313xc5wc73rnn59ycm5zk8hcja9f0j877vz"; authors = [ "Dragoș Tiselice " ]; @@ -6438,7 +6535,7 @@ rec { dependencies = [ { name = "zerocopy"; - packageId = "zerocopy 0.8.23"; + packageId = "zerocopy 0.8.24"; features = [ "simd" ]; } ]; @@ -6631,6 +6728,19 @@ rec { }; resolvedDefaultFeatures = [ "default" "proc-macro" ]; }; + "r-efi" = rec { + crateName = "r-efi"; + version = "5.2.0"; + edition = "2018"; + sha256 = "1ig93jvpqyi87nc5kb6dri49p56q7r7qxrn8kfizmqkfj5nmyxkl"; + libName = "r_efi"; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "examples" = [ "native" ]; + "rustc-dep-of-std" = [ "compiler_builtins/rustc-dep-of-std" "core" ]; + }; + }; "rand" = rec { crateName = "rand"; version = "0.8.5"; @@ -6717,7 +6827,7 @@ rec { dependencies = [ { name = "getrandom"; - packageId = "getrandom"; + packageId = "getrandom 0.2.15"; optional = true; } ]; @@ -6731,9 +6841,9 @@ rec { }; "redox_syscall" = rec { crateName = "redox_syscall"; - version = "0.5.10"; + version = "0.5.11"; edition = "2021"; - sha256 = "1l9b638qx72312yzh8ykvda9b3lqd9gf6yqn66b23a331ck0r30b"; + sha256 = "18qijn18r10haiglv4261wb0yh1agqqlvs0nxfy8yjbpsb307wfj"; libName = "syscall"; authors = [ "Jeremy Soller " @@ -7199,7 +7309,7 @@ rec { } { name = "getrandom"; - packageId = "getrandom"; + packageId = "getrandom 0.2.15"; } { name = "libc"; @@ -7375,9 +7485,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.23"; + version = "0.23.25"; edition = "2021"; - sha256 = "15gk2bmry78cps3ya38a7cn4jxc36xv1r7gndr0fbz40qjc6qya7"; + sha256 = "0g5idwxm04i71k3n66ml30zyfbgv6p85a7jky2i09v64i8cfjbl2"; dependencies = [ { name = "log"; @@ -7426,10 +7536,10 @@ rec { ]; features = { "aws-lc-rs" = [ "aws_lc_rs" ]; - "aws_lc_rs" = [ "dep:aws-lc-rs" "webpki/aws_lc_rs" ]; + "aws_lc_rs" = [ "dep:aws-lc-rs" "webpki/aws-lc-rs" "aws-lc-rs/aws-lc-sys" "aws-lc-rs/prebuilt-nasm" ]; "brotli" = [ "dep:brotli" "dep:brotli-decompressor" "std" ]; "default" = [ "aws_lc_rs" "logging" "std" "tls12" ]; - "fips" = [ "aws_lc_rs" "aws-lc-rs?/fips" ]; + "fips" = [ "aws_lc_rs" "aws-lc-rs?/fips" "webpki/aws-lc-rs-fips" ]; "hashbrown" = [ "dep:hashbrown" ]; "log" = [ "dep:log" ]; "logging" = [ "log" ]; @@ -7542,9 +7652,9 @@ rec { }; "rustls-webpki" = rec { crateName = "rustls-webpki"; - version = "0.102.8"; + version = "0.103.1"; edition = "2021"; - sha256 = "1sdy8ks86b7jpabpnb2px2s7f1sq8v0nqf6fnlvwzm4vfk41pjk4"; + sha256 = "00rcdz0rb9ia2ivrq7412ry9qkvbh78pra2phl4p7kxck9vbiy7y"; libName = "webpki"; dependencies = [ { @@ -7566,8 +7676,9 @@ rec { ]; features = { "alloc" = [ "ring?/alloc" "pki-types/alloc" ]; - "aws_lc_rs" = [ "dep:aws-lc-rs" ]; - "default" = [ "std" "ring" ]; + "aws-lc-rs" = [ "dep:aws-lc-rs" "aws-lc-rs/aws-lc-sys" "aws-lc-rs/prebuilt-nasm" ]; + "aws-lc-rs-fips" = [ "dep:aws-lc-rs" "aws-lc-rs/fips" ]; + "default" = [ "std" ]; "ring" = [ "dep:ring" ]; "std" = [ "alloc" "pki-types/std" ]; }; @@ -8109,7 +8220,7 @@ rec { dependencies = [ { name = "indexmap"; - packageId = "indexmap 2.8.0"; + packageId = "indexmap 2.9.0"; } { name = "itoa"; @@ -8285,18 +8396,21 @@ rec { }; "smallvec" = rec { crateName = "smallvec"; - version = "1.14.0"; + version = "1.15.0"; edition = "2018"; - sha256 = "1z8wpr53x6jisklqhkkvkgyi8s5cn69h2d2alhqfxahzxwiq7kvz"; + sha256 = "1sgfw8z729nlxk8k13dhs0a762wnaxmlx70a7xlf3wz989bjh5w9"; authors = [ "The Servo Project Developers" ]; features = { "arbitrary" = [ "dep:arbitrary" ]; + "bincode" = [ "dep:bincode" ]; "const_new" = [ "const_generics" ]; "drain_keep_rest" = [ "drain_filter" ]; + "impl_bincode" = [ "bincode" "unty" ]; "malloc_size_of" = [ "dep:malloc_size_of" ]; "serde" = [ "dep:serde" ]; + "unty" = [ "dep:unty" ]; }; resolvedDefaultFeatures = [ "const_generics" "const_new" ]; }; @@ -8429,9 +8543,9 @@ rec { }; "socket2" = rec { crateName = "socket2"; - version = "0.5.8"; + version = "0.5.9"; edition = "2021"; - sha256 = "1s7vjmb5gzp3iaqi94rh9r63k9cj00kjgbfn7gn60kmnk6fjcw69"; + sha256 = "1vzds1wwwi0a51fn10r98j7cx3ir4shvhykpbk7md2h5h1ydapsg"; authors = [ "Alex Crichton " "Thomas de Zeeuw " @@ -8629,7 +8743,7 @@ rec { } { name = "indexmap"; - packageId = "indexmap 2.8.0"; + packageId = "indexmap 2.9.0"; } { name = "json-patch"; @@ -9301,9 +9415,9 @@ rec { }; "time" = rec { crateName = "time"; - version = "0.3.39"; + version = "0.3.41"; edition = "2021"; - sha256 = "1n6dmsj2xpk9jksdg4im5x0chz6vpqypxdl08nn3m8j03aq9ilns"; + sha256 = "0h0cpiyya8cjlrh00d2r72bmgg4lsdcncs76qpwy0rn2kghijxla"; authors = [ "Jacob Pratt " "Time contributors" @@ -9381,9 +9495,9 @@ rec { }; "time-core" = rec { crateName = "time-core"; - version = "0.1.3"; + version = "0.1.4"; edition = "2021"; - sha256 = "1vvn3vskn3dnvql1s0pvdlxazrjgvhksjzy2gcfw3dw5p6jrfp3n"; + sha256 = "0z5h9fknvdvbs2k2s1chpi3ab3jvgkfhdnqwrvixjngm263s7sf9"; libName = "time_core"; authors = [ "Jacob Pratt " @@ -9393,9 +9507,9 @@ rec { }; "time-macros" = rec { crateName = "time-macros"; - version = "0.2.20"; + version = "0.2.22"; edition = "2021"; - sha256 = "0p2w00wawnr2nzpdyi6a8mg5m6kcs0crdq4xhzvwafqwx31kn2g8"; + sha256 = "0jcaxpw220han2bzbrdlpqhy1s5k9i8ri3lw6n5zv4zcja9p69im"; procMacro = true; libName = "time_macros"; authors = [ @@ -9447,9 +9561,9 @@ rec { }; "tokio" = rec { crateName = "tokio"; - version = "1.44.1"; + version = "1.44.2"; edition = "2021"; - sha256 = "06n90q5hh1yd844s6nf4j3fwbrkm2bnq533kp3a488l4bdhxm0pk"; + sha256 = "0j4w3qvlcqzgbxlnap0czvspqj6x461vyk1sbqcf97g4rci8if76"; authors = [ "Tokio Contributors " ]; @@ -9740,7 +9854,7 @@ rec { dependencies = [ { name = "indexmap"; - packageId = "indexmap 2.8.0"; + packageId = "indexmap 2.9.0"; features = [ "std" ]; } { @@ -10932,7 +11046,7 @@ rec { ]; }; - "wasi" = rec { + "wasi 0.11.0+wasi-snapshot-preview1" = rec { crateName = "wasi"; version = "0.11.0+wasi-snapshot-preview1"; edition = "2018"; @@ -10949,6 +11063,29 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; + "wasi 0.14.2+wasi-0.2.4" = rec { + crateName = "wasi"; + version = "0.14.2+wasi-0.2.4"; + edition = "2021"; + sha256 = "1cwcqjr3dgdq8j325awgk8a715h0hg0f7jqzsb077n4qm6jzk0wn"; + authors = [ + "The Cranelift Project Developers" + ]; + dependencies = [ + { + name = "wit-bindgen-rt"; + packageId = "wit-bindgen-rt"; + features = [ "bitflags" ]; + } + ]; + features = { + "compiler_builtins" = [ "dep:compiler_builtins" ]; + "core" = [ "dep:core" ]; + "default" = [ "std" ]; + "rustc-dep-of-std" = [ "compiler_builtins" "core" "rustc-std-workspace-alloc" ]; + "rustc-std-workspace-alloc" = [ "dep:rustc-std-workspace-alloc" ]; + }; + }; "wasm-bindgen" = rec { crateName = "wasm-bindgen"; version = "0.2.100"; @@ -11712,711 +11849,113 @@ rec { ]; }; - "windows" = rec { - crateName = "windows"; - version = "0.52.0"; + "windows-core" = rec { + crateName = "windows-core"; + version = "0.61.0"; edition = "2021"; - sha256 = "1gnh210qjlprpd1szaq04rjm1zqgdm9j7l9absg0kawi2rwm72p4"; + sha256 = "104915nsby2cgp322pqqkmj2r82v5sg4hil0hxddg1hc67gc2qs7"; + libName = "windows_core"; authors = [ "Microsoft" ]; dependencies = [ { - name = "windows-core"; - packageId = "windows-core"; + name = "windows-implement"; + packageId = "windows-implement"; + usesDefaultFeatures = false; } { - name = "windows-targets"; - packageId = "windows-targets 0.52.6"; + name = "windows-interface"; + packageId = "windows-interface"; + usesDefaultFeatures = false; + } + { + name = "windows-link"; + packageId = "windows-link"; + usesDefaultFeatures = false; + } + { + name = "windows-result"; + packageId = "windows-result"; + usesDefaultFeatures = false; + } + { + name = "windows-strings"; + packageId = "windows-strings 0.4.0"; + usesDefaultFeatures = false; } ]; features = { - "AI_MachineLearning" = [ "AI" ]; - "ApplicationModel_Activation" = [ "ApplicationModel" ]; - "ApplicationModel_AppExtensions" = [ "ApplicationModel" ]; - "ApplicationModel_AppService" = [ "ApplicationModel" ]; - "ApplicationModel_Appointments" = [ "ApplicationModel" ]; - "ApplicationModel_Appointments_AppointmentsProvider" = [ "ApplicationModel_Appointments" ]; - "ApplicationModel_Appointments_DataProvider" = [ "ApplicationModel_Appointments" ]; - "ApplicationModel_Background" = [ "ApplicationModel" ]; - "ApplicationModel_Calls" = [ "ApplicationModel" ]; - "ApplicationModel_Calls_Background" = [ "ApplicationModel_Calls" ]; - "ApplicationModel_Calls_Provider" = [ "ApplicationModel_Calls" ]; - "ApplicationModel_Chat" = [ "ApplicationModel" ]; - "ApplicationModel_CommunicationBlocking" = [ "ApplicationModel" ]; - "ApplicationModel_Contacts" = [ "ApplicationModel" ]; - "ApplicationModel_Contacts_DataProvider" = [ "ApplicationModel_Contacts" ]; - "ApplicationModel_Contacts_Provider" = [ "ApplicationModel_Contacts" ]; - "ApplicationModel_ConversationalAgent" = [ "ApplicationModel" ]; - "ApplicationModel_Core" = [ "ApplicationModel" ]; - "ApplicationModel_DataTransfer" = [ "ApplicationModel" ]; - "ApplicationModel_DataTransfer_DragDrop" = [ "ApplicationModel_DataTransfer" ]; - "ApplicationModel_DataTransfer_DragDrop_Core" = [ "ApplicationModel_DataTransfer_DragDrop" ]; - "ApplicationModel_DataTransfer_ShareTarget" = [ "ApplicationModel_DataTransfer" ]; - "ApplicationModel_Email" = [ "ApplicationModel" ]; - "ApplicationModel_Email_DataProvider" = [ "ApplicationModel_Email" ]; - "ApplicationModel_ExtendedExecution" = [ "ApplicationModel" ]; - "ApplicationModel_ExtendedExecution_Foreground" = [ "ApplicationModel_ExtendedExecution" ]; - "ApplicationModel_Holographic" = [ "ApplicationModel" ]; - "ApplicationModel_LockScreen" = [ "ApplicationModel" ]; - "ApplicationModel_Payments" = [ "ApplicationModel" ]; - "ApplicationModel_Payments_Provider" = [ "ApplicationModel_Payments" ]; - "ApplicationModel_Preview" = [ "ApplicationModel" ]; - "ApplicationModel_Preview_Holographic" = [ "ApplicationModel_Preview" ]; - "ApplicationModel_Preview_InkWorkspace" = [ "ApplicationModel_Preview" ]; - "ApplicationModel_Preview_Notes" = [ "ApplicationModel_Preview" ]; - "ApplicationModel_Resources" = [ "ApplicationModel" ]; - "ApplicationModel_Resources_Core" = [ "ApplicationModel_Resources" ]; - "ApplicationModel_Resources_Management" = [ "ApplicationModel_Resources" ]; - "ApplicationModel_Search" = [ "ApplicationModel" ]; - "ApplicationModel_Search_Core" = [ "ApplicationModel_Search" ]; - "ApplicationModel_Store" = [ "ApplicationModel" ]; - "ApplicationModel_Store_LicenseManagement" = [ "ApplicationModel_Store" ]; - "ApplicationModel_Store_Preview" = [ "ApplicationModel_Store" ]; - "ApplicationModel_Store_Preview_InstallControl" = [ "ApplicationModel_Store_Preview" ]; - "ApplicationModel_UserActivities" = [ "ApplicationModel" ]; - "ApplicationModel_UserActivities_Core" = [ "ApplicationModel_UserActivities" ]; - "ApplicationModel_UserDataAccounts" = [ "ApplicationModel" ]; - "ApplicationModel_UserDataAccounts_Provider" = [ "ApplicationModel_UserDataAccounts" ]; - "ApplicationModel_UserDataAccounts_SystemAccess" = [ "ApplicationModel_UserDataAccounts" ]; - "ApplicationModel_UserDataTasks" = [ "ApplicationModel" ]; - "ApplicationModel_UserDataTasks_DataProvider" = [ "ApplicationModel_UserDataTasks" ]; - "ApplicationModel_VoiceCommands" = [ "ApplicationModel" ]; - "ApplicationModel_Wallet" = [ "ApplicationModel" ]; - "ApplicationModel_Wallet_System" = [ "ApplicationModel_Wallet" ]; - "Data_Html" = [ "Data" ]; - "Data_Json" = [ "Data" ]; - "Data_Pdf" = [ "Data" ]; - "Data_Text" = [ "Data" ]; - "Data_Xml" = [ "Data" ]; - "Data_Xml_Dom" = [ "Data_Xml" ]; - "Data_Xml_Xsl" = [ "Data_Xml" ]; - "Devices_Adc" = [ "Devices" ]; - "Devices_Adc_Provider" = [ "Devices_Adc" ]; - "Devices_Background" = [ "Devices" ]; - "Devices_Bluetooth" = [ "Devices" ]; - "Devices_Bluetooth_Advertisement" = [ "Devices_Bluetooth" ]; - "Devices_Bluetooth_Background" = [ "Devices_Bluetooth" ]; - "Devices_Bluetooth_GenericAttributeProfile" = [ "Devices_Bluetooth" ]; - "Devices_Bluetooth_Rfcomm" = [ "Devices_Bluetooth" ]; - "Devices_Custom" = [ "Devices" ]; - "Devices_Display" = [ "Devices" ]; - "Devices_Display_Core" = [ "Devices_Display" ]; - "Devices_Enumeration" = [ "Devices" ]; - "Devices_Enumeration_Pnp" = [ "Devices_Enumeration" ]; - "Devices_Geolocation" = [ "Devices" ]; - "Devices_Geolocation_Geofencing" = [ "Devices_Geolocation" ]; - "Devices_Geolocation_Provider" = [ "Devices_Geolocation" ]; - "Devices_Gpio" = [ "Devices" ]; - "Devices_Gpio_Provider" = [ "Devices_Gpio" ]; - "Devices_Haptics" = [ "Devices" ]; - "Devices_HumanInterfaceDevice" = [ "Devices" ]; - "Devices_I2c" = [ "Devices" ]; - "Devices_I2c_Provider" = [ "Devices_I2c" ]; - "Devices_Input" = [ "Devices" ]; - "Devices_Input_Preview" = [ "Devices_Input" ]; - "Devices_Lights" = [ "Devices" ]; - "Devices_Lights_Effects" = [ "Devices_Lights" ]; - "Devices_Midi" = [ "Devices" ]; - "Devices_PointOfService" = [ "Devices" ]; - "Devices_PointOfService_Provider" = [ "Devices_PointOfService" ]; - "Devices_Portable" = [ "Devices" ]; - "Devices_Power" = [ "Devices" ]; - "Devices_Printers" = [ "Devices" ]; - "Devices_Printers_Extensions" = [ "Devices_Printers" ]; - "Devices_Pwm" = [ "Devices" ]; - "Devices_Pwm_Provider" = [ "Devices_Pwm" ]; - "Devices_Radios" = [ "Devices" ]; - "Devices_Scanners" = [ "Devices" ]; - "Devices_Sensors" = [ "Devices" ]; - "Devices_Sensors_Custom" = [ "Devices_Sensors" ]; - "Devices_SerialCommunication" = [ "Devices" ]; - "Devices_SmartCards" = [ "Devices" ]; - "Devices_Sms" = [ "Devices" ]; - "Devices_Spi" = [ "Devices" ]; - "Devices_Spi_Provider" = [ "Devices_Spi" ]; - "Devices_Usb" = [ "Devices" ]; - "Devices_WiFi" = [ "Devices" ]; - "Devices_WiFiDirect" = [ "Devices" ]; - "Devices_WiFiDirect_Services" = [ "Devices_WiFiDirect" ]; - "Embedded_DeviceLockdown" = [ "Embedded" ]; - "Foundation_Collections" = [ "Foundation" ]; - "Foundation_Diagnostics" = [ "Foundation" ]; - "Foundation_Metadata" = [ "Foundation" ]; - "Foundation_Numerics" = [ "Foundation" ]; - "Gaming_Input" = [ "Gaming" ]; - "Gaming_Input_Custom" = [ "Gaming_Input" ]; - "Gaming_Input_ForceFeedback" = [ "Gaming_Input" ]; - "Gaming_Input_Preview" = [ "Gaming_Input" ]; - "Gaming_Preview" = [ "Gaming" ]; - "Gaming_Preview_GamesEnumeration" = [ "Gaming_Preview" ]; - "Gaming_UI" = [ "Gaming" ]; - "Gaming_XboxLive" = [ "Gaming" ]; - "Gaming_XboxLive_Storage" = [ "Gaming_XboxLive" ]; - "Globalization_Collation" = [ "Globalization" ]; - "Globalization_DateTimeFormatting" = [ "Globalization" ]; - "Globalization_Fonts" = [ "Globalization" ]; - "Globalization_NumberFormatting" = [ "Globalization" ]; - "Globalization_PhoneNumberFormatting" = [ "Globalization" ]; - "Graphics_Capture" = [ "Graphics" ]; - "Graphics_DirectX" = [ "Graphics" ]; - "Graphics_DirectX_Direct3D11" = [ "Graphics_DirectX" ]; - "Graphics_Display" = [ "Graphics" ]; - "Graphics_Display_Core" = [ "Graphics_Display" ]; - "Graphics_Effects" = [ "Graphics" ]; - "Graphics_Holographic" = [ "Graphics" ]; - "Graphics_Imaging" = [ "Graphics" ]; - "Graphics_Printing" = [ "Graphics" ]; - "Graphics_Printing3D" = [ "Graphics" ]; - "Graphics_Printing_OptionDetails" = [ "Graphics_Printing" ]; - "Graphics_Printing_PrintSupport" = [ "Graphics_Printing" ]; - "Graphics_Printing_PrintTicket" = [ "Graphics_Printing" ]; - "Graphics_Printing_Workflow" = [ "Graphics_Printing" ]; - "Management_Core" = [ "Management" ]; - "Management_Deployment" = [ "Management" ]; - "Management_Deployment_Preview" = [ "Management_Deployment" ]; - "Management_Policies" = [ "Management" ]; - "Management_Update" = [ "Management" ]; - "Management_Workplace" = [ "Management" ]; - "Media_AppBroadcasting" = [ "Media" ]; - "Media_AppRecording" = [ "Media" ]; - "Media_Audio" = [ "Media" ]; - "Media_Capture" = [ "Media" ]; - "Media_Capture_Core" = [ "Media_Capture" ]; - "Media_Capture_Frames" = [ "Media_Capture" ]; - "Media_Casting" = [ "Media" ]; - "Media_ClosedCaptioning" = [ "Media" ]; - "Media_ContentRestrictions" = [ "Media" ]; - "Media_Control" = [ "Media" ]; - "Media_Core" = [ "Media" ]; - "Media_Core_Preview" = [ "Media_Core" ]; - "Media_Devices" = [ "Media" ]; - "Media_Devices_Core" = [ "Media_Devices" ]; - "Media_DialProtocol" = [ "Media" ]; - "Media_Editing" = [ "Media" ]; - "Media_Effects" = [ "Media" ]; - "Media_FaceAnalysis" = [ "Media" ]; - "Media_Import" = [ "Media" ]; - "Media_MediaProperties" = [ "Media" ]; - "Media_Miracast" = [ "Media" ]; - "Media_Ocr" = [ "Media" ]; - "Media_PlayTo" = [ "Media" ]; - "Media_Playback" = [ "Media" ]; - "Media_Playlists" = [ "Media" ]; - "Media_Protection" = [ "Media" ]; - "Media_Protection_PlayReady" = [ "Media_Protection" ]; - "Media_Render" = [ "Media" ]; - "Media_SpeechRecognition" = [ "Media" ]; - "Media_SpeechSynthesis" = [ "Media" ]; - "Media_Streaming" = [ "Media" ]; - "Media_Streaming_Adaptive" = [ "Media_Streaming" ]; - "Media_Transcoding" = [ "Media" ]; - "Networking_BackgroundTransfer" = [ "Networking" ]; - "Networking_Connectivity" = [ "Networking" ]; - "Networking_NetworkOperators" = [ "Networking" ]; - "Networking_Proximity" = [ "Networking" ]; - "Networking_PushNotifications" = [ "Networking" ]; - "Networking_ServiceDiscovery" = [ "Networking" ]; - "Networking_ServiceDiscovery_Dnssd" = [ "Networking_ServiceDiscovery" ]; - "Networking_Sockets" = [ "Networking" ]; - "Networking_Vpn" = [ "Networking" ]; - "Networking_XboxLive" = [ "Networking" ]; - "Perception_Automation" = [ "Perception" ]; - "Perception_Automation_Core" = [ "Perception_Automation" ]; - "Perception_People" = [ "Perception" ]; - "Perception_Spatial" = [ "Perception" ]; - "Perception_Spatial_Preview" = [ "Perception_Spatial" ]; - "Perception_Spatial_Surfaces" = [ "Perception_Spatial" ]; - "Phone_ApplicationModel" = [ "Phone" ]; - "Phone_Devices" = [ "Phone" ]; - "Phone_Devices_Notification" = [ "Phone_Devices" ]; - "Phone_Devices_Power" = [ "Phone_Devices" ]; - "Phone_Management" = [ "Phone" ]; - "Phone_Management_Deployment" = [ "Phone_Management" ]; - "Phone_Media" = [ "Phone" ]; - "Phone_Media_Devices" = [ "Phone_Media" ]; - "Phone_Notification" = [ "Phone" ]; - "Phone_Notification_Management" = [ "Phone_Notification" ]; - "Phone_PersonalInformation" = [ "Phone" ]; - "Phone_PersonalInformation_Provisioning" = [ "Phone_PersonalInformation" ]; - "Phone_Speech" = [ "Phone" ]; - "Phone_Speech_Recognition" = [ "Phone_Speech" ]; - "Phone_StartScreen" = [ "Phone" ]; - "Phone_System" = [ "Phone" ]; - "Phone_System_Power" = [ "Phone_System" ]; - "Phone_System_Profile" = [ "Phone_System" ]; - "Phone_System_UserProfile" = [ "Phone_System" ]; - "Phone_System_UserProfile_GameServices" = [ "Phone_System_UserProfile" ]; - "Phone_System_UserProfile_GameServices_Core" = [ "Phone_System_UserProfile_GameServices" ]; - "Phone_UI" = [ "Phone" ]; - "Phone_UI_Input" = [ "Phone_UI" ]; - "Security_Authentication" = [ "Security" ]; - "Security_Authentication_Identity" = [ "Security_Authentication" ]; - "Security_Authentication_Identity_Core" = [ "Security_Authentication_Identity" ]; - "Security_Authentication_OnlineId" = [ "Security_Authentication" ]; - "Security_Authentication_Web" = [ "Security_Authentication" ]; - "Security_Authentication_Web_Core" = [ "Security_Authentication_Web" ]; - "Security_Authentication_Web_Provider" = [ "Security_Authentication_Web" ]; - "Security_Authorization" = [ "Security" ]; - "Security_Authorization_AppCapabilityAccess" = [ "Security_Authorization" ]; - "Security_Credentials" = [ "Security" ]; - "Security_Credentials_UI" = [ "Security_Credentials" ]; - "Security_Cryptography" = [ "Security" ]; - "Security_Cryptography_Certificates" = [ "Security_Cryptography" ]; - "Security_Cryptography_Core" = [ "Security_Cryptography" ]; - "Security_Cryptography_DataProtection" = [ "Security_Cryptography" ]; - "Security_DataProtection" = [ "Security" ]; - "Security_EnterpriseData" = [ "Security" ]; - "Security_ExchangeActiveSyncProvisioning" = [ "Security" ]; - "Security_Isolation" = [ "Security" ]; - "Services_Maps" = [ "Services" ]; - "Services_Maps_Guidance" = [ "Services_Maps" ]; - "Services_Maps_LocalSearch" = [ "Services_Maps" ]; - "Services_Maps_OfflineMaps" = [ "Services_Maps" ]; - "Services_Store" = [ "Services" ]; - "Services_TargetedContent" = [ "Services" ]; - "Storage_AccessCache" = [ "Storage" ]; - "Storage_BulkAccess" = [ "Storage" ]; - "Storage_Compression" = [ "Storage" ]; - "Storage_FileProperties" = [ "Storage" ]; - "Storage_Pickers" = [ "Storage" ]; - "Storage_Pickers_Provider" = [ "Storage_Pickers" ]; - "Storage_Provider" = [ "Storage" ]; - "Storage_Search" = [ "Storage" ]; - "Storage_Streams" = [ "Storage" ]; - "System_Diagnostics" = [ "System" ]; - "System_Diagnostics_DevicePortal" = [ "System_Diagnostics" ]; - "System_Diagnostics_Telemetry" = [ "System_Diagnostics" ]; - "System_Diagnostics_TraceReporting" = [ "System_Diagnostics" ]; - "System_Display" = [ "System" ]; - "System_Implementation" = [ "System" ]; - "System_Implementation_FileExplorer" = [ "System_Implementation" ]; - "System_Inventory" = [ "System" ]; - "System_Power" = [ "System" ]; - "System_Profile" = [ "System" ]; - "System_Profile_SystemManufacturers" = [ "System_Profile" ]; - "System_RemoteDesktop" = [ "System" ]; - "System_RemoteDesktop_Input" = [ "System_RemoteDesktop" ]; - "System_RemoteSystems" = [ "System" ]; - "System_Threading" = [ "System" ]; - "System_Threading_Core" = [ "System_Threading" ]; - "System_Update" = [ "System" ]; - "System_UserProfile" = [ "System" ]; - "UI_Accessibility" = [ "UI" ]; - "UI_ApplicationSettings" = [ "UI" ]; - "UI_Composition" = [ "UI" ]; - "UI_Composition_Core" = [ "UI_Composition" ]; - "UI_Composition_Desktop" = [ "UI_Composition" ]; - "UI_Composition_Diagnostics" = [ "UI_Composition" ]; - "UI_Composition_Effects" = [ "UI_Composition" ]; - "UI_Composition_Interactions" = [ "UI_Composition" ]; - "UI_Composition_Scenes" = [ "UI_Composition" ]; - "UI_Core" = [ "UI" ]; - "UI_Core_AnimationMetrics" = [ "UI_Core" ]; - "UI_Core_Preview" = [ "UI_Core" ]; - "UI_Input" = [ "UI" ]; - "UI_Input_Core" = [ "UI_Input" ]; - "UI_Input_Inking" = [ "UI_Input" ]; - "UI_Input_Inking_Analysis" = [ "UI_Input_Inking" ]; - "UI_Input_Inking_Core" = [ "UI_Input_Inking" ]; - "UI_Input_Inking_Preview" = [ "UI_Input_Inking" ]; - "UI_Input_Preview" = [ "UI_Input" ]; - "UI_Input_Preview_Injection" = [ "UI_Input_Preview" ]; - "UI_Input_Spatial" = [ "UI_Input" ]; - "UI_Notifications" = [ "UI" ]; - "UI_Notifications_Management" = [ "UI_Notifications" ]; - "UI_Popups" = [ "UI" ]; - "UI_Shell" = [ "UI" ]; - "UI_StartScreen" = [ "UI" ]; - "UI_Text" = [ "UI" ]; - "UI_Text_Core" = [ "UI_Text" ]; - "UI_UIAutomation" = [ "UI" ]; - "UI_UIAutomation_Core" = [ "UI_UIAutomation" ]; - "UI_ViewManagement" = [ "UI" ]; - "UI_ViewManagement_Core" = [ "UI_ViewManagement" ]; - "UI_WebUI" = [ "UI" ]; - "UI_WebUI_Core" = [ "UI_WebUI" ]; - "UI_WindowManagement" = [ "UI" ]; - "UI_WindowManagement_Preview" = [ "UI_WindowManagement" ]; - "Wdk_Foundation" = [ "Wdk" ]; - "Wdk_Graphics" = [ "Wdk" ]; - "Wdk_Graphics_Direct3D" = [ "Wdk_Graphics" ]; - "Wdk_Storage" = [ "Wdk" ]; - "Wdk_Storage_FileSystem" = [ "Wdk_Storage" ]; - "Wdk_Storage_FileSystem_Minifilters" = [ "Wdk_Storage_FileSystem" ]; - "Wdk_System" = [ "Wdk" ]; - "Wdk_System_IO" = [ "Wdk_System" ]; - "Wdk_System_OfflineRegistry" = [ "Wdk_System" ]; - "Wdk_System_Registry" = [ "Wdk_System" ]; - "Wdk_System_SystemInformation" = [ "Wdk_System" ]; - "Wdk_System_SystemServices" = [ "Wdk_System" ]; - "Wdk_System_Threading" = [ "Wdk_System" ]; - "Web_AtomPub" = [ "Web" ]; - "Web_Http" = [ "Web" ]; - "Web_Http_Diagnostics" = [ "Web_Http" ]; - "Web_Http_Filters" = [ "Web_Http" ]; - "Web_Http_Headers" = [ "Web_Http" ]; - "Web_Syndication" = [ "Web" ]; - "Web_UI" = [ "Web" ]; - "Web_UI_Interop" = [ "Web_UI" ]; - "Win32_AI" = [ "Win32" ]; - "Win32_AI_MachineLearning" = [ "Win32_AI" ]; - "Win32_AI_MachineLearning_DirectML" = [ "Win32_AI_MachineLearning" ]; - "Win32_AI_MachineLearning_WinML" = [ "Win32_AI_MachineLearning" ]; - "Win32_Data" = [ "Win32" ]; - "Win32_Data_HtmlHelp" = [ "Win32_Data" ]; - "Win32_Data_RightsManagement" = [ "Win32_Data" ]; - "Win32_Data_Xml" = [ "Win32_Data" ]; - "Win32_Data_Xml_MsXml" = [ "Win32_Data_Xml" ]; - "Win32_Data_Xml_XmlLite" = [ "Win32_Data_Xml" ]; - "Win32_Devices" = [ "Win32" ]; - "Win32_Devices_AllJoyn" = [ "Win32_Devices" ]; - "Win32_Devices_BiometricFramework" = [ "Win32_Devices" ]; - "Win32_Devices_Bluetooth" = [ "Win32_Devices" ]; - "Win32_Devices_Communication" = [ "Win32_Devices" ]; - "Win32_Devices_DeviceAccess" = [ "Win32_Devices" ]; - "Win32_Devices_DeviceAndDriverInstallation" = [ "Win32_Devices" ]; - "Win32_Devices_DeviceQuery" = [ "Win32_Devices" ]; - "Win32_Devices_Display" = [ "Win32_Devices" ]; - "Win32_Devices_Enumeration" = [ "Win32_Devices" ]; - "Win32_Devices_Enumeration_Pnp" = [ "Win32_Devices_Enumeration" ]; - "Win32_Devices_Fax" = [ "Win32_Devices" ]; - "Win32_Devices_FunctionDiscovery" = [ "Win32_Devices" ]; - "Win32_Devices_Geolocation" = [ "Win32_Devices" ]; - "Win32_Devices_HumanInterfaceDevice" = [ "Win32_Devices" ]; - "Win32_Devices_ImageAcquisition" = [ "Win32_Devices" ]; - "Win32_Devices_PortableDevices" = [ "Win32_Devices" ]; - "Win32_Devices_Properties" = [ "Win32_Devices" ]; - "Win32_Devices_Pwm" = [ "Win32_Devices" ]; - "Win32_Devices_Sensors" = [ "Win32_Devices" ]; - "Win32_Devices_SerialCommunication" = [ "Win32_Devices" ]; - "Win32_Devices_Tapi" = [ "Win32_Devices" ]; - "Win32_Devices_Usb" = [ "Win32_Devices" ]; - "Win32_Devices_WebServicesOnDevices" = [ "Win32_Devices" ]; - "Win32_Foundation" = [ "Win32" ]; - "Win32_Gaming" = [ "Win32" ]; - "Win32_Globalization" = [ "Win32" ]; - "Win32_Graphics" = [ "Win32" ]; - "Win32_Graphics_CompositionSwapchain" = [ "Win32_Graphics" ]; - "Win32_Graphics_DXCore" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct2D" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct2D_Common" = [ "Win32_Graphics_Direct2D" ]; - "Win32_Graphics_Direct3D" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct3D10" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct3D11" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct3D11on12" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct3D12" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct3D9" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct3D9on12" = [ "Win32_Graphics" ]; - "Win32_Graphics_Direct3D_Dxc" = [ "Win32_Graphics_Direct3D" ]; - "Win32_Graphics_Direct3D_Fxc" = [ "Win32_Graphics_Direct3D" ]; - "Win32_Graphics_DirectComposition" = [ "Win32_Graphics" ]; - "Win32_Graphics_DirectDraw" = [ "Win32_Graphics" ]; - "Win32_Graphics_DirectManipulation" = [ "Win32_Graphics" ]; - "Win32_Graphics_DirectWrite" = [ "Win32_Graphics" ]; - "Win32_Graphics_Dwm" = [ "Win32_Graphics" ]; - "Win32_Graphics_Dxgi" = [ "Win32_Graphics" ]; - "Win32_Graphics_Dxgi_Common" = [ "Win32_Graphics_Dxgi" ]; - "Win32_Graphics_Gdi" = [ "Win32_Graphics" ]; - "Win32_Graphics_GdiPlus" = [ "Win32_Graphics" ]; - "Win32_Graphics_Hlsl" = [ "Win32_Graphics" ]; - "Win32_Graphics_Imaging" = [ "Win32_Graphics" ]; - "Win32_Graphics_Imaging_D2D" = [ "Win32_Graphics_Imaging" ]; - "Win32_Graphics_OpenGL" = [ "Win32_Graphics" ]; - "Win32_Graphics_Printing" = [ "Win32_Graphics" ]; - "Win32_Graphics_Printing_PrintTicket" = [ "Win32_Graphics_Printing" ]; - "Win32_Management" = [ "Win32" ]; - "Win32_Management_MobileDeviceManagementRegistration" = [ "Win32_Management" ]; - "Win32_Media" = [ "Win32" ]; - "Win32_Media_Audio" = [ "Win32_Media" ]; - "Win32_Media_Audio_Apo" = [ "Win32_Media_Audio" ]; - "Win32_Media_Audio_DirectMusic" = [ "Win32_Media_Audio" ]; - "Win32_Media_Audio_DirectSound" = [ "Win32_Media_Audio" ]; - "Win32_Media_Audio_Endpoints" = [ "Win32_Media_Audio" ]; - "Win32_Media_Audio_XAudio2" = [ "Win32_Media_Audio" ]; - "Win32_Media_DeviceManager" = [ "Win32_Media" ]; - "Win32_Media_DirectShow" = [ "Win32_Media" ]; - "Win32_Media_DirectShow_Tv" = [ "Win32_Media_DirectShow" ]; - "Win32_Media_DirectShow_Xml" = [ "Win32_Media_DirectShow" ]; - "Win32_Media_DxMediaObjects" = [ "Win32_Media" ]; - "Win32_Media_KernelStreaming" = [ "Win32_Media" ]; - "Win32_Media_LibrarySharingServices" = [ "Win32_Media" ]; - "Win32_Media_MediaFoundation" = [ "Win32_Media" ]; - "Win32_Media_MediaPlayer" = [ "Win32_Media" ]; - "Win32_Media_Multimedia" = [ "Win32_Media" ]; - "Win32_Media_PictureAcquisition" = [ "Win32_Media" ]; - "Win32_Media_Speech" = [ "Win32_Media" ]; - "Win32_Media_Streaming" = [ "Win32_Media" ]; - "Win32_Media_WindowsMediaFormat" = [ "Win32_Media" ]; - "Win32_NetworkManagement" = [ "Win32" ]; - "Win32_NetworkManagement_Dhcp" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Dns" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_InternetConnectionWizard" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_IpHelper" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_MobileBroadband" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Multicast" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Ndis" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetBios" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetManagement" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetShell" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetworkDiagnosticsFramework" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_NetworkPolicyServer" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_P2P" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_QoS" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Rras" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_Snmp" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WNet" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WebDav" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WiFi" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsConnectNow" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsConnectionManager" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsFilteringPlatform" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsFirewall" = [ "Win32_NetworkManagement" ]; - "Win32_NetworkManagement_WindowsNetworkVirtualization" = [ "Win32_NetworkManagement" ]; - "Win32_Networking" = [ "Win32" ]; - "Win32_Networking_ActiveDirectory" = [ "Win32_Networking" ]; - "Win32_Networking_BackgroundIntelligentTransferService" = [ "Win32_Networking" ]; - "Win32_Networking_Clustering" = [ "Win32_Networking" ]; - "Win32_Networking_HttpServer" = [ "Win32_Networking" ]; - "Win32_Networking_Ldap" = [ "Win32_Networking" ]; - "Win32_Networking_NetworkListManager" = [ "Win32_Networking" ]; - "Win32_Networking_RemoteDifferentialCompression" = [ "Win32_Networking" ]; - "Win32_Networking_WebSocket" = [ "Win32_Networking" ]; - "Win32_Networking_WinHttp" = [ "Win32_Networking" ]; - "Win32_Networking_WinInet" = [ "Win32_Networking" ]; - "Win32_Networking_WinSock" = [ "Win32_Networking" ]; - "Win32_Networking_WindowsWebServices" = [ "Win32_Networking" ]; - "Win32_Security" = [ "Win32" ]; - "Win32_Security_AppLocker" = [ "Win32_Security" ]; - "Win32_Security_Authentication" = [ "Win32_Security" ]; - "Win32_Security_Authentication_Identity" = [ "Win32_Security_Authentication" ]; - "Win32_Security_Authentication_Identity_Provider" = [ "Win32_Security_Authentication_Identity" ]; - "Win32_Security_Authorization" = [ "Win32_Security" ]; - "Win32_Security_Authorization_UI" = [ "Win32_Security_Authorization" ]; - "Win32_Security_ConfigurationSnapin" = [ "Win32_Security" ]; - "Win32_Security_Credentials" = [ "Win32_Security" ]; - "Win32_Security_Cryptography" = [ "Win32_Security" ]; - "Win32_Security_Cryptography_Catalog" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_Cryptography_Certificates" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_Cryptography_Sip" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_Cryptography_UI" = [ "Win32_Security_Cryptography" ]; - "Win32_Security_DiagnosticDataQuery" = [ "Win32_Security" ]; - "Win32_Security_DirectoryServices" = [ "Win32_Security" ]; - "Win32_Security_EnterpriseData" = [ "Win32_Security" ]; - "Win32_Security_ExtensibleAuthenticationProtocol" = [ "Win32_Security" ]; - "Win32_Security_Isolation" = [ "Win32_Security" ]; - "Win32_Security_LicenseProtection" = [ "Win32_Security" ]; - "Win32_Security_NetworkAccessProtection" = [ "Win32_Security" ]; - "Win32_Security_Tpm" = [ "Win32_Security" ]; - "Win32_Security_WinTrust" = [ "Win32_Security" ]; - "Win32_Security_WinWlx" = [ "Win32_Security" ]; - "Win32_Storage" = [ "Win32" ]; - "Win32_Storage_Cabinets" = [ "Win32_Storage" ]; - "Win32_Storage_CloudFilters" = [ "Win32_Storage" ]; - "Win32_Storage_Compression" = [ "Win32_Storage" ]; - "Win32_Storage_DataDeduplication" = [ "Win32_Storage" ]; - "Win32_Storage_DistributedFileSystem" = [ "Win32_Storage" ]; - "Win32_Storage_EnhancedStorage" = [ "Win32_Storage" ]; - "Win32_Storage_FileHistory" = [ "Win32_Storage" ]; - "Win32_Storage_FileServerResourceManager" = [ "Win32_Storage" ]; - "Win32_Storage_FileSystem" = [ "Win32_Storage" ]; - "Win32_Storage_Imapi" = [ "Win32_Storage" ]; - "Win32_Storage_IndexServer" = [ "Win32_Storage" ]; - "Win32_Storage_InstallableFileSystems" = [ "Win32_Storage" ]; - "Win32_Storage_IscsiDisc" = [ "Win32_Storage" ]; - "Win32_Storage_Jet" = [ "Win32_Storage" ]; - "Win32_Storage_Nvme" = [ "Win32_Storage" ]; - "Win32_Storage_OfflineFiles" = [ "Win32_Storage" ]; - "Win32_Storage_OperationRecorder" = [ "Win32_Storage" ]; - "Win32_Storage_Packaging" = [ "Win32_Storage" ]; - "Win32_Storage_Packaging_Appx" = [ "Win32_Storage_Packaging" ]; - "Win32_Storage_Packaging_Opc" = [ "Win32_Storage_Packaging" ]; - "Win32_Storage_ProjectedFileSystem" = [ "Win32_Storage" ]; - "Win32_Storage_StructuredStorage" = [ "Win32_Storage" ]; - "Win32_Storage_Vhd" = [ "Win32_Storage" ]; - "Win32_Storage_VirtualDiskService" = [ "Win32_Storage" ]; - "Win32_Storage_Vss" = [ "Win32_Storage" ]; - "Win32_Storage_Xps" = [ "Win32_Storage" ]; - "Win32_Storage_Xps_Printing" = [ "Win32_Storage_Xps" ]; - "Win32_System" = [ "Win32" ]; - "Win32_System_AddressBook" = [ "Win32_System" ]; - "Win32_System_Antimalware" = [ "Win32_System" ]; - "Win32_System_ApplicationInstallationAndServicing" = [ "Win32_System" ]; - "Win32_System_ApplicationVerifier" = [ "Win32_System" ]; - "Win32_System_AssessmentTool" = [ "Win32_System" ]; - "Win32_System_ClrHosting" = [ "Win32_System" ]; - "Win32_System_Com" = [ "Win32_System" ]; - "Win32_System_Com_CallObj" = [ "Win32_System_Com" ]; - "Win32_System_Com_ChannelCredentials" = [ "Win32_System_Com" ]; - "Win32_System_Com_Events" = [ "Win32_System_Com" ]; - "Win32_System_Com_Marshal" = [ "Win32_System_Com" ]; - "Win32_System_Com_StructuredStorage" = [ "Win32_System_Com" ]; - "Win32_System_Com_UI" = [ "Win32_System_Com" ]; - "Win32_System_Com_Urlmon" = [ "Win32_System_Com" ]; - "Win32_System_ComponentServices" = [ "Win32_System" ]; - "Win32_System_Console" = [ "Win32_System" ]; - "Win32_System_Contacts" = [ "Win32_System" ]; - "Win32_System_CorrelationVector" = [ "Win32_System" ]; - "Win32_System_DataExchange" = [ "Win32_System" ]; - "Win32_System_DeploymentServices" = [ "Win32_System" ]; - "Win32_System_DesktopSharing" = [ "Win32_System" ]; - "Win32_System_DeveloperLicensing" = [ "Win32_System" ]; - "Win32_System_Diagnostics" = [ "Win32_System" ]; - "Win32_System_Diagnostics_Ceip" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_ClrProfiling" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_Debug" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_Debug_ActiveScript" = [ "Win32_System_Diagnostics_Debug" ]; - "Win32_System_Diagnostics_Debug_Extensions" = [ "Win32_System_Diagnostics_Debug" ]; - "Win32_System_Diagnostics_Etw" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_ProcessSnapshotting" = [ "Win32_System_Diagnostics" ]; - "Win32_System_Diagnostics_ToolHelp" = [ "Win32_System_Diagnostics" ]; - "Win32_System_DistributedTransactionCoordinator" = [ "Win32_System" ]; - "Win32_System_Environment" = [ "Win32_System" ]; - "Win32_System_ErrorReporting" = [ "Win32_System" ]; - "Win32_System_EventCollector" = [ "Win32_System" ]; - "Win32_System_EventLog" = [ "Win32_System" ]; - "Win32_System_EventNotificationService" = [ "Win32_System" ]; - "Win32_System_GroupPolicy" = [ "Win32_System" ]; - "Win32_System_HostCompute" = [ "Win32_System" ]; - "Win32_System_HostComputeNetwork" = [ "Win32_System" ]; - "Win32_System_HostComputeSystem" = [ "Win32_System" ]; - "Win32_System_Hypervisor" = [ "Win32_System" ]; - "Win32_System_IO" = [ "Win32_System" ]; - "Win32_System_Iis" = [ "Win32_System" ]; - "Win32_System_Ioctl" = [ "Win32_System" ]; - "Win32_System_JobObjects" = [ "Win32_System" ]; - "Win32_System_Js" = [ "Win32_System" ]; - "Win32_System_Kernel" = [ "Win32_System" ]; - "Win32_System_LibraryLoader" = [ "Win32_System" ]; - "Win32_System_Mailslots" = [ "Win32_System" ]; - "Win32_System_Mapi" = [ "Win32_System" ]; - "Win32_System_Memory" = [ "Win32_System" ]; - "Win32_System_Memory_NonVolatile" = [ "Win32_System_Memory" ]; - "Win32_System_MessageQueuing" = [ "Win32_System" ]; - "Win32_System_MixedReality" = [ "Win32_System" ]; - "Win32_System_Mmc" = [ "Win32_System" ]; - "Win32_System_Ole" = [ "Win32_System" ]; - "Win32_System_ParentalControls" = [ "Win32_System" ]; - "Win32_System_PasswordManagement" = [ "Win32_System" ]; - "Win32_System_Performance" = [ "Win32_System" ]; - "Win32_System_Performance_HardwareCounterProfiling" = [ "Win32_System_Performance" ]; - "Win32_System_Pipes" = [ "Win32_System" ]; - "Win32_System_Power" = [ "Win32_System" ]; - "Win32_System_ProcessStatus" = [ "Win32_System" ]; - "Win32_System_RealTimeCommunications" = [ "Win32_System" ]; - "Win32_System_Recovery" = [ "Win32_System" ]; - "Win32_System_Registry" = [ "Win32_System" ]; - "Win32_System_RemoteAssistance" = [ "Win32_System" ]; - "Win32_System_RemoteDesktop" = [ "Win32_System" ]; - "Win32_System_RemoteManagement" = [ "Win32_System" ]; - "Win32_System_RestartManager" = [ "Win32_System" ]; - "Win32_System_Restore" = [ "Win32_System" ]; - "Win32_System_Rpc" = [ "Win32_System" ]; - "Win32_System_Search" = [ "Win32_System" ]; - "Win32_System_Search_Common" = [ "Win32_System_Search" ]; - "Win32_System_SecurityCenter" = [ "Win32_System" ]; - "Win32_System_ServerBackup" = [ "Win32_System" ]; - "Win32_System_Services" = [ "Win32_System" ]; - "Win32_System_SettingsManagementInfrastructure" = [ "Win32_System" ]; - "Win32_System_SetupAndMigration" = [ "Win32_System" ]; - "Win32_System_Shutdown" = [ "Win32_System" ]; - "Win32_System_SideShow" = [ "Win32_System" ]; - "Win32_System_StationsAndDesktops" = [ "Win32_System" ]; - "Win32_System_SubsystemForLinux" = [ "Win32_System" ]; - "Win32_System_SystemInformation" = [ "Win32_System" ]; - "Win32_System_SystemServices" = [ "Win32_System" ]; - "Win32_System_TaskScheduler" = [ "Win32_System" ]; - "Win32_System_Threading" = [ "Win32_System" ]; - "Win32_System_Time" = [ "Win32_System" ]; - "Win32_System_TpmBaseServices" = [ "Win32_System" ]; - "Win32_System_TransactionServer" = [ "Win32_System" ]; - "Win32_System_UpdateAgent" = [ "Win32_System" ]; - "Win32_System_UpdateAssessment" = [ "Win32_System" ]; - "Win32_System_UserAccessLogging" = [ "Win32_System" ]; - "Win32_System_Variant" = [ "Win32_System" ]; - "Win32_System_VirtualDosMachines" = [ "Win32_System" ]; - "Win32_System_WinRT" = [ "Win32_System" ]; - "Win32_System_WinRT_AllJoyn" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Composition" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_CoreInputView" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Direct3D11" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Display" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Graphics" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Graphics_Capture" = [ "Win32_System_WinRT_Graphics" ]; - "Win32_System_WinRT_Graphics_Direct2D" = [ "Win32_System_WinRT_Graphics" ]; - "Win32_System_WinRT_Graphics_Imaging" = [ "Win32_System_WinRT_Graphics" ]; - "Win32_System_WinRT_Holographic" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Isolation" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_ML" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Media" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Metadata" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Pdf" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Printing" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Shell" = [ "Win32_System_WinRT" ]; - "Win32_System_WinRT_Storage" = [ "Win32_System_WinRT" ]; - "Win32_System_WindowsProgramming" = [ "Win32_System" ]; - "Win32_System_WindowsSync" = [ "Win32_System" ]; - "Win32_System_Wmi" = [ "Win32_System" ]; - "Win32_UI" = [ "Win32" ]; - "Win32_UI_Accessibility" = [ "Win32_UI" ]; - "Win32_UI_Animation" = [ "Win32_UI" ]; - "Win32_UI_ColorSystem" = [ "Win32_UI" ]; - "Win32_UI_Controls" = [ "Win32_UI" ]; - "Win32_UI_Controls_Dialogs" = [ "Win32_UI_Controls" ]; - "Win32_UI_Controls_RichEdit" = [ "Win32_UI_Controls" ]; - "Win32_UI_HiDpi" = [ "Win32_UI" ]; - "Win32_UI_Input" = [ "Win32_UI" ]; - "Win32_UI_Input_Ime" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_Ink" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_KeyboardAndMouse" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_Pointer" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_Radial" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_Touch" = [ "Win32_UI_Input" ]; - "Win32_UI_Input_XboxController" = [ "Win32_UI_Input" ]; - "Win32_UI_InteractionContext" = [ "Win32_UI" ]; - "Win32_UI_LegacyWindowsEnvironmentFeatures" = [ "Win32_UI" ]; - "Win32_UI_Magnification" = [ "Win32_UI" ]; - "Win32_UI_Notifications" = [ "Win32_UI" ]; - "Win32_UI_Ribbon" = [ "Win32_UI" ]; - "Win32_UI_Shell" = [ "Win32_UI" ]; - "Win32_UI_Shell_Common" = [ "Win32_UI_Shell" ]; - "Win32_UI_Shell_PropertiesSystem" = [ "Win32_UI_Shell" ]; - "Win32_UI_TabletPC" = [ "Win32_UI" ]; - "Win32_UI_TextServices" = [ "Win32_UI" ]; - "Win32_UI_WindowsAndMessaging" = [ "Win32_UI" ]; - "Win32_UI_Wpf" = [ "Win32_UI" ]; - "Win32_Web" = [ "Win32" ]; - "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; - "implement" = [ "windows-implement" "windows-interface" "windows-core/implement" ]; - "windows-implement" = [ "dep:windows-implement" ]; - "windows-interface" = [ "dep:windows-interface" ]; + "default" = [ "std" ]; + "std" = [ "windows-result/std" "windows-strings/std" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_System" "Win32_System_SystemInformation" "default" ]; + resolvedDefaultFeatures = [ "default" "std" ]; }; - "windows-core" = rec { - crateName = "windows-core"; - version = "0.52.0"; + "windows-implement" = rec { + crateName = "windows-implement"; + version = "0.60.0"; edition = "2021"; - sha256 = "1nc3qv7sy24x0nlnb32f7alzpd6f72l4p24vl65vydbyil669ark"; - libName = "windows_core"; + sha256 = "0dm88k3hlaax85xkls4gf597ar4z8m5vzjjagzk910ph7b8xszx4"; + procMacro = true; + libName = "windows_implement"; authors = [ "Microsoft" ]; dependencies = [ { - name = "windows-targets"; - packageId = "windows-targets 0.52.6"; + name = "proc-macro2"; + packageId = "proc-macro2"; + usesDefaultFeatures = false; + } + { + name = "quote"; + packageId = "quote"; + usesDefaultFeatures = false; + } + { + name = "syn"; + packageId = "syn 2.0.100"; + usesDefaultFeatures = false; + features = [ "parsing" "proc-macro" "printing" "full" "clone-impls" ]; } ]; - features = { - }; - resolvedDefaultFeatures = [ "default" ]; + + }; + "windows-interface" = rec { + crateName = "windows-interface"; + version = "0.59.1"; + edition = "2021"; + sha256 = "1a4zr8740gyzzhq02xgl6vx8l669jwfby57xgf0zmkcdkyv134mx"; + procMacro = true; + libName = "windows_interface"; + authors = [ + "Microsoft" + ]; + dependencies = [ + { + name = "proc-macro2"; + packageId = "proc-macro2"; + usesDefaultFeatures = false; + } + { + name = "quote"; + packageId = "quote"; + usesDefaultFeatures = false; + } + { + name = "syn"; + packageId = "syn 2.0.100"; + usesDefaultFeatures = false; + features = [ "parsing" "proc-macro" "printing" "full" "clone-impls" ]; + } + ]; + }; "windows-link" = rec { crateName = "windows-link"; - version = "0.1.0"; + version = "0.1.1"; edition = "2021"; - sha256 = "1qr0srnkw148wbrws3726pm640h2vxgcdlxn0cxpbcg27irzvk3d"; + sha256 = "0f2cq7imbrppsmmnz8899hfhg07cp5gq6rh0bjhb1qb6nwshk13n"; libName = "windows_link"; authors = [ "Microsoft" @@ -12440,7 +11979,7 @@ rec { } { name = "windows-strings"; - packageId = "windows-strings"; + packageId = "windows-strings 0.3.1"; usesDefaultFeatures = false; } { @@ -12456,9 +11995,9 @@ rec { }; "windows-result" = rec { crateName = "windows-result"; - version = "0.3.1"; + version = "0.3.2"; edition = "2021"; - sha256 = "12dihsnl408sjjlyairc8vwjig68dvlfc00mi17pxawghpz4wdq6"; + sha256 = "0li2f76anf0rg7i966d9qs5iprsg555g9rgyzj7gcpfr9wdd2ky6"; libName = "windows_result"; authors = [ "Microsoft" @@ -12467,6 +12006,7 @@ rec { { name = "windows-link"; packageId = "windows-link"; + usesDefaultFeatures = false; } ]; features = { @@ -12474,7 +12014,7 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; - "windows-strings" = rec { + "windows-strings 0.3.1" = rec { crateName = "windows-strings"; version = "0.3.1"; edition = "2021"; @@ -12494,6 +12034,27 @@ rec { }; resolvedDefaultFeatures = [ "std" ]; }; + "windows-strings 0.4.0" = rec { + crateName = "windows-strings"; + version = "0.4.0"; + edition = "2021"; + sha256 = "15rg6a0ha1d231wwps2qlgyqrgkyj1r8v9vsb8nlbvih4ijajavs"; + libName = "windows_strings"; + authors = [ + "Microsoft" + ]; + dependencies = [ + { + name = "windows-link"; + packageId = "windows-link"; + usesDefaultFeatures = false; + } + ]; + features = { + "default" = [ "std" ]; + }; + resolvedDefaultFeatures = [ "std" ]; + }; "windows-sys 0.52.0" = rec { crateName = "windows-sys"; version = "0.52.0"; @@ -13269,9 +12830,9 @@ rec { }; "winnow" = rec { crateName = "winnow"; - version = "0.7.4"; + version = "0.7.6"; edition = "2021"; - sha256 = "0dmbsz6zfddcgsqzzqxw1h8f7zy19x407g7zl3hyp6vf2m2bb5qf"; + sha256 = "047abhm7qqgc32pf9a2arini5wsrx7p9wsbx3s106jx4pgczrlv3"; dependencies = [ { name = "memchr"; @@ -13289,6 +12850,25 @@ rec { }; resolvedDefaultFeatures = [ "alloc" "default" "std" ]; }; + "wit-bindgen-rt" = rec { + crateName = "wit-bindgen-rt"; + version = "0.39.0"; + edition = "2021"; + sha256 = "1hd65pa5hp0nl664m94bg554h4zlhrzmkjsf6lsgsb7yc4734hkg"; + libName = "wit_bindgen_rt"; + dependencies = [ + { + name = "bitflags"; + packageId = "bitflags"; + optional = true; + } + ]; + features = { + "async" = [ "dep:futures" "dep:once_cell" ]; + "bitflags" = [ "dep:bitflags" ]; + }; + resolvedDefaultFeatures = [ "bitflags" ]; + }; "write16" = rec { crateName = "write16"; version = "1.0.0"; @@ -13440,11 +13020,11 @@ rec { }; resolvedDefaultFeatures = [ "simd" ]; }; - "zerocopy 0.8.23" = rec { + "zerocopy 0.8.24" = rec { crateName = "zerocopy"; - version = "0.8.23"; + version = "0.8.24"; edition = "2021"; - sha256 = "1inbxgqhsxghawsss8x8517g30fpp8s3ll2ywy88ncm40m6l95zx"; + sha256 = "0yb8hyzfnwzr2wg4p7cnqmjps8fsw8xqnprafgpmfs8qisigx1i5"; authors = [ "Joshua Liebow-Feeser " "Jack Wrenn " @@ -13452,19 +13032,19 @@ rec { dependencies = [ { name = "zerocopy-derive"; - packageId = "zerocopy-derive 0.8.23"; + packageId = "zerocopy-derive 0.8.24"; optional = true; } { name = "zerocopy-derive"; - packageId = "zerocopy-derive 0.8.23"; + packageId = "zerocopy-derive 0.8.24"; target = { target, features }: false; } ]; devDependencies = [ { name = "zerocopy-derive"; - packageId = "zerocopy-derive 0.8.23"; + packageId = "zerocopy-derive 0.8.24"; } ]; features = { @@ -13502,11 +13082,11 @@ rec { ]; }; - "zerocopy-derive 0.8.23" = rec { + "zerocopy-derive 0.8.24" = rec { crateName = "zerocopy-derive"; - version = "0.8.23"; + version = "0.8.24"; edition = "2021"; - sha256 = "0m7iwisxz111sgkski722nyxv0rixbs0a9iylrcvhpfx1qfw0lk3"; + sha256 = "1gk9047pbq1yjj2jyiv0s37nqc53maqbmhcsjp6lhi2w7kvai5m9"; procMacro = true; libName = "zerocopy_derive"; authors = [ From 5e7b41bfb43d19e810a5ee84144f39d98f9a8fc2 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 9 Apr 2025 17:37:43 +0200 Subject: [PATCH 07/39] assign listener-class to all roles and consider all ports --- deploy/helm/airflow-operator/crds/crds.yaml | 670 +++++++++--------- .../operator-binary/src/airflow_controller.rs | 64 +- rust/operator-binary/src/crd/mod.rs | 264 +++---- rust/operator-binary/src/discovery.rs | 13 +- 4 files changed, 512 insertions(+), 499 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 8056e240..6b78f423 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -78,6 +78,14 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -295,6 +303,14 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -906,6 +922,14 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -1123,6 +1147,14 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -1287,183 +1319,40 @@ spec: config: default: {} properties: - airflowConfig: + affinity: default: - affinity: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - gracefulShutdownTimeout: null - logging: - containers: {} - enableVectorAgent: null - resources: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: - affinity: - default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). - properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - type: object - memory: - properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. - type: object - type: object - storage: - type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object + x-kubernetes-preserve-unknown-fields: true type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. enum: @@ -1472,6 +1361,128 @@ spec: - external-stable nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object type: object configOverrides: additionalProperties: @@ -1533,183 +1544,40 @@ spec: config: default: {} properties: - airflowConfig: + affinity: default: - affinity: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - gracefulShutdownTimeout: null - logging: - containers: {} - enableVectorAgent: null - resources: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: - affinity: - default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). - properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - type: object - memory: - properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. - type: object - type: object - storage: - type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object + x-kubernetes-preserve-unknown-fields: true type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. enum: @@ -1718,6 +1586,128 @@ spec: - external-stable nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object type: object configOverrides: additionalProperties: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 994595e9..a6a80127 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -41,9 +41,8 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, EmptyDirVolumeSource, EnvVar, PersistentVolumeClaim, PodTemplateSpec, - Probe, Service, ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, - VolumeMount, + ConfigMap, EmptyDirVolumeSource, EnvVar, PodTemplateSpec, Probe, Service, + ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, VolumeMount, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -983,14 +982,8 @@ fn build_server_rolegroup_statefulset( .context(AddVolumeMountSnafu)?; } - let mut pvcs: Option> = None; - // for roles with an http endpoint if let Some(http_port) = airflow_role.get_http_port() { - airflow_container - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; - let probe = Probe { tcp_socket: Some(TCPSocketAction { port: IntOrString::Int(http_port.into()), @@ -1004,35 +997,34 @@ fn build_server_rolegroup_statefulset( airflow_container.readiness_probe(probe.clone()); airflow_container.liveness_probe(probe); airflow_container.add_container_port(HTTP_PORT_NAME, http_port.into()); - - pvcs = if let Some(listener_class) = - airflow.merged_listener_class(airflow_role, &rolegroup_ref.role_group) - { - // externally-reachable listener endpoints should use a pvc volume... - if listener_class.discoverable() { - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerClass(listener_class.to_string()), - &recommended_labels, - ) - .context(BuildListenerVolumeSnafu)? - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerVolumeSnafu)?; - Some(vec![pvc]) - } else { - // ...whereas others will use ephemeral volumes - pb.add_listener_volume_by_listener_class( - LISTENER_VOLUME_NAME, - &listener_class.to_string(), - &recommended_labels, - ) - .context(AddVolumeSnafu)?; - None - } - } else { - None - }; } + let listener_class = &merged_airflow_config.listener_class; + // externally-reachable listener endpoints should use a pvc volume... + let pvcs = if listener_class.discoverable() { + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass(listener_class.to_string()), + &recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + Some(vec![pvc]) + } else { + // ...whereas others will use ephemeral volumes + pb.add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &listener_class.to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; + None + }; + + airflow_container + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + pb.add_container(airflow_container.build()); let metrics_container = ContainerBuilder::new("metrics") diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 4b514c98..47d4bc2c 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -34,7 +34,7 @@ use stackable_operator::{ }, role_utils::{ CommonConfiguration, GenericProductSpecificCommonConfig, GenericRoleConfig, Role, - RoleGroup, RoleGroupRef, + RoleGroupRef, }, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, @@ -205,7 +205,7 @@ pub mod versioned { /// The `webserver` role provides the main UI for user interaction. #[serde(default, skip_serializing_if = "Option::is_none")] - pub webservers: Option>, + pub webservers: Option>, /// The `scheduler` is responsible for triggering jobs and persisting their metadata to the backend database. /// Jobs are scheduled on the workers/executors. @@ -288,14 +288,7 @@ impl v1alpha1::AirflowCluster { /// the kubernetes executor is specified) pub fn get_role(&self, role: &AirflowRole) -> Option> { match role { - AirflowRole::Webserver => { - if let Some(webserver_config) = self.spec.webservers.to_owned() { - let role = extract_role_from_webserver_config(webserver_config); - Some(role) - } else { - None - } - } + AirflowRole::Webserver => self.spec.webservers.to_owned(), AirflowRole::Scheduler => self.spec.schedulers.to_owned(), AirflowRole::Worker => { if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { @@ -356,12 +349,13 @@ impl v1alpha1::AirflowCluster { let role = match role { AirflowRole::Webserver => { - &extract_role_from_webserver_config(self.spec.webservers.to_owned().context( - UnknownAirflowRoleSnafu { + self.spec + .webservers + .as_ref() + .context(UnknownAirflowRoleSnafu { role: role.to_string(), roles: AirflowRole::roles(), - }, - )?) + })? } AirflowRole::Worker => { if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { @@ -407,28 +401,47 @@ impl v1alpha1::AirflowCluster { &self, role: &AirflowRole, rolegroup_name: &String, - ) -> Option { - if role == &AirflowRole::Webserver { - if let Some(webservers) = self.spec.webservers.as_ref() { - let conf_defaults = Some(SupportedListenerClasses::ClusterInternal); - let mut conf_role = webservers.config.config.listener_class.to_owned(); - let mut conf_rolegroup = webservers - .role_groups - .get(rolegroup_name) - .map(|rg| rg.config.config.listener_class.clone()) - .unwrap_or_default(); - - conf_role.merge(&conf_defaults); - conf_rolegroup.merge(&conf_role); - - tracing::debug!("Merged listener-class: {:?}", conf_rolegroup); - conf_rolegroup - } else { - None + ) -> Result, Error> { + let listener_class_default = Some(SupportedListenerClasses::ClusterInternal); + + let role = match role { + AirflowRole::Webserver => { + self.spec + .webservers + .as_ref() + .context(UnknownAirflowRoleSnafu { + role: role.to_string(), + roles: AirflowRole::roles(), + })? } - } else { - None - } + AirflowRole::Worker => { + if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { + config + } else { + return Err(Error::NoRoleForExecutorFailure); + } + } + AirflowRole::Scheduler => { + self.spec + .schedulers + .as_ref() + .context(UnknownAirflowRoleSnafu { + role: role.to_string(), + roles: AirflowRole::roles(), + })? + } + }; + + let mut listener_class_role = role.config.config.listener_class.to_owned(); + let mut listener_class_rolegroup = role + .role_groups + .get(rolegroup_name) + .map(|rg| rg.config.config.listener_class.clone()) + .unwrap_or_default(); + listener_class_role.merge(&listener_class_default); + listener_class_rolegroup.merge(&listener_class_role); + tracing::info!("Merged listener-class: {:?}", listener_class_rolegroup); + Ok(listener_class_rolegroup) } /// Retrieve and merge resource configs for the executor template @@ -482,12 +495,7 @@ impl v1alpha1::AirflowCluster { .collect::>() .into_iter() .filter(|(rolegroup_name, _)| { - let listener_class = self.merged_listener_class(role, rolegroup_name); - if let Some(listener_class) = listener_class { - listener_class.discoverable() - } else { - false - } + self.resolved_listener_class_discoverable(role, rolegroup_name) }) .map(|(rolegroup_name, role_group)| { ( @@ -496,81 +504,114 @@ impl v1alpha1::AirflowCluster { ) }) .collect(), - AirflowRole::Scheduler | AirflowRole::Worker => vec![], + AirflowRole::Scheduler => self + .spec + .schedulers + .iter() + .flat_map(|role| &role.role_groups) + // Order rolegroups consistently, to avoid spurious downstream rewrites + .collect::>() + .into_iter() + .filter(|(rolegroup_name, _)| { + self.resolved_listener_class_discoverable(role, rolegroup_name) + }) + .map(|(rolegroup_name, role_group)| { + ( + self.rolegroup_ref(AirflowRole::Scheduler.to_string(), rolegroup_name), + role_group.replicas.unwrap_or_default(), + ) + }) + .collect(), + AirflowRole::Worker => { + if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { + config + .role_groups + .iter() + // Order rolegroups consistently, to avoid spurious downstream rewrites + .collect::>() + .into_iter() + .filter(|(rolegroup_name, _)| { + self.resolved_listener_class_discoverable(role, rolegroup_name) + }) + .map(|(rolegroup_name, role_group)| { + ( + self.rolegroup_ref( + AirflowRole::Scheduler.to_string(), + rolegroup_name, + ), + role_group.replicas.unwrap_or_default(), + ) + }) + .collect() + } else { + vec![] + } + } } } - pub fn pod_refs(&self, role: &AirflowRole) -> Result, Error> { - if let Some(port) = role.get_http_port() { - let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; - let rolegroup_ref_and_replicas = self.rolegroup_ref_and_replicas(role); - - Ok(rolegroup_ref_and_replicas - .iter() - .flat_map(|(rolegroup_ref, replicas)| { - let ns = ns.clone(); - (0..*replicas).map(move |i| PodRef { - namespace: ns.clone(), - role_group_service_name: rolegroup_ref.object_name(), - pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), - ports: HashMap::from([ - (HTTP_PORT_NAME.to_owned(), port), - (METRICS_PORT_NAME.to_owned(), METRICS_PORT), - ]), - fqdn_override: None, - }) - }) - .collect()) + fn resolved_listener_class_discoverable( + &self, + role: &AirflowRole, + rolegroup_name: &&String, + ) -> bool { + if let Ok(Some(listener_class)) = self.merged_listener_class(role, rolegroup_name) { + tracing::info!( + "Merged listener-class for role is discoverable?: {}/{}/{}", + role, + listener_class, + listener_class.discoverable() + ); + listener_class.discoverable() } else { - Ok(vec![]) + // merged_listener_class returns an error is one of the roles was not found: + // all roles are mandatory for airflow to work, but a missing role will by + // definition not have a listener class + tracing::info!( + "Merged listener-class for role is NOT discoverable?: {}/{}", + role, + rolegroup_name + ); + false } } + pub fn pod_refs(&self, role: &AirflowRole) -> Result, Error> { + let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; + let rolegroup_ref_and_replicas = self.rolegroup_ref_and_replicas(role); + + Ok(rolegroup_ref_and_replicas + .iter() + .flat_map(|(rolegroup_ref, replicas)| { + let ns = ns.clone(); + (0..*replicas).map(move |i| PodRef { + namespace: ns.clone(), + role_group_service_name: rolegroup_ref.object_name(), + pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), + ports: HashMap::from([ + (HTTP_PORT_NAME.to_owned(), HTTP_PORT), + (METRICS_PORT_NAME.to_owned(), METRICS_PORT), + ]), + fqdn_override: None, + }) + }) + .collect()) + } + pub async fn listener_refs( &self, client: &stackable_operator::client::Client, role: &AirflowRole, ) -> Result, Error> { let pod_refs = self.pod_refs(role)?; + + tracing::info!("pod_refs for role {role}: {:#?}", pod_refs); get_persisted_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) .await .context(ListenerPodRefSnafu) } } -fn extract_role_from_webserver_config( - fragment: Role, -) -> Role { - Role { - config: CommonConfiguration { - config: fragment.config.config.airflow_config, - config_overrides: fragment.config.config_overrides, - env_overrides: fragment.config.env_overrides, - cli_overrides: fragment.config.cli_overrides, - pod_overrides: fragment.config.pod_overrides, - product_specific_common_config: fragment.config.product_specific_common_config, - }, - role_config: fragment.role_config, - role_groups: fragment - .role_groups - .into_iter() - .map(|(k, v)| { - (k, RoleGroup { - config: CommonConfiguration { - config: v.config.config.airflow_config, - config_overrides: v.config.config_overrides, - env_overrides: v.config.env_overrides, - cli_overrides: v.config.cli_overrides, - pod_overrides: v.config.pod_overrides, - product_specific_common_config: v.config.product_specific_common_config, - }, - replicas: v.replicas, - }) - }) - .collect(), - } -} - #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AirflowAuthorization { @@ -868,30 +909,6 @@ pub struct ExecutorConfig { pub graceful_shutdown_timeout: Option, } -#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] -#[fragment_attrs( - derive( - Clone, - Debug, - Default, - Deserialize, - Merge, - JsonSchema, - PartialEq, - Serialize - ), - serde(rename_all = "camelCase") -)] -pub struct WebserverConfig { - #[fragment_attrs(serde(default))] - #[serde(flatten)] - pub airflow_config: AirflowConfig, - - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. - #[serde(default)] - pub listener_class: SupportedListenerClasses, -} - #[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] #[fragment_attrs( derive( @@ -919,6 +936,10 @@ pub struct AirflowConfig { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + #[serde(default)] + pub listener_class: SupportedListenerClasses, } impl AirflowConfig { @@ -936,6 +957,7 @@ impl AirflowConfig { } AirflowRole::Worker => DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT, }), + listener_class: Some(SupportedListenerClasses::ClusterInternal), } } } diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index 33671ab6..a2943c0f 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -10,7 +10,10 @@ use stackable_operator::{ use crate::{ airflow_controller::AIRFLOW_CONTROLLER_NAME, - crd::{AirflowRole, HTTP_PORT_NAME, build_recommended_labels, utils::PodRef, v1alpha1}, + crd::{ + AirflowRole, HTTP_PORT_NAME, METRICS_PORT_NAME, build_recommended_labels, utils::PodRef, + v1alpha1, + }, }; type Result = std::result::Result; @@ -71,10 +74,16 @@ pub fn build_discovery_configmap( { if let Some(ui_port) = ports.get(HTTP_PORT_NAME) { cmm.add_data( - format!("{pod_name}.http"), + format!("{pod_name}.{HTTP_PORT_NAME}"), format!("{fqdn_override}:{ui_port}"), ); } + if let Some(metrics_port) = ports.get(METRICS_PORT_NAME) { + cmm.add_data( + format!("{pod_name}.{METRICS_PORT_NAME}"), + format!("{fqdn_override}:{metrics_port}"), + ); + } } } } From 8706a3ef13c1912bb7b7dc6cadb3ff2511ff8469 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 9 Apr 2025 18:03:56 +0200 Subject: [PATCH 08/39] fixed podref name for workers --- .../operator-binary/src/airflow_controller.rs | 2 +- rust/operator-binary/src/crd/mod.rs | 20 +++---------------- 2 files changed, 4 insertions(+), 18 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index a6a80127..baef758d 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -589,7 +589,7 @@ pub async fn reconcile_airflow( } } - tracing::info!( + tracing::debug!( "Listener references prepared for the ConfigMap {:#?}", listener_refs ); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 47d4bc2c..f9dd2c72 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -440,7 +440,7 @@ impl v1alpha1::AirflowCluster { .unwrap_or_default(); listener_class_role.merge(&listener_class_default); listener_class_rolegroup.merge(&listener_class_role); - tracing::info!("Merged listener-class: {:?}", listener_class_rolegroup); + tracing::debug!("Merged listener-class: {:?}", listener_class_rolegroup); Ok(listener_class_rolegroup) } @@ -535,10 +535,7 @@ impl v1alpha1::AirflowCluster { }) .map(|(rolegroup_name, role_group)| { ( - self.rolegroup_ref( - AirflowRole::Scheduler.to_string(), - rolegroup_name, - ), + self.rolegroup_ref(AirflowRole::Worker.to_string(), rolegroup_name), role_group.replicas.unwrap_or_default(), ) }) @@ -556,22 +553,11 @@ impl v1alpha1::AirflowCluster { rolegroup_name: &&String, ) -> bool { if let Ok(Some(listener_class)) = self.merged_listener_class(role, rolegroup_name) { - tracing::info!( - "Merged listener-class for role is discoverable?: {}/{}/{}", - role, - listener_class, - listener_class.discoverable() - ); listener_class.discoverable() } else { // merged_listener_class returns an error is one of the roles was not found: // all roles are mandatory for airflow to work, but a missing role will by // definition not have a listener class - tracing::info!( - "Merged listener-class for role is NOT discoverable?: {}/{}", - role, - rolegroup_name - ); false } } @@ -605,7 +591,7 @@ impl v1alpha1::AirflowCluster { ) -> Result, Error> { let pod_refs = self.pod_refs(role)?; - tracing::info!("pod_refs for role {role}: {:#?}", pod_refs); + tracing::debug!("Pod references for role {role}: {:#?}", pod_refs); get_persisted_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) .await .context(ListenerPodRefSnafu) From 07efc9f0f2c249ac7c96bb4c0a47a7824762bac7 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 10 Apr 2025 10:47:30 +0200 Subject: [PATCH 09/39] added integration test --- .../kuttl/external-access/00-patch-ns.yaml.j2 | 9 ++ .../kuttl/external-access/00-rbac.yaml.j2 | 29 ++++++ .../kuttl/external-access/10-assert.yaml | 14 +++ .../10-install-postgresql.yaml | 12 +++ .../kuttl/external-access/20-assert.yaml.j2 | 24 +++++ .../external-access/20-install-redis.yaml.j2 | 14 +++ .../kuttl/external-access/30-assert.yaml.j2 | 10 ++ ...tor-aggregator-discovery-configmap.yaml.j2 | 9 ++ .../kuttl/external-access/40-assert.yaml.j2 | 97 ++++++++++++++++++ .../40-install-airflow-cluster.yaml.j2 | 64 ++++++++++++ .../external-access/50-access-airflow.txt.j2 | 99 +++++++++++++++++++ .../external-access/50-access-airflow.yaml | 6 ++ .../kuttl/external-access/50-assert.yaml | 11 +++ .../helm-bitnami-postgresql-values.yaml.j2 | 30 ++++++ .../helm-bitnami-redis-values.yaml.j2 | 43 ++++++++ tests/test-definition.yaml | 4 + 16 files changed, 475 insertions(+) create mode 100644 tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/00-rbac.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/10-assert.yaml create mode 100644 tests/templates/kuttl/external-access/10-install-postgresql.yaml create mode 100644 tests/templates/kuttl/external-access/20-assert.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/20-install-redis.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/30-assert.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/40-assert.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/50-access-airflow.txt.j2 create mode 100644 tests/templates/kuttl/external-access/50-access-airflow.yaml create mode 100644 tests/templates/kuttl/external-access/50-assert.yaml create mode 100644 tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 create mode 100644 tests/templates/kuttl/external-access/helm-bitnami-redis-values.yaml.j2 diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-rbac.yaml.j2 b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/external-access/10-assert.yaml b/tests/templates/kuttl/external-access/10-assert.yaml new file mode 100644 index 00000000..319e927a --- /dev/null +++ b/tests/templates/kuttl/external-access/10-assert.yaml @@ -0,0 +1,14 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-postgresql +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/10-install-postgresql.yaml b/tests/templates/kuttl/external-access/10-install-postgresql.yaml new file mode 100644 index 00000000..9e0529d1 --- /dev/null +++ b/tests/templates/kuttl/external-access/10-install-postgresql.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-postgresql + --namespace $NAMESPACE + --version 16.4.2 + -f helm-bitnami-postgresql-values.yaml + oci://registry-1.docker.io/bitnamicharts/postgresql + --wait + timeout: 600 diff --git a/tests/templates/kuttl/external-access/20-assert.yaml.j2 b/tests/templates/kuttl/external-access/20-assert.yaml.j2 new file mode 100644 index 00000000..8d585401 --- /dev/null +++ b/tests/templates/kuttl/external-access/20-assert.yaml.j2 @@ -0,0 +1,24 @@ +{% if test_scenario['values']['executor'] == 'celery' %} +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-redis +timeout: 360 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-master +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-redis-replicas +status: + readyReplicas: 1 + replicas: 1 +{% endif %} diff --git a/tests/templates/kuttl/external-access/20-install-redis.yaml.j2 b/tests/templates/kuttl/external-access/20-install-redis.yaml.j2 new file mode 100644 index 00000000..3a07199d --- /dev/null +++ b/tests/templates/kuttl/external-access/20-install-redis.yaml.j2 @@ -0,0 +1,14 @@ +{% if test_scenario['values']['executor'] == 'celery' %} +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: >- + helm install airflow-redis + --namespace $NAMESPACE + --version 17.11.3 + -f helm-bitnami-redis-values.yaml + --repo https://charts.bitnami.com/bitnami redis + --wait + timeout: 600 +{% endif %} diff --git a/tests/templates/kuttl/external-access/30-assert.yaml.j2 b/tests/templates/kuttl/external-access/30-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/external-access/40-assert.yaml.j2 b/tests/templates/kuttl/external-access/40-assert.yaml.j2 new file mode 100644 index 00000000..331bb1a9 --- /dev/null +++ b/tests/templates/kuttl/external-access/40-assert.yaml.j2 @@ -0,0 +1,97 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-available-condition +timeout: 600 +commands: + - script: kubectl -n $NAMESPACE wait --for=condition=available airflowclusters.airflow.stackable.tech/airflow --timeout 301s +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-airflow-cluster +timeout: 1200 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-webserver-default +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-webserver-external-unstable +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-webserver-cluster-internal +status: + readyReplicas: 1 + replicas: 1 +{% if test_scenario['values']['executor'] == 'celery' %} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-worker-default +spec: + template: + spec: + terminationGracePeriodSeconds: 300 +status: + readyReplicas: 2 + replicas: 2 +{% endif %} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-scheduler-default +spec: + template: + spec: + terminationGracePeriodSeconds: 120 +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-webserver +status: + expectedPods: 3 + currentHealthy: 3 + disruptionsAllowed: 1 +{% if test_scenario['values']['executor'] == 'celery' %} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-worker +status: + expectedPods: 2 + currentHealthy: 2 + disruptionsAllowed: 1 +{% endif %} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: airflow-scheduler +status: + expectedPods: 1 + currentHealthy: 1 + disruptionsAllowed: 1 diff --git a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 new file mode 100644 index 00000000..8d5af32b --- /dev/null +++ b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 @@ -0,0 +1,64 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +metadata: + name: install-airflow-db +timeout: 480 +--- +apiVersion: v1 +kind: Secret +metadata: + name: test-airflow-credentials +type: Opaque +stringData: + adminUser.username: airflow + adminUser.firstname: Airflow + adminUser.lastname: Admin + adminUser.email: airflow@airflow.com + adminUser.password: airflow + connections.secretKey: thisISaSECRET_1234 + connections.sqlalchemyDatabaseUri: postgresql+psycopg2://airflow:airflow@airflow-postgresql/airflow + connections.celeryResultBackend: db+postgresql://airflow:airflow@airflow-postgresql/airflow + connections.celeryBrokerUrl: redis://:redis@airflow-redis-master:6379/0 +--- +apiVersion: airflow.stackable.tech/v1alpha1 +kind: AirflowCluster +metadata: + name: airflow +spec: + image: +{% if test_scenario['values']['airflow'].find(",") > 0 %} + custom: "{{ test_scenario['values']['airflow'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['airflow'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['airflow'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + loadExamples: true + credentialsSecret: test-airflow-credentials + webservers: + config: + listenerClass: external-stable + roleGroups: + default: + replicas: 1 + external-unstable: + replicas: 1 + config: + listenerClass: external-unstable + cluster-internal: + replicas: 1 + config: + listenerClass: cluster-internal + celeryExecutors: + config: + listenerClass: external-stable + roleGroups: + default: + replicas: 2 + schedulers: + config: + listenerClass: external-unstable + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 b/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 new file mode 100644 index 00000000..82318342 --- /dev/null +++ b/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 @@ -0,0 +1,99 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: access-airflow +spec: + template: + spec: + serviceAccountName: test-sa + containers: + - name: access-airflow +{% if test_scenario['values']['airflow'].find(",") > 0 %} + image: "{{ test_scenario['values']['airflow'].split(',')[1] }}" +{% else %} + image: oci.stackable.tech/sdp/airflow:{{ test_scenario['values']['airflow'] }}-stackable0.0.0-dev +{% endif %} + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - /tmp/script/script.sh + env: + - name: SCHEDULER_METRICS + valueFrom: + configMapKeyRef: + name: airflow + key: airflow-scheduler-default-0.metrics + - name: WORKER_0_METRICS + valueFrom: + configMapKeyRef: + name: airflow + key: airflow-worker-default-0.metrics + - name: WORKER_1_METRICS + valueFrom: + configMapKeyRef: + name: airflow + key: airflow-worker-default-1.metrics + - name: WEBSERVER_DEFAULT_HTTP + valueFrom: + configMapKeyRef: + name: airflow + key: airflow-webserver-default-0.http + - name: WEBSERVER_DEFAULT_METRICS + valueFrom: + configMapKeyRef: + name: airflow + key: airflow-webserver-default-0.metrics + - name: WEBSERVER_EXTERNAL_HTTP + valueFrom: + configMapKeyRef: + name: airflow + key: airflow-webserver-external-unstable-0.http + - name: WEBSERVER_EXTERNAL_METRICS + valueFrom: + configMapKeyRef: + name: airflow + key: airflow-webserver-external-unstable-0.metrics + volumeMounts: + - name: script + mountPath: /tmp/script + volumes: + - name: script + configMap: + name: access-airflow-script + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsUser: 1000 + restartPolicy: OnFailure +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: access-airflow-script +data: + script.sh: | + set -euxo pipefail + + echo "Attempting to reach master at $SCHEDULER_METRICS..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${SCHEDULER_METRICS}" | grep 200 + + echo "Attempting to reach master at $WORKER_0_METRICS..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WORKER_0_METRICS}" | grep 200 + + echo "Attempting to reach region-server at $WORKER_1_METRICS..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WORKER_1_METRICS}" | grep 200 + + echo "Attempting to reach rest-server at $WEBSERVER_DEFAULT_HTTP..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_DEFAULT_HTTP}" | grep 200 + + echo "Attempting to reach rest-server at $WEBSERVER_DEFAULT_METRICS..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_DEFAULT_METRICS}" | grep 200 + + echo "Attempting to reach rest-server at $WEBSERVER_EXTERNAL_HTTP..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_EXTERNAL_HTTP}" | grep 200 + + echo "Attempting to reach rest-server at $WEBSERVER_EXTERNAL_METRICS..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_EXTERNAL_METRICS}" | grep 200 + + echo "All tests successful!" diff --git a/tests/templates/kuttl/external-access/50-access-airflow.yaml b/tests/templates/kuttl/external-access/50-access-airflow.yaml new file mode 100644 index 00000000..f1ea0f7f --- /dev/null +++ b/tests/templates/kuttl/external-access/50-access-airflow.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 50-access-airflow.txt | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/50-assert.yaml b/tests/templates/kuttl/external-access/50-assert.yaml new file mode 100644 index 00000000..5e7d2c20 --- /dev/null +++ b/tests/templates/kuttl/external-access/50-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: access-airflow +status: + succeeded: 1 diff --git a/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 new file mode 100644 index 00000000..f1320d2d --- /dev/null +++ b/tests/templates/kuttl/external-access/helm-bitnami-postgresql-values.yaml.j2 @@ -0,0 +1,30 @@ +--- +volumePermissions: + enabled: false + securityContext: + runAsUser: auto + +primary: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "400m" +shmVolume: + chmod: + enabled: false + +auth: + username: airflow + password: airflow + database: airflow diff --git a/tests/templates/kuttl/external-access/helm-bitnami-redis-values.yaml.j2 b/tests/templates/kuttl/external-access/helm-bitnami-redis-values.yaml.j2 new file mode 100644 index 00000000..d920abc5 --- /dev/null +++ b/tests/templates/kuttl/external-access/helm-bitnami-redis-values.yaml.j2 @@ -0,0 +1,43 @@ +--- +volumePermissions: + enabled: false + containerSecurityContext: + runAsUser: auto + +master: + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "200m" + limits: + memory: "128Mi" + cpu: "800m" + +replica: + replicaCount: 1 + podSecurityContext: +{% if test_scenario['values']['openshift'] == 'true' %} + enabled: false +{% else %} + enabled: true +{% endif %} + containerSecurityContext: + enabled: false + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "128Mi" + cpu: "400m" + +auth: + password: redis diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 82f11035..73e8948e 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -83,6 +83,10 @@ tests: dimensions: - airflow-latest - openshift + - name: external-access + dimensions: + - airflow + - openshift suites: - name: nightly # Run nightly with the latest airflow From 87a129584fe805dc95f628acbb93110c03c6fde5 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 10 Apr 2025 11:46:22 +0200 Subject: [PATCH 10/39] changelog --- CHANGELOG.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index aab6611c..9c5d66e6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- Added listener support for Airflow ([#604]). + ### Changed - Replace stackable-operator `initialize_logging` with stackable-telemetry `Tracing` ([#601]). @@ -14,6 +18,7 @@ - Use `json` file extension for log files ([#607]). [#601]: https://github.com/stackabletech/airflow-operator/pull/601 +[#604]: https://github.com/stackabletech/airflow-operator/pull/604 [#607]: https://github.com/stackabletech/airflow-operator/pull/607 ## [25.3.0] - 2025-03-21 From e6467002ca354618af1972f1a9a61e1598bada0f Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 10 Apr 2025 15:12:02 +0200 Subject: [PATCH 11/39] added docs --- .../pages/usage-guide/listenerclass.adoc | 37 ++++++++++++++----- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc index 67c9f330..34758f3f 100644 --- a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc @@ -1,18 +1,37 @@ = Service exposition with ListenerClasses :description: Configure Airflow service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable. -Airflow offers a web UI and an API, both are exposed by the webserver process under the `webserver` role. -The Operator deploys a service called `-webserver` (where `` is the name of the AirflowCluster) through which Airflow can be reached. +The operator deploys a xref:listener-operator:listener.adoc[Listener] for each Scheduler, Webserver and (Celery-)Worker pod. +They all default to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.{schedulers,celeryExecutors,webservers}.config.listenerClass`: -This service can have three different types: `cluster-internal`, `external-unstable` and `external-stable`. -Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. +[source,yaml] +---- +spec: + schedulers: + config: + listenerClass: external-unstable # <1> + webservers: + config: + listenerClass: external-unstable + celeryExecutors: + config: + listenerClass: external-unstable +---- +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). +This can be set separately for all three roles (but not for kuberneetesExecutors as the resulting worker pods are temporary). -This is how the listener class is configured: +Externally-reachable endpoints (i.e. where listener-class = `external-unstable` or `external-unstable`) are written to a ConfigMap called ``, listing each rolegroup by replica. +Both the http and metrics ports are exposed: [source,yaml] ---- -spec: - clusterConfig: - listenerClass: cluster-internal # <1> +apiVersion: v1 +data: + airflow-scheduler-default-0.metrics: 172.19.0.5:30589 + airflow-webserver-default-0.http: 172.19.0.3:31891 + airflow-webserver-default-0.metrics: 172.19.0.3:30414 + airflow-worker-default-0.metrics: 172.19.0.5:30371 + airflow-worker-default-1.metrics: 172.19.0.3:30514 +kind: ConfigMap +... ---- -<1> The default `cluster-internal` setting. From 64175f3cd674d00b66951b9d7c2089a087a8596b Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 16 Apr 2025 15:25:45 +0200 Subject: [PATCH 12/39] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/crd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index a3e62f31..e8a834d7 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -923,7 +923,7 @@ pub struct AirflowConfig { #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the airflow services. #[serde(default)] pub listener_class: SupportedListenerClasses, } From 7837af3a43900cd724024f6c3cb544ead66ee099 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 16 Apr 2025 15:26:18 +0200 Subject: [PATCH 13/39] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/crd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e8a834d7..b9aeac5f 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -499,7 +499,7 @@ impl v1alpha1::AirflowCluster { }) .map(|(rolegroup_name, role_group)| { ( - self.rolegroup_ref(AirflowRole::Webserver.to_string(), rolegroup_name), + self.rolegroup_ref(role.to_string(), rolegroup_name), role_group.replicas.unwrap_or_default(), ) }) From 6b359d16c104cabffe15887ab7689bfc2d5be0f1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 16 Apr 2025 15:26:27 +0200 Subject: [PATCH 14/39] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/crd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index b9aeac5f..afac2853 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -517,7 +517,7 @@ impl v1alpha1::AirflowCluster { }) .map(|(rolegroup_name, role_group)| { ( - self.rolegroup_ref(AirflowRole::Scheduler.to_string(), rolegroup_name), + self.rolegroup_ref(role.to_string(), rolegroup_name), role_group.replicas.unwrap_or_default(), ) }) From ebe8079b776ab42855cd499d2642ffc6505f4a1c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Wed, 16 Apr 2025 15:26:37 +0200 Subject: [PATCH 15/39] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/crd/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index afac2853..f86ca3fe 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -535,7 +535,7 @@ impl v1alpha1::AirflowCluster { }) .map(|(rolegroup_name, role_group)| { ( - self.rolegroup_ref(AirflowRole::Worker.to_string(), rolegroup_name), + self.rolegroup_ref(role.to_string(), rolegroup_name), role_group.replicas.unwrap_or_default(), ) }) From acbca3e2d3148069ae2af7a5160aded424938ffe Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Wed, 16 Apr 2025 15:29:17 +0200 Subject: [PATCH 16/39] corrected callout comments and regenerate charts --- deploy/helm/airflow-operator/crds/crds.yaml | 12 ++++++------ .../kuttl/external-access/50-access-airflow.txt.j2 | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 6b78f423..f0bb11a9 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -79,7 +79,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. enum: - cluster-internal - external-unstable @@ -304,7 +304,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. enum: - cluster-internal - external-unstable @@ -923,7 +923,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. enum: - cluster-internal - external-unstable @@ -1148,7 +1148,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. enum: - cluster-internal - external-unstable @@ -1354,7 +1354,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. enum: - cluster-internal - external-unstable @@ -1579,7 +1579,7 @@ spec: nullable: true type: string listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. enum: - cluster-internal - external-unstable diff --git a/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 b/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 index 82318342..f53a30b8 100644 --- a/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 +++ b/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 @@ -75,25 +75,25 @@ data: script.sh: | set -euxo pipefail - echo "Attempting to reach master at $SCHEDULER_METRICS..." + echo "Attempting to reach scheduler at $SCHEDULER_METRICS..." curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${SCHEDULER_METRICS}" | grep 200 - echo "Attempting to reach master at $WORKER_0_METRICS..." + echo "Attempting to reach worker at $WORKER_0_METRICS..." curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WORKER_0_METRICS}" | grep 200 - echo "Attempting to reach region-server at $WORKER_1_METRICS..." + echo "Attempting to reach worker at $WORKER_1_METRICS..." curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WORKER_1_METRICS}" | grep 200 - echo "Attempting to reach rest-server at $WEBSERVER_DEFAULT_HTTP..." + echo "Attempting to reach webserver at $WEBSERVER_DEFAULT_HTTP..." curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_DEFAULT_HTTP}" | grep 200 - echo "Attempting to reach rest-server at $WEBSERVER_DEFAULT_METRICS..." + echo "Attempting to reach webserver at $WEBSERVER_DEFAULT_METRICS..." curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_DEFAULT_METRICS}" | grep 200 - echo "Attempting to reach rest-server at $WEBSERVER_EXTERNAL_HTTP..." + echo "Attempting to reach webserver at $WEBSERVER_EXTERNAL_HTTP..." curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_EXTERNAL_HTTP}" | grep 200 - echo "Attempting to reach rest-server at $WEBSERVER_EXTERNAL_METRICS..." + echo "Attempting to reach webserver at $WEBSERVER_EXTERNAL_METRICS..." curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_EXTERNAL_METRICS}" | grep 200 echo "All tests successful!" From 8ae33031fde8059ccb97600c16db7994292271bd Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 17 Apr 2025 13:45:15 +0200 Subject: [PATCH 17/39] use ephemeral listeners classes; fix test missing dimension --- .../operator-binary/src/airflow_controller.rs | 34 +++++-------------- rust/operator-binary/src/crd/mod.rs | 6 ++-- rust/operator-binary/src/crd/utils.rs | 8 ++--- .../{20-assert.yaml.j2 => 20-assert.yaml} | 2 -- ...ll-redis.yaml.j2 => 20-install-redis.yaml} | 2 -- .../{40-assert.yaml.j2 => 40-assert.yaml} | 4 --- 6 files changed, 16 insertions(+), 40 deletions(-) rename tests/templates/kuttl/external-access/{20-assert.yaml.j2 => 20-assert.yaml} (83%) rename tests/templates/kuttl/external-access/{20-install-redis.yaml.j2 => 20-install-redis.yaml} (80%) rename tests/templates/kuttl/external-access/{40-assert.yaml.j2 => 40-assert.yaml} (92%) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 80f60782..b89008d7 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -23,10 +23,7 @@ use stackable_operator::{ container::ContainerBuilder, resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, - volume::{ - ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, - ListenerReference, VolumeBuilder, - }, + volume::{ListenerOperatorVolumeSourceBuilderError, VolumeBuilder}, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -980,26 +977,14 @@ fn build_server_rolegroup_statefulset( } let listener_class = &merged_airflow_config.listener_class; - // externally-reachable listener endpoints should use a pvc volume... - let pvcs = if listener_class.discoverable() { - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerClass(listener_class.to_string()), - &recommended_labels, - ) - .context(BuildListenerVolumeSnafu)? - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerVolumeSnafu)?; - Some(vec![pvc]) - } else { - // ...whereas others will use ephemeral volumes - pb.add_listener_volume_by_listener_class( - LISTENER_VOLUME_NAME, - &listener_class.to_string(), - &recommended_labels, - ) - .context(AddVolumeSnafu)?; - None - }; + // all listeners will use ephemeral volumes as they can/should + // be removed when the pods is re-started, and no data needs to be preserved + pb.add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &listener_class.to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; airflow_container .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) @@ -1157,7 +1142,6 @@ fn build_server_rolegroup_statefulset( }, service_name: rolegroup_ref.object_name(), template: pod_template, - volume_claim_templates: pvcs, ..StatefulSetSpec::default() }; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f86ca3fe..f548d263 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -43,7 +43,7 @@ use stackable_operator::{ versioned::versioned, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; -use utils::{PodRef, get_persisted_listener_podrefs}; +use utils::{PodRef, get_listener_podrefs}; use crate::crd::{ affinity::{get_affinity, get_executor_affinity}, @@ -555,7 +555,7 @@ impl v1alpha1::AirflowCluster { if let Ok(Some(listener_class)) = self.merged_listener_class(role, rolegroup_name) { listener_class.discoverable() } else { - // merged_listener_class returns an error is one of the roles was not found: + // merged_listener_class returns an error if one of the roles was not found: // all roles are mandatory for airflow to work, but a missing role will by // definition not have a listener class false @@ -592,7 +592,7 @@ impl v1alpha1::AirflowCluster { let pod_refs = self.pod_refs(role)?; tracing::debug!("Pod references for role {role}: {:#?}", pod_refs); - get_persisted_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) + get_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) .await .context(ListenerPodRefSnafu) } diff --git a/rust/operator-binary/src/crd/utils.rs b/rust/operator-binary/src/crd/utils.rs index c4b7b585..a54cd0bf 100644 --- a/rust/operator-binary/src/crd/utils.rs +++ b/rust/operator-binary/src/crd/utils.rs @@ -59,15 +59,15 @@ impl PodRef { } } -pub async fn get_persisted_listener_podrefs( +pub async fn get_listener_podrefs( client: &stackable_operator::client::Client, pod_refs: Vec, listener_prefix: &str, ) -> Result, Error> { try_join_all(pod_refs.into_iter().map(|pod_ref| async { - // N.B. use the naming convention for persistent listener volumes as we - // only want externally-reachable endpoints. - let listener_name = format!("{listener_prefix}-{}", pod_ref.pod_name); + // N.B. use the naming convention for ephemeral listener volumes as we + // have defined all listeners to be so. + let listener_name = format!("{}-{listener_prefix}", pod_ref.pod_name); let listener_ref = || ObjectRef::::new(&listener_name).within(&pod_ref.namespace); let pod_obj_ref = || ObjectRef::::new(&pod_ref.pod_name).within(&pod_ref.namespace); let listener = client diff --git a/tests/templates/kuttl/external-access/20-assert.yaml.j2 b/tests/templates/kuttl/external-access/20-assert.yaml similarity index 83% rename from tests/templates/kuttl/external-access/20-assert.yaml.j2 rename to tests/templates/kuttl/external-access/20-assert.yaml index 8d585401..f038df42 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml.j2 +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -1,4 +1,3 @@ -{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: kuttl.dev/v1beta1 kind: TestAssert @@ -21,4 +20,3 @@ metadata: status: readyReplicas: 1 replicas: 1 -{% endif %} diff --git a/tests/templates/kuttl/external-access/20-install-redis.yaml.j2 b/tests/templates/kuttl/external-access/20-install-redis.yaml similarity index 80% rename from tests/templates/kuttl/external-access/20-install-redis.yaml.j2 rename to tests/templates/kuttl/external-access/20-install-redis.yaml index 3a07199d..cc1edc53 100644 --- a/tests/templates/kuttl/external-access/20-install-redis.yaml.j2 +++ b/tests/templates/kuttl/external-access/20-install-redis.yaml @@ -1,4 +1,3 @@ -{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: kuttl.dev/v1beta1 kind: TestStep @@ -11,4 +10,3 @@ commands: --repo https://charts.bitnami.com/bitnami redis --wait timeout: 600 -{% endif %} diff --git a/tests/templates/kuttl/external-access/40-assert.yaml.j2 b/tests/templates/kuttl/external-access/40-assert.yaml similarity index 92% rename from tests/templates/kuttl/external-access/40-assert.yaml.j2 rename to tests/templates/kuttl/external-access/40-assert.yaml index 331bb1a9..dbee54fc 100644 --- a/tests/templates/kuttl/external-access/40-assert.yaml.j2 +++ b/tests/templates/kuttl/external-access/40-assert.yaml @@ -40,7 +40,6 @@ metadata: status: readyReplicas: 1 replicas: 1 -{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: apps/v1 kind: StatefulSet @@ -53,7 +52,6 @@ spec: status: readyReplicas: 2 replicas: 2 -{% endif %} --- apiVersion: apps/v1 kind: StatefulSet @@ -75,7 +73,6 @@ status: expectedPods: 3 currentHealthy: 3 disruptionsAllowed: 1 -{% if test_scenario['values']['executor'] == 'celery' %} --- apiVersion: policy/v1 kind: PodDisruptionBudget @@ -85,7 +82,6 @@ status: expectedPods: 2 currentHealthy: 2 disruptionsAllowed: 1 -{% endif %} --- apiVersion: policy/v1 kind: PodDisruptionBudget From f4ecd4364a2c8d954ed748e8c0dacf4df04df640 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 22 Apr 2025 09:33:07 +0200 Subject: [PATCH 18/39] allow config map endpoints to be removed if cluster is stopped --- rust/operator-binary/src/airflow_controller.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index b89008d7..4a5f4fc9 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -575,7 +575,11 @@ pub async fn reconcile_airflow( listener_refs ); - if !listener_refs.is_empty() { + // if paused or stopped the podrefs have not been collected (see comment + // above): the config map should remain unchanged if paused, but the + // refs removed from it if the cluster is stopped as replicas will + // have been set to 0. + if !airflow.spec.cluster_operation.reconciliation_paused { let endpoint_cm = build_discovery_configmap(airflow, &resolved_product_image, &listener_refs) .context(BuildDiscoveryConfigMapSnafu)?; From 5c7022c90a34140e1c89ade4bc29bf3431250bff Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 22 Apr 2025 11:01:42 +0200 Subject: [PATCH 19/39] review feedback --- rust/operator-binary/src/airflow_controller.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 4a5f4fc9..90d34113 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -338,12 +338,12 @@ pub enum Error { #[snafu(display("cannot collect discovery configuration"))] CollectDiscoveryConfig { source: crate::crd::Error }, - #[snafu(display("failed to apply discovery configmap"))] + #[snafu(display("failed to apply discovery ConfigMap"))] ApplyDiscoveryConfigMap { source: stackable_operator::cluster_resources::Error, }, - #[snafu(display("failed to build discovery configmap"))] + #[snafu(display("failed to build discovery ConfigMap"))] BuildDiscoveryConfigMap { source: super::discovery::Error }, } @@ -539,15 +539,18 @@ pub async fn reconcile_airflow( })?; } - // if the replicas are changed at the same time as the reconciliation + // If the replicas are changed at the same time as the reconciliation // being paused, it may be possible to have listeners that are *expected* // (according to their replica number) but which are not yet created, so - // deactivate this action in such cases. + // deactivate this action in such cases. If the cluster is stopping or + // scaling down (which may take a while depending on the graceful + // shutdown period), the discovery configmap will be empty, but the + // listeners will exist (and endpoints reachable) until the Pod is gone. if airflow.spec.cluster_operation.reconciliation_paused || airflow.spec.cluster_operation.stopped { tracing::info!( - "Cluster is in a transitional state so do not attempt to collect listener information that will only be active once cluster has returned to a non-transitional state." + "Cluster is in a transitional state (either the cluster or its reconciliation has been stopped) so do not attempt to collect listener information that will only be active once cluster has returned to a non-transitional state." ); } else { listener_refs.insert( @@ -982,7 +985,8 @@ fn build_server_rolegroup_statefulset( let listener_class = &merged_airflow_config.listener_class; // all listeners will use ephemeral volumes as they can/should - // be removed when the pods is re-started, and no data needs to be preserved + // be removed when the pods are *terminated* (ephemeral PVCs will + // survive re-starts) pb.add_listener_volume_by_listener_class( LISTENER_VOLUME_NAME, &listener_class.to_string(), From 1003da25d1cb03b8692abc65234d21bbf37c5cfc Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 22 Apr 2025 11:34:25 +0200 Subject: [PATCH 20/39] check for port lower-bound --- rust/operator-binary/src/crd/utils.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rust/operator-binary/src/crd/utils.rs b/rust/operator-binary/src/crd/utils.rs index a54cd0bf..2a489f2c 100644 --- a/rust/operator-binary/src/crd/utils.rs +++ b/rust/operator-binary/src/crd/utils.rs @@ -22,7 +22,7 @@ pub enum Error { pod: ObjectRef, }, - #[snafu(display("port {port} ({port_name:?}) is out of bounds, must be within {range:?}", range = 0..=u16::MAX))] + #[snafu(display("port {port} ({port_name:?}) must be in-bounds and cannot be privileged: must be within {range:?}", range = 1024..=u16::MAX))] PortOutOfBounds { source: TryFromIntError, port_name: String, From dc803f41ae8f9e9b127cc5fb4d8e3c4350d2ddd1 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 24 Apr 2025 13:48:53 +0200 Subject: [PATCH 21/39] reworked to only offer listener for webserver role --- deploy/helm/airflow-operator/crds/crds.yaml | 674 +++++++++--------- .../operator-binary/src/airflow_controller.rs | 86 +-- rust/operator-binary/src/crd/mod.rs | 268 +++---- rust/operator-binary/src/crd/utils.rs | 104 --- rust/operator-binary/src/discovery.rs | 92 --- rust/operator-binary/src/main.rs | 1 - .../40-install-airflow-cluster.yaml.j2 | 4 - .../external-access/50-access-airflow.txt.j2 | 99 --- .../external-access/50-access-airflow.yaml | 6 - .../kuttl/external-access/50-assert.yaml | 11 - 10 files changed, 445 insertions(+), 900 deletions(-) delete mode 100644 rust/operator-binary/src/crd/utils.rs delete mode 100644 rust/operator-binary/src/discovery.rs delete mode 100644 tests/templates/kuttl/external-access/50-access-airflow.txt.j2 delete mode 100644 tests/templates/kuttl/external-access/50-access-airflow.yaml delete mode 100644 tests/templates/kuttl/external-access/50-assert.yaml diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index f0bb11a9..8056e240 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -78,14 +78,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. - enum: - - cluster-internal - - external-unstable - - external-stable - nullable: true - type: string logging: default: containers: {} @@ -303,14 +295,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. - enum: - - cluster-internal - - external-unstable - - external-stable - nullable: true - type: string logging: default: containers: {} @@ -922,14 +906,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. - enum: - - cluster-internal - - external-unstable - - external-stable - nullable: true - type: string logging: default: containers: {} @@ -1147,14 +1123,6 @@ spec: description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. - enum: - - cluster-internal - - external-unstable - - external-stable - nullable: true - type: string logging: default: containers: {} @@ -1319,170 +1287,191 @@ spec: config: default: {} properties: - affinity: + airflowConfig: default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + affinity: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + gracefulShutdownTimeout: null + logging: + containers: {} + enableVectorAgent: null + resources: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. - nullable: true - type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. - enum: - - cluster-internal - - external-unstable - - external-stable - nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container properties: - configMap: - description: ConfigMap containing the log configuration files + console: + description: Configuration for the console appender nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean - type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. nullable: true - type: string + type: boolean type: object - memory: + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: type: object - type: object - storage: type: object type: object + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string type: object configOverrides: additionalProperties: @@ -1544,170 +1533,191 @@ spec: config: default: {} properties: - affinity: + airflowConfig: default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + affinity: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + gracefulShutdownTimeout: null + logging: + containers: {} + enableVectorAgent: null + resources: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true + affinity: + default: + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). + properties: + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. - nullable: true - type: string - listenerClass: - description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the airflow services. - enum: - - cluster-internal - - external-unstable - - external-stable - nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap + type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container properties: - configMap: - description: ConfigMap containing the log configuration files + console: + description: Configuration for the console appender nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender nullable: true - type: string + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean - type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. nullable: true - type: string + type: boolean type: object - memory: + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: type: object - type: object - storage: type: object type: object + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string type: object configOverrides: additionalProperties: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 90d34113..0ade7265 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -88,10 +88,8 @@ use crate::{ authorization::AirflowAuthorizationResolved, build_recommended_labels, git_sync::{GIT_SYNC_CONTENT, GIT_SYNC_NAME, GIT_SYNC_ROOT, GitSync}, - utils::PodRef, v1alpha1, }, - discovery::build_discovery_configmap, env_vars::{ self, build_airflow_template_envs, build_gitsync_statefulset_envs, build_gitsync_template, }, @@ -334,17 +332,6 @@ pub enum Error { LabelBuild { source: stackable_operator::kvp::LabelError, }, - - #[snafu(display("cannot collect discovery configuration"))] - CollectDiscoveryConfig { source: crate::crd::Error }, - - #[snafu(display("failed to apply discovery ConfigMap"))] - ApplyDiscoveryConfigMap { - source: stackable_operator::cluster_resources::Error, - }, - - #[snafu(display("failed to build discovery ConfigMap"))] - BuildDiscoveryConfigMap { source: super::discovery::Error }, } type Result = std::result::Result; @@ -468,8 +455,6 @@ pub async fn reconcile_airflow( .await?; } - let mut listener_refs: BTreeMap> = BTreeMap::new(); - for (role_name, role_config) in validated_role_config.iter() { let airflow_role = AirflowRole::from_str(role_name).context(UnidentifiedAirflowRoleSnafu { @@ -539,29 +524,6 @@ pub async fn reconcile_airflow( })?; } - // If the replicas are changed at the same time as the reconciliation - // being paused, it may be possible to have listeners that are *expected* - // (according to their replica number) but which are not yet created, so - // deactivate this action in such cases. If the cluster is stopping or - // scaling down (which may take a while depending on the graceful - // shutdown period), the discovery configmap will be empty, but the - // listeners will exist (and endpoints reachable) until the Pod is gone. - if airflow.spec.cluster_operation.reconciliation_paused - || airflow.spec.cluster_operation.stopped - { - tracing::info!( - "Cluster is in a transitional state (either the cluster or its reconciliation has been stopped) so do not attempt to collect listener information that will only be active once cluster has returned to a non-transitional state." - ); - } else { - listener_refs.insert( - airflow_role.to_string(), - airflow - .listener_refs(client, &airflow_role) - .await - .context(CollectDiscoveryConfigSnafu)?, - ); - } - let role_config = airflow.role_config(&airflow_role); if let Some(GenericRoleConfig { pod_disruption_budget: pdb, @@ -573,25 +535,6 @@ pub async fn reconcile_airflow( } } - tracing::debug!( - "Listener references prepared for the ConfigMap {:#?}", - listener_refs - ); - - // if paused or stopped the podrefs have not been collected (see comment - // above): the config map should remain unchanged if paused, but the - // refs removed from it if the cluster is stopped as replicas will - // have been set to 0. - if !airflow.spec.cluster_operation.reconciliation_paused { - let endpoint_cm = - build_discovery_configmap(airflow, &resolved_product_image, &listener_refs) - .context(BuildDiscoveryConfigMapSnafu)?; - cluster_resources - .add(client, endpoint_cm) - .await - .context(ApplyDiscoveryConfigMapSnafu)?; - } - cluster_resources .delete_orphaned_resources(client) .await @@ -983,20 +926,23 @@ fn build_server_rolegroup_statefulset( airflow_container.add_container_port(HTTP_PORT_NAME, http_port.into()); } - let listener_class = &merged_airflow_config.listener_class; - // all listeners will use ephemeral volumes as they can/should - // be removed when the pods are *terminated* (ephemeral PVCs will - // survive re-starts) - pb.add_listener_volume_by_listener_class( - LISTENER_VOLUME_NAME, - &listener_class.to_string(), - &recommended_labels, - ) - .context(AddVolumeSnafu)?; + if let Some(listener_class) = + airflow.merged_listener_class(airflow_role, &rolegroup_ref.role_group) + { + // all listeners will use ephemeral volumes as they can/should + // be removed when the pods are *terminated* (ephemeral PVCs will + // survive re-starts) + pb.add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &listener_class.to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; - airflow_container - .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) - .context(AddVolumeMountSnafu)?; + airflow_container + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)?; + } pb.add_container(airflow_container.build()); diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index f548d263..087bf8b5 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,4 +1,4 @@ -use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::collections::{BTreeMap, BTreeSet}; use product_config::flask_app_config_writer::{FlaskAppConfigOptions, PythonType}; use serde::{Deserialize, Serialize}; @@ -23,7 +23,7 @@ use stackable_operator::{ api::core::v1::{Volume, VolumeMount}, apimachinery::pkg::api::resource::Quantity, }, - kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, + kube::{CustomResource, ResourceExt}, kvp::ObjectLabels, memory::{BinaryMultiple, MemoryQuantity}, product_config_utils::{self, Configuration}, @@ -34,7 +34,7 @@ use stackable_operator::{ }, role_utils::{ CommonConfiguration, GenericProductSpecificCommonConfig, GenericRoleConfig, Role, - RoleGroupRef, + RoleGroup, RoleGroupRef, }, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, @@ -43,7 +43,6 @@ use stackable_operator::{ versioned::versioned, }; use strum::{Display, EnumIter, EnumString, IntoEnumIterator}; -use utils::{PodRef, get_listener_podrefs}; use crate::crd::{ affinity::{get_affinity, get_executor_affinity}, @@ -58,7 +57,6 @@ pub mod affinity; pub mod authentication; pub mod authorization; pub mod git_sync; -pub mod utils; pub const AIRFLOW_UID: i64 = 1000; pub const APP_NAME: &str = "airflow"; @@ -103,9 +101,6 @@ pub enum Error { #[snafu(display("object has no associated namespace"))] NoNamespace, - - #[snafu(display("listener podrefs could not be resolved"))] - ListenerPodRef { source: utils::Error }, } #[derive(Display, EnumIter, EnumString)] @@ -205,7 +200,7 @@ pub mod versioned { /// The `webserver` role provides the main UI for user interaction. #[serde(default, skip_serializing_if = "Option::is_none")] - pub webservers: Option>, + pub webservers: Option>, /// The `scheduler` is responsible for triggering jobs and persisting their metadata to the backend database. /// Jobs are scheduled on the workers/executors. @@ -288,7 +283,14 @@ impl v1alpha1::AirflowCluster { /// the kubernetes executor is specified) pub fn get_role(&self, role: &AirflowRole) -> Option> { match role { - AirflowRole::Webserver => self.spec.webservers.to_owned(), + AirflowRole::Webserver => { + if let Some(webserver_config) = self.spec.webservers.to_owned() { + let role = extract_role_from_webserver_config(webserver_config); + Some(role) + } else { + None + } + } AirflowRole::Scheduler => self.spec.schedulers.to_owned(), AirflowRole::Worker => { if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { @@ -349,13 +351,12 @@ impl v1alpha1::AirflowCluster { let role = match role { AirflowRole::Webserver => { - self.spec - .webservers - .as_ref() - .context(UnknownAirflowRoleSnafu { + &extract_role_from_webserver_config(self.spec.webservers.to_owned().context( + UnknownAirflowRoleSnafu { role: role.to_string(), roles: AirflowRole::roles(), - })? + }, + )?) } AirflowRole::Worker => { if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { @@ -401,47 +402,28 @@ impl v1alpha1::AirflowCluster { &self, role: &AirflowRole, rolegroup_name: &String, - ) -> Result, Error> { - let listener_class_default = Some(SupportedListenerClasses::ClusterInternal); - - let role = match role { - AirflowRole::Webserver => { - self.spec - .webservers - .as_ref() - .context(UnknownAirflowRoleSnafu { - role: role.to_string(), - roles: AirflowRole::roles(), - })? + ) -> Option { + if role == &AirflowRole::Webserver { + if let Some(webservers) = self.spec.webservers.as_ref() { + let conf_defaults = Some(SupportedListenerClasses::ClusterInternal); + let mut conf_role = webservers.config.config.listener_class.to_owned(); + let mut conf_rolegroup = webservers + .role_groups + .get(rolegroup_name) + .map(|rg| rg.config.config.listener_class.clone()) + .unwrap_or_default(); + + conf_role.merge(&conf_defaults); + conf_rolegroup.merge(&conf_role); + + tracing::debug!("Merged listener-class: {:?}", conf_rolegroup); + conf_rolegroup + } else { + None } - AirflowRole::Worker => { - if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { - config - } else { - return Err(Error::NoRoleForExecutorFailure); - } - } - AirflowRole::Scheduler => { - self.spec - .schedulers - .as_ref() - .context(UnknownAirflowRoleSnafu { - role: role.to_string(), - roles: AirflowRole::roles(), - })? - } - }; - - let mut listener_class_role = role.config.config.listener_class.to_owned(); - let mut listener_class_rolegroup = role - .role_groups - .get(rolegroup_name) - .map(|rg| rg.config.config.listener_class.clone()) - .unwrap_or_default(); - listener_class_role.merge(&listener_class_default); - listener_class_rolegroup.merge(&listener_class_role); - tracing::debug!("Merged listener-class: {:?}", listener_class_rolegroup); - Ok(listener_class_rolegroup) + } else { + None + } } /// Retrieve and merge resource configs for the executor template @@ -468,133 +450,38 @@ impl v1alpha1::AirflowCluster { tracing::debug!("Merged executor config: {:?}", conf_executor); fragment::validate(conf_executor).context(FragmentValidationFailureSnafu) } +} - pub fn rolegroup_ref( - &self, - role_name: impl Into, - group_name: impl Into, - ) -> RoleGroupRef { - RoleGroupRef { - cluster: ObjectRef::from_obj(self), - role: role_name.into(), - role_group: group_name.into(), - } - } - - pub fn rolegroup_ref_and_replicas( - &self, - role: &AirflowRole, - ) -> Vec<(RoleGroupRef, u16)> { - match role { - AirflowRole::Webserver => self - .spec - .webservers - .iter() - .flat_map(|role| &role.role_groups) - // Order rolegroups consistently, to avoid spurious downstream rewrites - .collect::>() - .into_iter() - .filter(|(rolegroup_name, _)| { - self.resolved_listener_class_discoverable(role, rolegroup_name) - }) - .map(|(rolegroup_name, role_group)| { - ( - self.rolegroup_ref(role.to_string(), rolegroup_name), - role_group.replicas.unwrap_or_default(), - ) - }) - .collect(), - AirflowRole::Scheduler => self - .spec - .schedulers - .iter() - .flat_map(|role| &role.role_groups) - // Order rolegroups consistently, to avoid spurious downstream rewrites - .collect::>() - .into_iter() - .filter(|(rolegroup_name, _)| { - self.resolved_listener_class_discoverable(role, rolegroup_name) - }) - .map(|(rolegroup_name, role_group)| { - ( - self.rolegroup_ref(role.to_string(), rolegroup_name), - role_group.replicas.unwrap_or_default(), - ) - }) - .collect(), - AirflowRole::Worker => { - if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { - config - .role_groups - .iter() - // Order rolegroups consistently, to avoid spurious downstream rewrites - .collect::>() - .into_iter() - .filter(|(rolegroup_name, _)| { - self.resolved_listener_class_discoverable(role, rolegroup_name) - }) - .map(|(rolegroup_name, role_group)| { - ( - self.rolegroup_ref(role.to_string(), rolegroup_name), - role_group.replicas.unwrap_or_default(), - ) - }) - .collect() - } else { - vec![] - } - } - } - } - - fn resolved_listener_class_discoverable( - &self, - role: &AirflowRole, - rolegroup_name: &&String, - ) -> bool { - if let Ok(Some(listener_class)) = self.merged_listener_class(role, rolegroup_name) { - listener_class.discoverable() - } else { - // merged_listener_class returns an error if one of the roles was not found: - // all roles are mandatory for airflow to work, but a missing role will by - // definition not have a listener class - false - } - } - - pub fn pod_refs(&self, role: &AirflowRole) -> Result, Error> { - let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; - let rolegroup_ref_and_replicas = self.rolegroup_ref_and_replicas(role); - - Ok(rolegroup_ref_and_replicas - .iter() - .flat_map(|(rolegroup_ref, replicas)| { - let ns = ns.clone(); - (0..*replicas).map(move |i| PodRef { - namespace: ns.clone(), - role_group_service_name: rolegroup_ref.object_name(), - pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), - ports: HashMap::from([ - (HTTP_PORT_NAME.to_owned(), HTTP_PORT), - (METRICS_PORT_NAME.to_owned(), METRICS_PORT), - ]), - fqdn_override: None, +fn extract_role_from_webserver_config( + fragment: Role, +) -> Role { + Role { + config: CommonConfiguration { + config: fragment.config.config.airflow_config, + config_overrides: fragment.config.config_overrides, + env_overrides: fragment.config.env_overrides, + cli_overrides: fragment.config.cli_overrides, + pod_overrides: fragment.config.pod_overrides, + product_specific_common_config: fragment.config.product_specific_common_config, + }, + role_config: fragment.role_config, + role_groups: fragment + .role_groups + .into_iter() + .map(|(k, v)| { + (k, RoleGroup { + config: CommonConfiguration { + config: v.config.config.airflow_config, + config_overrides: v.config.config_overrides, + env_overrides: v.config.env_overrides, + cli_overrides: v.config.cli_overrides, + pod_overrides: v.config.pod_overrides, + product_specific_common_config: v.config.product_specific_common_config, + }, + replicas: v.replicas, }) }) - .collect()) - } - - pub async fn listener_refs( - &self, - client: &stackable_operator::client::Client, - role: &AirflowRole, - ) -> Result, Error> { - let pod_refs = self.pod_refs(role)?; - - tracing::debug!("Pod references for role {role}: {:#?}", pod_refs); - get_listener_podrefs(client, pod_refs, LISTENER_VOLUME_NAME) - .await - .context(ListenerPodRefSnafu) + .collect(), } } @@ -922,8 +809,28 @@ pub struct AirflowConfig { /// Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. #[fragment_attrs(serde(default))] pub graceful_shutdown_timeout: Option, +} + +#[derive(Clone, Debug, Default, Fragment, JsonSchema, PartialEq)] +#[fragment_attrs( + derive( + Clone, + Debug, + Default, + Deserialize, + Merge, + JsonSchema, + PartialEq, + Serialize + ), + serde(rename_all = "camelCase") +)] +pub struct WebserverConfig { + #[fragment_attrs(serde(default))] + #[serde(flatten)] + pub airflow_config: AirflowConfig, - /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the airflow services. + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. #[serde(default)] pub listener_class: SupportedListenerClasses, } @@ -943,7 +850,6 @@ impl AirflowConfig { } AirflowRole::Worker => DEFAULT_WORKER_GRACEFUL_SHUTDOWN_TIMEOUT, }), - listener_class: Some(SupportedListenerClasses::ClusterInternal), } } } diff --git a/rust/operator-binary/src/crd/utils.rs b/rust/operator-binary/src/crd/utils.rs deleted file mode 100644 index 2a489f2c..00000000 --- a/rust/operator-binary/src/crd/utils.rs +++ /dev/null @@ -1,104 +0,0 @@ -use std::{borrow::Cow, collections::HashMap, num::TryFromIntError}; - -use futures::future::try_join_all; -use snafu::{OptionExt, ResultExt, Snafu}; -use stackable_operator::{ - commons::listener::Listener, k8s_openapi::api::core::v1::Pod, - kube::runtime::reflector::ObjectRef, utils::cluster_info::KubernetesClusterInfo, -}; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("unable to get {listener} (for {pod})"))] - GetPodListener { - source: stackable_operator::client::Error, - listener: ObjectRef, - pod: ObjectRef, - }, - - #[snafu(display("{listener} (for {pod}) has no address"))] - PodListenerHasNoAddress { - listener: ObjectRef, - pod: ObjectRef, - }, - - #[snafu(display("port {port} ({port_name:?}) must be in-bounds and cannot be privileged: must be within {range:?}", range = 1024..=u16::MAX))] - PortOutOfBounds { - source: TryFromIntError, - port_name: String, - port: i32, - }, -} - -/// Reference to a single `Pod` that is a component of the product cluster -/// -/// Used for service discovery. -#[derive(Debug)] -pub struct PodRef { - pub namespace: String, - pub role_group_service_name: String, - pub pod_name: String, - pub fqdn_override: Option, - pub ports: HashMap, -} - -impl PodRef { - pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Cow { - self.fqdn_override.as_deref().map_or_else( - || { - Cow::Owned(format!( - "{pod_name}.{role_group_service_name}.{namespace}.svc.{cluster_domain}", - pod_name = self.pod_name, - role_group_service_name = self.role_group_service_name, - namespace = self.namespace, - cluster_domain = cluster_info.cluster_domain, - )) - }, - Cow::Borrowed, - ) - } -} - -pub async fn get_listener_podrefs( - client: &stackable_operator::client::Client, - pod_refs: Vec, - listener_prefix: &str, -) -> Result, Error> { - try_join_all(pod_refs.into_iter().map(|pod_ref| async { - // N.B. use the naming convention for ephemeral listener volumes as we - // have defined all listeners to be so. - let listener_name = format!("{}-{listener_prefix}", pod_ref.pod_name); - let listener_ref = || ObjectRef::::new(&listener_name).within(&pod_ref.namespace); - let pod_obj_ref = || ObjectRef::::new(&pod_ref.pod_name).within(&pod_ref.namespace); - let listener = client - .get::(&listener_name, &pod_ref.namespace) - .await - .context(GetPodListenerSnafu { - listener: listener_ref(), - pod: pod_obj_ref(), - })?; - let listener_address = listener - .status - .and_then(|s| s.ingress_addresses?.into_iter().next()) - .context(PodListenerHasNoAddressSnafu { - listener: listener_ref(), - pod: pod_obj_ref(), - })?; - Ok(PodRef { - fqdn_override: Some(listener_address.address), - ports: listener_address - .ports - .into_iter() - .map(|(port_name, port)| { - let port = u16::try_from(port).context(PortOutOfBoundsSnafu { - port_name: &port_name, - port, - })?; - Ok((port_name, port)) - }) - .collect::>()?, - ..pod_ref - }) - })) - .await -} diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs deleted file mode 100644 index a2943c0f..00000000 --- a/rust/operator-binary/src/discovery.rs +++ /dev/null @@ -1,92 +0,0 @@ -use std::collections::BTreeMap; - -use snafu::{ResultExt, Snafu}; -use stackable_operator::{ - builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, - commons::product_image_selection::ResolvedProductImage, - k8s_openapi::api::core::v1::ConfigMap, - kube::runtime::reflector::ObjectRef, -}; - -use crate::{ - airflow_controller::AIRFLOW_CONTROLLER_NAME, - crd::{ - AirflowRole, HTTP_PORT_NAME, METRICS_PORT_NAME, build_recommended_labels, utils::PodRef, - v1alpha1, - }, -}; - -type Result = std::result::Result; - -#[derive(Snafu, Debug)] -pub enum Error { - #[snafu(display("object {airflow} is missing metadata to build owner reference"))] - ObjectMissingMetadataForOwnerRef { - source: stackable_operator::builder::meta::Error, - airflow: ObjectRef, - }, - - #[snafu(display("failed to build ConfigMap"))] - BuildConfigMap { - source: stackable_operator::builder::configmap::Error, - }, - - #[snafu(display("failed to build object meta data"))] - ObjectMeta { - source: stackable_operator::builder::meta::Error, - }, -} - -/// Creates a discovery config map containing the webserver endpoint for clients. -pub fn build_discovery_configmap( - airflow: &v1alpha1::AirflowCluster, - resolved_product_image: &ResolvedProductImage, - role_podrefs: &BTreeMap>, -) -> Result { - let mut cm = ConfigMapBuilder::new(); - - let cmm = cm.metadata( - ObjectMetaBuilder::new() - .name_and_namespace(airflow) - .ownerreference_from_resource(airflow, None, Some(true)) - .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { - airflow: ObjectRef::from_obj(airflow), - })? - .with_recommended_labels(build_recommended_labels( - airflow, - AIRFLOW_CONTROLLER_NAME, - &resolved_product_image.app_version_label, - &AirflowRole::Webserver.to_string(), - "discovery", - )) - .context(ObjectMetaSnafu)? - .build(), - ); - - for role_podref in role_podrefs { - for podref in role_podref.1 { - if let PodRef { - fqdn_override: Some(fqdn_override), - ports, - pod_name, - .. - } = podref - { - if let Some(ui_port) = ports.get(HTTP_PORT_NAME) { - cmm.add_data( - format!("{pod_name}.{HTTP_PORT_NAME}"), - format!("{fqdn_override}:{ui_port}"), - ); - } - if let Some(metrics_port) = ports.get(METRICS_PORT_NAME) { - cmm.add_data( - format!("{pod_name}.{METRICS_PORT_NAME}"), - format!("{fqdn_override}:{metrics_port}"), - ); - } - } - } - } - - cm.build().context(BuildConfigMapSnafu) -} diff --git a/rust/operator-binary/src/main.rs b/rust/operator-binary/src/main.rs index b8683e52..cbb42e79 100644 --- a/rust/operator-binary/src/main.rs +++ b/rust/operator-binary/src/main.rs @@ -34,7 +34,6 @@ mod airflow_controller; mod config; mod controller_commons; mod crd; -mod discovery; mod env_vars; mod operations; mod product_logging; diff --git a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 index 8d5af32b..f6a9e614 100644 --- a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 @@ -51,14 +51,10 @@ spec: config: listenerClass: cluster-internal celeryExecutors: - config: - listenerClass: external-stable roleGroups: default: replicas: 2 schedulers: - config: - listenerClass: external-unstable roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 b/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 deleted file mode 100644 index f53a30b8..00000000 --- a/tests/templates/kuttl/external-access/50-access-airflow.txt.j2 +++ /dev/null @@ -1,99 +0,0 @@ ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: access-airflow -spec: - template: - spec: - serviceAccountName: test-sa - containers: - - name: access-airflow -{% if test_scenario['values']['airflow'].find(",") > 0 %} - image: "{{ test_scenario['values']['airflow'].split(',')[1] }}" -{% else %} - image: oci.stackable.tech/sdp/airflow:{{ test_scenario['values']['airflow'] }}-stackable0.0.0-dev -{% endif %} - imagePullPolicy: IfNotPresent - command: - - /bin/bash - - /tmp/script/script.sh - env: - - name: SCHEDULER_METRICS - valueFrom: - configMapKeyRef: - name: airflow - key: airflow-scheduler-default-0.metrics - - name: WORKER_0_METRICS - valueFrom: - configMapKeyRef: - name: airflow - key: airflow-worker-default-0.metrics - - name: WORKER_1_METRICS - valueFrom: - configMapKeyRef: - name: airflow - key: airflow-worker-default-1.metrics - - name: WEBSERVER_DEFAULT_HTTP - valueFrom: - configMapKeyRef: - name: airflow - key: airflow-webserver-default-0.http - - name: WEBSERVER_DEFAULT_METRICS - valueFrom: - configMapKeyRef: - name: airflow - key: airflow-webserver-default-0.metrics - - name: WEBSERVER_EXTERNAL_HTTP - valueFrom: - configMapKeyRef: - name: airflow - key: airflow-webserver-external-unstable-0.http - - name: WEBSERVER_EXTERNAL_METRICS - valueFrom: - configMapKeyRef: - name: airflow - key: airflow-webserver-external-unstable-0.metrics - volumeMounts: - - name: script - mountPath: /tmp/script - volumes: - - name: script - configMap: - name: access-airflow-script - securityContext: - fsGroup: 1000 - runAsGroup: 1000 - runAsUser: 1000 - restartPolicy: OnFailure ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: access-airflow-script -data: - script.sh: | - set -euxo pipefail - - echo "Attempting to reach scheduler at $SCHEDULER_METRICS..." - curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${SCHEDULER_METRICS}" | grep 200 - - echo "Attempting to reach worker at $WORKER_0_METRICS..." - curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WORKER_0_METRICS}" | grep 200 - - echo "Attempting to reach worker at $WORKER_1_METRICS..." - curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WORKER_1_METRICS}" | grep 200 - - echo "Attempting to reach webserver at $WEBSERVER_DEFAULT_HTTP..." - curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_DEFAULT_HTTP}" | grep 200 - - echo "Attempting to reach webserver at $WEBSERVER_DEFAULT_METRICS..." - curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_DEFAULT_METRICS}" | grep 200 - - echo "Attempting to reach webserver at $WEBSERVER_EXTERNAL_HTTP..." - curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_EXTERNAL_HTTP}" | grep 200 - - echo "Attempting to reach webserver at $WEBSERVER_EXTERNAL_METRICS..." - curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${WEBSERVER_EXTERNAL_METRICS}" | grep 200 - - echo "All tests successful!" diff --git a/tests/templates/kuttl/external-access/50-access-airflow.yaml b/tests/templates/kuttl/external-access/50-access-airflow.yaml deleted file mode 100644 index f1ea0f7f..00000000 --- a/tests/templates/kuttl/external-access/50-access-airflow.yaml +++ /dev/null @@ -1,6 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -commands: - # We need to replace $NAMESPACE (by KUTTL) - - script: envsubst '$NAMESPACE' < 50-access-airflow.txt | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/50-assert.yaml b/tests/templates/kuttl/external-access/50-assert.yaml deleted file mode 100644 index 5e7d2c20..00000000 --- a/tests/templates/kuttl/external-access/50-assert.yaml +++ /dev/null @@ -1,11 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -timeout: 600 ---- -apiVersion: batch/v1 -kind: Job -metadata: - name: access-airflow -status: - succeeded: 1 From a4ec80142c23546d2e1ad4da8791be0c02da0340 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Thu, 24 Apr 2025 13:55:16 +0200 Subject: [PATCH 22/39] reworked docs --- .../pages/usage-guide/listenerclass.adoc | 28 ++++--------------- .../operator-binary/src/airflow_controller.rs | 12 ++------ 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc index 34758f3f..7ff05af8 100644 --- a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc @@ -1,8 +1,8 @@ = Service exposition with ListenerClasses :description: Configure Airflow service exposure with ListenerClasses: cluster-internal, external-unstable, or external-stable. -The operator deploys a xref:listener-operator:listener.adoc[Listener] for each Scheduler, Webserver and (Celery-)Worker pod. -They all default to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.{schedulers,celeryExecutors,webservers}.config.listenerClass`: +The operator deploys a xref:listener-operator:listener.adoc[Listener] for the Webserver pod. +The listener defaults to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.webservers.config.listenerClass`: [source,yaml] ---- @@ -11,27 +11,9 @@ spec: config: listenerClass: external-unstable # <1> webservers: - config: - listenerClass: external-unstable + ... celeryExecutors: - config: - listenerClass: external-unstable + ... ---- <1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). -This can be set separately for all three roles (but not for kuberneetesExecutors as the resulting worker pods are temporary). - -Externally-reachable endpoints (i.e. where listener-class = `external-unstable` or `external-unstable`) are written to a ConfigMap called ``, listing each rolegroup by replica. -Both the http and metrics ports are exposed: - -[source,yaml] ----- -apiVersion: v1 -data: - airflow-scheduler-default-0.metrics: 172.19.0.5:30589 - airflow-webserver-default-0.http: 172.19.0.3:31891 - airflow-webserver-default-0.metrics: 172.19.0.3:30414 - airflow-worker-default-0.metrics: 172.19.0.5:30371 - airflow-worker-default-1.metrics: 172.19.0.3:30514 -kind: ConfigMap -... ----- +This can be set only for the webservers role. diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 0ade7265..4ec239a7 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -19,11 +19,8 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, - container::ContainerBuilder, - resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, - volume::{ListenerOperatorVolumeSourceBuilderError, VolumeBuilder}, + PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, volume::VolumeBuilder, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -323,11 +320,6 @@ pub enum Error { #[snafu(display("failed to build Statefulset environmental variables"))] BuildStatefulsetEnvVars { source: env_vars::Error }, - #[snafu(display("failed to build listener volume"))] - BuildListenerVolume { - source: ListenerOperatorVolumeSourceBuilderError, - }, - #[snafu(display("failed to build Labels"))] LabelBuild { source: stackable_operator::kvp::LabelError, From 063bded268cd7f9be76ba82111b1111df8372c21 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:42:51 +0200 Subject: [PATCH 23/39] Update docs/modules/airflow/pages/usage-guide/listenerclass.adoc Co-authored-by: Malte Sander --- docs/modules/airflow/pages/usage-guide/listenerclass.adoc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc index 7ff05af8..fe803174 100644 --- a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc @@ -7,10 +7,10 @@ The listener defaults to only being accessible from within the Kubernetes cluste [source,yaml] ---- spec: - schedulers: + webservers: config: listenerClass: external-unstable # <1> - webservers: + schedulers: ... celeryExecutors: ... From 25ce16096878abe9aededc4be3759c88921d3401 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Thu, 24 Apr 2025 15:43:30 +0200 Subject: [PATCH 24/39] Update rust/operator-binary/src/crd/mod.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/crd/mod.rs | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 087bf8b5..27402912 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -283,14 +283,11 @@ impl v1alpha1::AirflowCluster { /// the kubernetes executor is specified) pub fn get_role(&self, role: &AirflowRole) -> Option> { match role { - AirflowRole::Webserver => { - if let Some(webserver_config) = self.spec.webservers.to_owned() { - let role = extract_role_from_webserver_config(webserver_config); - Some(role) - } else { - None - } - } + AirflowRole::Webserver => self + .spec + .webservers + .to_owned() + .map(extract_role_from_webserver_config), AirflowRole::Scheduler => self.spec.schedulers.to_owned(), AirflowRole::Worker => { if let AirflowExecutor::CeleryExecutor { config } = &self.spec.executor { From 0095ad996045d9b12d3d7561eddda8d5cab4f8e5 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 25 Apr 2025 12:06:45 +0200 Subject: [PATCH 25/39] added service check to test --- .../kuttl/external-access/40-assert.yaml | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/templates/kuttl/external-access/40-assert.yaml b/tests/templates/kuttl/external-access/40-assert.yaml index dbee54fc..342ebabf 100644 --- a/tests/templates/kuttl/external-access/40-assert.yaml +++ b/tests/templates/kuttl/external-access/40-assert.yaml @@ -91,3 +91,24 @@ status: expectedPods: 1 currentHealthy: 1 disruptionsAllowed: 1 +--- +apiVersion: v1 +kind: Service +metadata: + name: airflow-webserver-cluster-internal-0-listener +spec: + type: ClusterIP # cluster-internal +--- +apiVersion: v1 +kind: Service +metadata: + name: airflow-webserver-default-0-listener +spec: + type: NodePort # external-stable +--- +apiVersion: v1 +kind: Service +metadata: + name: airflow-webserver-external-unstable-0-listener +spec: + type: NodePort # external-unstable From 7da421cfae45e064e1f7a8203d61f5a95cd5c8f8 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 25 Apr 2025 14:39:43 +0200 Subject: [PATCH 26/39] correctly flatten WebserverConfig --- deploy/helm/airflow-operator/crds/crds.yaml | 638 +++++++++----------- rust/operator-binary/src/crd/mod.rs | 3 +- 2 files changed, 299 insertions(+), 342 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 8056e240..979bc19b 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -1287,183 +1287,40 @@ spec: config: default: {} properties: - airflowConfig: + affinity: default: - affinity: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - gracefulShutdownTimeout: null - logging: - containers: {} - enableVectorAgent: null - resources: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: - affinity: - default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). - properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - type: object - memory: - properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. - type: object - type: object - storage: - type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. enum: @@ -1472,6 +1329,128 @@ spec: - external-stable nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object type: object configOverrides: additionalProperties: @@ -1533,183 +1512,40 @@ spec: config: default: {} properties: - airflowConfig: + affinity: default: - affinity: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - gracefulShutdownTimeout: null - logging: - containers: {} - enableVectorAgent: null - resources: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} + nodeAffinity: null + nodeSelector: null + podAffinity: null + podAntiAffinity: null + description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). properties: - affinity: - default: - nodeAffinity: null - nodeSelector: null - podAffinity: null - podAntiAffinity: null - description: These configuration settings control [Pod placement](https://docs.stackable.tech/home/nightly/concepts/operations/pod_placement). - properties: - nodeAffinity: - description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - nodeSelector: - additionalProperties: - type: string - description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - podAffinity: - description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true - podAntiAffinity: - description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) - nullable: true - type: object - x-kubernetes-preserve-unknown-fields: true + nodeAffinity: + description: Same as the `spec.affinity.nodeAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object - gracefulShutdownTimeout: - description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + x-kubernetes-preserve-unknown-fields: true + nodeSelector: + additionalProperties: + type: string + description: Simple key-value pairs forming a nodeSelector, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) nullable: true - type: string - logging: - default: - containers: {} - enableVectorAgent: null - description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). - properties: - containers: - additionalProperties: - anyOf: - - required: - - custom - - {} - description: Log configuration of the container - properties: - console: - description: Configuration for the console appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - custom: - description: Custom log configuration provided in a ConfigMap - properties: - configMap: - description: ConfigMap containing the log configuration files - nullable: true - type: string - type: object - file: - description: Configuration for the file appender - nullable: true - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - loggers: - additionalProperties: - description: Configuration of a logger - properties: - level: - description: The log level threshold. Log events with a lower log level are discarded. - enum: - - TRACE - - DEBUG - - INFO - - WARN - - ERROR - - FATAL - - NONE - nullable: true - type: string - type: object - default: {} - description: Configuration per logger - type: object - type: object - description: Log configuration per container. - type: object - enableVectorAgent: - description: Wether or not to deploy a container with the Vector log agent. - nullable: true - type: boolean type: object - resources: - default: - cpu: - max: null - min: null - memory: - limit: null - runtimeLimits: {} - storage: {} - description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. - properties: - cpu: - default: - max: null - min: null - properties: - max: - description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - min: - description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. - nullable: true - type: string - type: object - memory: - properties: - limit: - description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' - nullable: true - type: string - runtimeLimits: - description: Additional options that can be specified. - type: object - type: object - storage: - type: object + podAffinity: + description: Same as the `spec.affinity.podAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + podAntiAffinity: + description: Same as the `spec.affinity.podAntiAffinity` field on the Pod, see the [Kubernetes docs](https://kubernetes.io/docs/concepts/scheduling-eviction/assign-pod-node) + nullable: true type: object + x-kubernetes-preserve-unknown-fields: true type: object + gracefulShutdownTimeout: + description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details. + nullable: true + type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. enum: @@ -1718,6 +1554,128 @@ spec: - external-stable nullable: true type: string + logging: + default: + containers: {} + enableVectorAgent: null + description: Logging configuration, learn more in the [logging concept documentation](https://docs.stackable.tech/home/nightly/concepts/logging). + properties: + containers: + additionalProperties: + anyOf: + - required: + - custom + - {} + description: Log configuration of the container + properties: + console: + description: Configuration for the console appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + custom: + description: Custom log configuration provided in a ConfigMap + properties: + configMap: + description: ConfigMap containing the log configuration files + nullable: true + type: string + type: object + file: + description: Configuration for the file appender + nullable: true + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + loggers: + additionalProperties: + description: Configuration of a logger + properties: + level: + description: The log level threshold. Log events with a lower log level are discarded. + enum: + - TRACE + - DEBUG + - INFO + - WARN + - ERROR + - FATAL + - NONE + nullable: true + type: string + type: object + default: {} + description: Configuration per logger + type: object + type: object + description: Log configuration per container. + type: object + enableVectorAgent: + description: Wether or not to deploy a container with the Vector log agent. + nullable: true + type: boolean + type: object + resources: + default: + cpu: + max: null + min: null + memory: + limit: null + runtimeLimits: {} + storage: {} + description: Resource usage is configured here, this includes CPU usage, memory usage and disk storage usage, if this role needs any. + properties: + cpu: + default: + max: null + min: null + properties: + max: + description: The maximum amount of CPU cores that can be requested by Pods. Equivalent to the `limit` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + min: + description: The minimal amount of CPU cores that Pods need to run. Equivalent to the `request` for Pod resource configuration. Cores are specified either as a decimal point number or as milli units. For example:`1.5` will be 1.5 cores, also written as `1500m`. + nullable: true + type: string + type: object + memory: + properties: + limit: + description: 'The maximum amount of memory that should be available to the Pod. Specified as a byte [Quantity](https://kubernetes.io/docs/reference/kubernetes-api/common-definitions/quantity/), which means these suffixes are supported: E, P, T, G, M, k. You can also use the power-of-two equivalents: Ei, Pi, Ti, Gi, Mi, Ki. For example, the following represent roughly the same value: `128974848, 129e6, 129M, 128974848000m, 123Mi`' + nullable: true + type: string + runtimeLimits: + description: Additional options that can be specified. + type: object + type: object + storage: + type: object + type: object type: object configOverrides: additionalProperties: diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 27402912..c3afdc5d 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -823,8 +823,7 @@ pub struct AirflowConfig { serde(rename_all = "camelCase") )] pub struct WebserverConfig { - #[fragment_attrs(serde(default))] - #[serde(flatten)] + #[fragment_attrs(serde(default, flatten))] pub airflow_config: AirflowConfig, /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. From 7ac6077e53df8262e4b28a4b986af344843a85c8 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Fri, 25 Apr 2025 17:41:43 +0200 Subject: [PATCH 27/39] use pvcs for externally reachable endpoints --- .../operator-binary/src/airflow_controller.rs | 53 ++++++++++++++----- .../kuttl/external-access/40-assert.yaml | 4 +- 2 files changed, 41 insertions(+), 16 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 4ec239a7..65b3f26a 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -19,8 +19,14 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, - security::PodSecurityContextBuilder, volume::VolumeBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, + security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, VolumeBuilder, + }, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -35,8 +41,9 @@ use stackable_operator::{ api::{ apps::v1::{StatefulSet, StatefulSetSpec}, core::v1::{ - ConfigMap, EmptyDirVolumeSource, EnvVar, PodTemplateSpec, Probe, Service, - ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, VolumeMount, + ConfigMap, EmptyDirVolumeSource, EnvVar, PersistentVolumeClaim, PodTemplateSpec, + Probe, Service, ServiceAccount, ServicePort, ServiceSpec, TCPSocketAction, + VolumeMount, }, }, apimachinery::pkg::{apis::meta::v1::LabelSelector, util::intstr::IntOrString}, @@ -324,6 +331,11 @@ pub enum Error { LabelBuild { source: stackable_operator::kvp::LabelError, }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, } type Result = std::result::Result; @@ -918,19 +930,31 @@ fn build_server_rolegroup_statefulset( airflow_container.add_container_port(HTTP_PORT_NAME, http_port.into()); } + let mut pvcs: Option> = None; + if let Some(listener_class) = airflow.merged_listener_class(airflow_role, &rolegroup_ref.role_group) { - // all listeners will use ephemeral volumes as they can/should - // be removed when the pods are *terminated* (ephemeral PVCs will - // survive re-starts) - pb.add_listener_volume_by_listener_class( - LISTENER_VOLUME_NAME, - &listener_class.to_string(), - &recommended_labels, - ) - .context(AddVolumeSnafu)?; - + if listener_class.discoverable() { + // externally reachable listener endpoints will use persistent volumes + // so that load balancers can hard-code the target addresses + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass(listener_class.to_string()), + &recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + pvcs = Some(vec![pvc]); + } else { + // non-reachable endpoints use ephemeral volumes + pb.add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &listener_class.to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; + } airflow_container .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) .context(AddVolumeMountSnafu)?; @@ -1088,6 +1112,7 @@ fn build_server_rolegroup_statefulset( }, service_name: rolegroup_ref.object_name(), template: pod_template, + volume_claim_templates: pvcs, ..StatefulSetSpec::default() }; diff --git a/tests/templates/kuttl/external-access/40-assert.yaml b/tests/templates/kuttl/external-access/40-assert.yaml index 342ebabf..4a61ca08 100644 --- a/tests/templates/kuttl/external-access/40-assert.yaml +++ b/tests/templates/kuttl/external-access/40-assert.yaml @@ -102,13 +102,13 @@ spec: apiVersion: v1 kind: Service metadata: - name: airflow-webserver-default-0-listener + name: listener-airflow-webserver-default-0 spec: type: NodePort # external-stable --- apiVersion: v1 kind: Service metadata: - name: airflow-webserver-external-unstable-0-listener + name: listener-airflow-webserver-external-unstable-0 spec: type: NodePort # external-unstable From 0f650d311314ecad5ca8669418903c7304cfc0be Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 28 Apr 2025 10:47:35 +0200 Subject: [PATCH 28/39] use group listener for webservers --- .../airflow-operator/templates/roles.yaml | 5 ++ .../operator-binary/src/airflow_controller.rs | 89 +++++++++++++++++-- rust/operator-binary/src/crd/mod.rs | 7 ++ .../kuttl/external-access/40-assert.yaml | 12 +-- .../40-install-airflow-cluster.yaml.j2 | 2 +- 5 files changed, 103 insertions(+), 12 deletions(-) diff --git a/deploy/helm/airflow-operator/templates/roles.yaml b/deploy/helm/airflow-operator/templates/roles.yaml index 119590e3..dbe39470 100644 --- a/deploy/helm/airflow-operator/templates/roles.yaml +++ b/deploy/helm/airflow-operator/templates/roles.yaml @@ -90,6 +90,11 @@ rules: - listeners verbs: - get + - list + - watch + - patch + - create + - delete - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 65b3f26a..0e6d1446 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -32,6 +32,7 @@ use stackable_operator::{ cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, commons::{ authentication::{AuthenticationClass, ldap}, + listener::{Listener, ListenerPort, ListenerSpec}, product_image_selection::ResolvedProductImage, rbac::build_rbac_resources, }, @@ -83,9 +84,10 @@ use crate::{ crd::{ self, AIRFLOW_CONFIG_FILENAME, AIRFLOW_UID, APP_NAME, AirflowClusterStatus, AirflowConfig, AirflowConfigOptions, AirflowExecutor, AirflowRole, CONFIG_PATH, Container, ExecutorConfig, - ExecutorConfigFragment, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, - LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, STACKABLE_LOG_DIR, - TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, TEMPLATE_VOLUME_NAME, + ExecutorConfigFragment, HTTP_PORT, HTTP_PORT_NAME, LISTENER_VOLUME_DIR, + LISTENER_VOLUME_NAME, LOG_CONFIG_DIR, METRICS_PORT, METRICS_PORT_NAME, OPERATOR_NAME, + STACKABLE_LOG_DIR, TEMPLATE_CONFIGMAP_NAME, TEMPLATE_LOCATION, TEMPLATE_NAME, + TEMPLATE_VOLUME_NAME, authentication::{ AirflowAuthenticationClassResolved, AirflowClientAuthenticationDetailsResolved, }, @@ -336,6 +338,11 @@ pub enum Error { BuildListenerVolume { source: ListenerOperatorVolumeSourceBuilderError, }, + + #[snafu(display("failed to apply group listener"))] + ApplyGroupListener { + source: stackable_operator::cluster_resources::Error, + }, } type Result = std::result::Result; @@ -501,6 +508,23 @@ pub async fn reconcile_airflow( airflow_executor, )?; + if let Some(listener_class) = + airflow.merged_listener_class(&airflow_role, &rolegroup.role_group) + { + if listener_class.discoverable() { + let rg_group_listener = build_group_listener( + airflow, + &resolved_product_image, + &rolegroup, + listener_class.to_string(), + )?; + cluster_resources + .add(client, rg_group_listener) + .await + .context(ApplyGroupListenerSnafu)?; + } + } + ss_cond_builder.add( cluster_resources .add(client, rg_statefulset) @@ -805,6 +829,51 @@ fn build_rolegroup_metadata( Ok(metadata) } +pub fn build_group_listener( + airflow: &v1alpha1::AirflowCluster, + resolved_product_image: &ResolvedProductImage, + rolegroup: &RoleGroupRef, + listener_class: String, +) -> Result { + Ok(Listener { + metadata: ObjectMetaBuilder::new() + .name_and_namespace(airflow) + .name(airflow.group_listener_name(rolegroup)) + .ownerreference_from_resource(airflow, None, Some(true)) + .context(ObjectMissingMetadataForOwnerRefSnafu)? + .with_recommended_labels(build_recommended_labels( + airflow, + AIRFLOW_CONTROLLER_NAME, + &resolved_product_image.app_version_label, + &rolegroup.role, + &rolegroup.role_group, + )) + .context(ObjectMetaSnafu)? + .build(), + spec: ListenerSpec { + class_name: Some(listener_class), + ports: Some(listener_ports()), + ..ListenerSpec::default() + }, + status: None, + }) +} + +fn listener_ports() -> Vec { + vec![ + ListenerPort { + name: METRICS_PORT_NAME.to_string(), + port: METRICS_PORT.into(), + protocol: Some("TCP".to_string()), + }, + ListenerPort { + name: HTTP_PORT_NAME.to_string(), + port: HTTP_PORT.into(), + protocol: Some("TCP".to_string()), + }, + ] +} + /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. /// /// The [`Pod`](`stackable_operator::k8s_openapi::api::core::v1::Pod`)s are accessible through the corresponding [`Service`] (from [`build_rolegroup_service`]). @@ -836,6 +905,16 @@ fn build_server_rolegroup_statefulset( ); let recommended_labels = Labels::recommended(recommended_object_labels.clone()).context(LabelBuildSnafu)?; + // Used for PVC templates that cannot be modified once they are deployed + let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( + airflow, + AIRFLOW_CONTROLLER_NAME, + // A version value is required, and we do want to use the "recommended" format for the other desired labels + "none", + &rolegroup_ref.role, + &rolegroup_ref.role_group, + )) + .context(LabelBuildSnafu)?; let pb_metadata = ObjectMetaBuilder::new() .with_recommended_labels(recommended_object_labels) @@ -939,8 +1018,8 @@ fn build_server_rolegroup_statefulset( // externally reachable listener endpoints will use persistent volumes // so that load balancers can hard-code the target addresses let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerClass(listener_class.to_string()), - &recommended_labels, + &ListenerReference::ListenerName(airflow.group_listener_name(rolegroup_ref)), + &unversioned_recommended_labels, ) .context(BuildListenerVolumeSnafu)? .build_pvc(LISTENER_VOLUME_NAME.to_string()) diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index c3afdc5d..301c91d6 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -279,6 +279,13 @@ impl HasStatusCondition for v1alpha1::AirflowCluster { } impl v1alpha1::AirflowCluster { + /// The name of the group-listener provided for a specific role-group. + /// Webservers will use this group listener so that only one load balancer + /// is needed (per role group). + pub fn group_listener_name(&self, rolegroup: &RoleGroupRef) -> String { + format!("{}-group", rolegroup.object_name()) + } + /// the worker role will not be returned if airflow provisions pods as needed (i.e. when /// the kubernetes executor is specified) pub fn get_role(&self, role: &AirflowRole) -> Option> { diff --git a/tests/templates/kuttl/external-access/40-assert.yaml b/tests/templates/kuttl/external-access/40-assert.yaml index 4a61ca08..3ecb7473 100644 --- a/tests/templates/kuttl/external-access/40-assert.yaml +++ b/tests/templates/kuttl/external-access/40-assert.yaml @@ -22,8 +22,8 @@ spec: spec: terminationGracePeriodSeconds: 120 status: - readyReplicas: 1 - replicas: 1 + readyReplicas: 2 + replicas: 2 --- apiVersion: apps/v1 kind: StatefulSet @@ -70,8 +70,8 @@ kind: PodDisruptionBudget metadata: name: airflow-webserver status: - expectedPods: 3 - currentHealthy: 3 + expectedPods: 4 + currentHealthy: 4 disruptionsAllowed: 1 --- apiVersion: policy/v1 @@ -102,13 +102,13 @@ spec: apiVersion: v1 kind: Service metadata: - name: listener-airflow-webserver-default-0 + name: airflow-webserver-default-group spec: type: NodePort # external-stable --- apiVersion: v1 kind: Service metadata: - name: listener-airflow-webserver-external-unstable-0 + name: airflow-webserver-external-unstable-group spec: type: NodePort # external-unstable diff --git a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 index f6a9e614..a67a6e3a 100644 --- a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 @@ -41,7 +41,7 @@ spec: listenerClass: external-stable roleGroups: default: - replicas: 1 + replicas: 2 external-unstable: replicas: 1 config: From dcc21f92dd65f3c5f69f935b4583278157bc9f12 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 28 Apr 2025 13:18:05 +0200 Subject: [PATCH 29/39] replace enum with string; use consistent webserver address for all listener classes --- deploy/helm/airflow-operator/crds/crds.yaml | 8 --- .../operator-binary/src/airflow_controller.rs | 62 ++++++++----------- rust/operator-binary/src/crd/mod.rs | 37 ++--------- .../kuttl/external-access/40-assert.yaml | 2 +- 4 files changed, 31 insertions(+), 78 deletions(-) diff --git a/deploy/helm/airflow-operator/crds/crds.yaml b/deploy/helm/airflow-operator/crds/crds.yaml index 979bc19b..49e32896 100644 --- a/deploy/helm/airflow-operator/crds/crds.yaml +++ b/deploy/helm/airflow-operator/crds/crds.yaml @@ -1323,10 +1323,6 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - enum: - - cluster-internal - - external-unstable - - external-stable nullable: true type: string logging: @@ -1548,10 +1544,6 @@ spec: type: string listenerClass: description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose the webserver. - enum: - - cluster-internal - - external-unstable - - external-stable nullable: true type: string logging: diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 0e6d1446..afd74709 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -511,18 +511,16 @@ pub async fn reconcile_airflow( if let Some(listener_class) = airflow.merged_listener_class(&airflow_role, &rolegroup.role_group) { - if listener_class.discoverable() { - let rg_group_listener = build_group_listener( - airflow, - &resolved_product_image, - &rolegroup, - listener_class.to_string(), - )?; - cluster_resources - .add(client, rg_group_listener) - .await - .context(ApplyGroupListenerSnafu)?; - } + let rg_group_listener = build_group_listener( + airflow, + &resolved_product_image, + &rolegroup, + listener_class.to_string(), + )?; + cluster_resources + .add(client, rg_group_listener) + .await + .context(ApplyGroupListenerSnafu)?; } ss_cond_builder.add( @@ -903,8 +901,6 @@ fn build_server_rolegroup_statefulset( &rolegroup_ref.role, &rolegroup_ref.role_group, ); - let recommended_labels = - Labels::recommended(recommended_object_labels.clone()).context(LabelBuildSnafu)?; // Used for PVC templates that cannot be modified once they are deployed let unversioned_recommended_labels = Labels::recommended(build_recommended_labels( airflow, @@ -1011,29 +1007,23 @@ fn build_server_rolegroup_statefulset( let mut pvcs: Option> = None; - if let Some(listener_class) = - airflow.merged_listener_class(airflow_role, &rolegroup_ref.role_group) + if airflow + .merged_listener_class(airflow_role, &rolegroup_ref.role_group) + .is_some() { - if listener_class.discoverable() { - // externally reachable listener endpoints will use persistent volumes - // so that load balancers can hard-code the target addresses - let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(airflow.group_listener_name(rolegroup_ref)), - &unversioned_recommended_labels, - ) - .context(BuildListenerVolumeSnafu)? - .build_pvc(LISTENER_VOLUME_NAME.to_string()) - .context(BuildListenerVolumeSnafu)?; - pvcs = Some(vec![pvc]); - } else { - // non-reachable endpoints use ephemeral volumes - pb.add_listener_volume_by_listener_class( - LISTENER_VOLUME_NAME, - &listener_class.to_string(), - &recommended_labels, - ) - .context(AddVolumeSnafu)?; - } + // Listener endpoints for the Webserver role will use persistent volumes + // so that load balancers can hard-code the target addresses. This will + // be the case even when no class is set (and the value defaults to + // cluster-internal) as the address should still be consistent. + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerName(airflow.group_listener_name(rolegroup_ref)), + &unversioned_recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + pvcs = Some(vec![pvc]); + airflow_container .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) .context(AddVolumeMountSnafu)?; diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 301c91d6..18c9691d 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -17,7 +17,7 @@ use stackable_operator::{ }, config::{ fragment::{self, Fragment, ValidationError}, - merge::{Atomic, Merge}, + merge::Merge, }, k8s_openapi::{ api::core::v1::{Volume, VolumeMount}, @@ -406,10 +406,10 @@ impl v1alpha1::AirflowCluster { &self, role: &AirflowRole, rolegroup_name: &String, - ) -> Option { + ) -> Option { if role == &AirflowRole::Webserver { if let Some(webservers) = self.spec.webservers.as_ref() { - let conf_defaults = Some(SupportedListenerClasses::ClusterInternal); + let conf_defaults = Some("cluster-internal".to_string()); let mut conf_role = webservers.config.config.listener_class.to_owned(); let mut conf_rolegroup = webservers .role_groups @@ -505,35 +505,6 @@ pub struct AirflowOpaConfig { pub cache: UserInformationCache, } -#[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] -#[serde(rename_all = "PascalCase")] -pub enum SupportedListenerClasses { - #[default] - #[serde(rename = "cluster-internal")] - #[strum(serialize = "cluster-internal")] - ClusterInternal, - - #[serde(rename = "external-unstable")] - #[strum(serialize = "external-unstable")] - ExternalUnstable, - - #[serde(rename = "external-stable")] - #[strum(serialize = "external-stable")] - ExternalStable, -} - -impl Atomic for SupportedListenerClasses {} - -impl SupportedListenerClasses { - pub fn discoverable(&self) -> bool { - match self { - SupportedListenerClasses::ClusterInternal => false, - SupportedListenerClasses::ExternalUnstable => true, - SupportedListenerClasses::ExternalStable => true, - } - } -} - #[derive(Clone, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct AirflowCredentials { @@ -835,7 +806,7 @@ pub struct WebserverConfig { /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose the webserver. #[serde(default)] - pub listener_class: SupportedListenerClasses, + pub listener_class: String, } impl AirflowConfig { diff --git a/tests/templates/kuttl/external-access/40-assert.yaml b/tests/templates/kuttl/external-access/40-assert.yaml index 3ecb7473..8cad5b4a 100644 --- a/tests/templates/kuttl/external-access/40-assert.yaml +++ b/tests/templates/kuttl/external-access/40-assert.yaml @@ -95,7 +95,7 @@ status: apiVersion: v1 kind: Service metadata: - name: airflow-webserver-cluster-internal-0-listener + name: airflow-webserver-cluster-internal-group spec: type: ClusterIP # cluster-internal --- From 3d2f3e0af6b45175f7666230c24f8c45f5b927dc Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Mon, 28 Apr 2025 15:33:33 +0200 Subject: [PATCH 30/39] removed unused test account --- .../kuttl/external-access/00-rbac.yaml.j2 | 29 ------------------- 1 file changed, 29 deletions(-) delete mode 100644 tests/templates/kuttl/external-access/00-rbac.yaml.j2 diff --git a/tests/templates/kuttl/external-access/00-rbac.yaml.j2 b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 deleted file mode 100644 index 7ee61d23..00000000 --- a/tests/templates/kuttl/external-access/00-rbac.yaml.j2 +++ /dev/null @@ -1,29 +0,0 @@ ---- -kind: Role -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: test-role -rules: -{% if test_scenario['values']['openshift'] == "true" %} - - apiGroups: ["security.openshift.io"] - resources: ["securitycontextconstraints"] - resourceNames: ["privileged"] - verbs: ["use"] -{% endif %} ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: test-sa ---- -kind: RoleBinding -apiVersion: rbac.authorization.k8s.io/v1 -metadata: - name: test-rb -subjects: - - kind: ServiceAccount - name: test-sa -roleRef: - kind: Role - name: test-role - apiGroup: rbac.authorization.k8s.io From 95597e3dfcc3d31361c93882bdb4a6d48fdc7265 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:03:29 +0200 Subject: [PATCH 31/39] Update docs/modules/airflow/pages/usage-guide/listenerclass.adoc MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- docs/modules/airflow/pages/usage-guide/listenerclass.adoc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc index fe803174..9e4ee761 100644 --- a/docs/modules/airflow/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/airflow/pages/usage-guide/listenerclass.adoc @@ -15,5 +15,5 @@ spec: celeryExecutors: ... ---- -<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). +<1> Specify a ListenerClass, such as `external-stable`, `external-unstable`, or `cluster-internal` (the default setting is `cluster-internal`). This can be set only for the webservers role. From 06e44af14104aa637af1870d1aa585d0954df8f9 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:05:41 +0200 Subject: [PATCH 32/39] Update tests/templates/kuttl/opa/41_check-authorization.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- tests/templates/kuttl/opa/41_check-authorization.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/opa/41_check-authorization.py b/tests/templates/kuttl/opa/41_check-authorization.py index d86502dc..d956cc7d 100644 --- a/tests/templates/kuttl/opa/41_check-authorization.py +++ b/tests/templates/kuttl/opa/41_check-authorization.py @@ -26,7 +26,7 @@ "password": "NvfpU518", } -url = "http://airflow-webserver-default:8080" +url = "http://airflow-webserver-default-group:8080" def create_user(user): From 2a6313edaca2a46c3a09d33aec0e67c3a60bee44 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy <1712947+adwk67@users.noreply.github.com> Date: Tue, 29 Apr 2025 13:05:54 +0200 Subject: [PATCH 33/39] Update tests/templates/kuttl/oidc/login.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Natalie Klestrup Röijezon --- tests/templates/kuttl/oidc/login.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index aa859803..3a87f9f8 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -10,7 +10,7 @@ ) session = requests.Session() -url = "http://airflow-webserver-default:8080" +url = "http://airflow-webserver-default-group:8080" # Click on "Sign In with keycloak" in Airflow login_page = session.get(f"{url}/login/keycloak?next=") From f9c41bc4869c2b56b6d05bb7ed6312cff5923457 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 29 Apr 2025 14:39:21 +0200 Subject: [PATCH 34/39] review feedback: tests --- .../templates/kuttl/external-access/10-assert.yaml | 14 -------------- .../templates/kuttl/external-access/20-assert.yaml | 12 ++++++++++-- .../kuttl/external-access/30-assert.yaml.j2 | 10 ---------- ...l-vector-aggregator-discovery-configmap.yaml.j2 | 9 --------- 4 files changed, 10 insertions(+), 35 deletions(-) delete mode 100644 tests/templates/kuttl/external-access/10-assert.yaml delete mode 100644 tests/templates/kuttl/external-access/30-assert.yaml.j2 delete mode 100644 tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 diff --git a/tests/templates/kuttl/external-access/10-assert.yaml b/tests/templates/kuttl/external-access/10-assert.yaml deleted file mode 100644 index 319e927a..00000000 --- a/tests/templates/kuttl/external-access/10-assert.yaml +++ /dev/null @@ -1,14 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -metadata: - name: test-airflow-postgresql -timeout: 480 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: airflow-postgresql -status: - readyReplicas: 1 - replicas: 1 diff --git a/tests/templates/kuttl/external-access/20-assert.yaml b/tests/templates/kuttl/external-access/20-assert.yaml index f038df42..0ae1697d 100644 --- a/tests/templates/kuttl/external-access/20-assert.yaml +++ b/tests/templates/kuttl/external-access/20-assert.yaml @@ -2,8 +2,16 @@ apiVersion: kuttl.dev/v1beta1 kind: TestAssert metadata: - name: test-airflow-redis -timeout: 360 + name: test-airflow-postgresql +timeout: 480 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: airflow-postgresql +status: + readyReplicas: 1 + replicas: 1 --- apiVersion: apps/v1 kind: StatefulSet diff --git a/tests/templates/kuttl/external-access/30-assert.yaml.j2 b/tests/templates/kuttl/external-access/30-assert.yaml.j2 deleted file mode 100644 index 50b1d4c3..00000000 --- a/tests/templates/kuttl/external-access/30-assert.yaml.j2 +++ /dev/null @@ -1,10 +0,0 @@ ---- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -{% if lookup('env', 'VECTOR_AGGREGATOR') %} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: vector-aggregator-discovery -{% endif %} diff --git a/tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 deleted file mode 100644 index 2d6a0df5..00000000 --- a/tests/templates/kuttl/external-access/30-install-vector-aggregator-discovery-configmap.yaml.j2 +++ /dev/null @@ -1,9 +0,0 @@ -{% if lookup('env', 'VECTOR_AGGREGATOR') %} ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: vector-aggregator-discovery -data: - ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} -{% endif %} From cb85a6a24d89d5b877ca11fc02fddf64463a0b5c Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 29 Apr 2025 15:56:38 +0200 Subject: [PATCH 35/39] review feedback: remove metrics port and re-work conditions --- .../operator-binary/src/airflow_controller.rs | 54 +++++++++---------- rust/operator-binary/src/crd/mod.rs | 11 +++- 2 files changed, 35 insertions(+), 30 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index afd74709..6c2f1fab 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -511,16 +511,21 @@ pub async fn reconcile_airflow( if let Some(listener_class) = airflow.merged_listener_class(&airflow_role, &rolegroup.role_group) { - let rg_group_listener = build_group_listener( - airflow, - &resolved_product_image, - &rolegroup, - listener_class.to_string(), - )?; - cluster_resources - .add(client, rg_group_listener) - .await - .context(ApplyGroupListenerSnafu)?; + if let Some(listener_group_name) = + airflow.group_listener_name(&airflow_role, &rolegroup) + { + let rg_group_listener = build_group_listener( + airflow, + &resolved_product_image, + &rolegroup, + listener_class.to_string(), + listener_group_name, + )?; + cluster_resources + .add(client, rg_group_listener) + .await + .context(ApplyGroupListenerSnafu)?; + } } ss_cond_builder.add( @@ -832,11 +837,12 @@ pub fn build_group_listener( resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, listener_class: String, + listener_group_name: String, ) -> Result { Ok(Listener { metadata: ObjectMetaBuilder::new() .name_and_namespace(airflow) - .name(airflow.group_listener_name(rolegroup)) + .name(listener_group_name) .ownerreference_from_resource(airflow, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( @@ -857,19 +863,14 @@ pub fn build_group_listener( }) } +/// We only use the http port here and intentionally omit +/// the metrics one. fn listener_ports() -> Vec { - vec![ - ListenerPort { - name: METRICS_PORT_NAME.to_string(), - port: METRICS_PORT.into(), - protocol: Some("TCP".to_string()), - }, - ListenerPort { - name: HTTP_PORT_NAME.to_string(), - port: HTTP_PORT.into(), - protocol: Some("TCP".to_string()), - }, - ] + vec![ListenerPort { + name: HTTP_PORT_NAME.to_string(), + port: HTTP_PORT.into(), + protocol: Some("TCP".to_string()), + }] } /// The rolegroup [`StatefulSet`] runs the rolegroup, as configured by the administrator. @@ -1007,16 +1008,13 @@ fn build_server_rolegroup_statefulset( let mut pvcs: Option> = None; - if airflow - .merged_listener_class(airflow_role, &rolegroup_ref.role_group) - .is_some() - { + if let Some(listener_group_name) = airflow.group_listener_name(airflow_role, rolegroup_ref) { // Listener endpoints for the Webserver role will use persistent volumes // so that load balancers can hard-code the target addresses. This will // be the case even when no class is set (and the value defaults to // cluster-internal) as the address should still be consistent. let pvc = ListenerOperatorVolumeSourceBuilder::new( - &ListenerReference::ListenerName(airflow.group_listener_name(rolegroup_ref)), + &ListenerReference::ListenerName(listener_group_name), &unversioned_recommended_labels, ) .context(BuildListenerVolumeSnafu)? diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 18c9691d..677a8d49 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -282,8 +282,15 @@ impl v1alpha1::AirflowCluster { /// The name of the group-listener provided for a specific role-group. /// Webservers will use this group listener so that only one load balancer /// is needed (per role group). - pub fn group_listener_name(&self, rolegroup: &RoleGroupRef) -> String { - format!("{}-group", rolegroup.object_name()) + pub fn group_listener_name( + &self, + role: &AirflowRole, + rolegroup: &RoleGroupRef, + ) -> Option { + match role { + AirflowRole::Webserver => Some(format!("{}-group", rolegroup.object_name())), + AirflowRole::Scheduler | AirflowRole::Worker => None, + } } /// the worker role will not be returned if airflow provisions pods as needed (i.e. when From 57e164431d79524d7c0eb96561dc4a6b7a96a8a6 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 29 Apr 2025 16:27:56 +0200 Subject: [PATCH 36/39] use custom listeners to stay independent of future changes to standard classes --- .../kuttl/external-access/30-assert.yaml | 27 +++++++++++++++++++ .../external-access/30-listener-classes.yaml | 21 +++++++++++++++ .../40-install-airflow-cluster.yaml.j2 | 6 ++--- 3 files changed, 51 insertions(+), 3 deletions(-) create mode 100644 tests/templates/kuttl/external-access/30-assert.yaml create mode 100644 tests/templates/kuttl/external-access/30-listener-classes.yaml diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml new file mode 100644 index 00000000..ea3da119 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +metadata: + name: test-listeners +timeout: 120 +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-cluster-internal +spec: + serviceType: ClusterIP +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-stable +spec: + serviceType: NodePort +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/external-access/30-listener-classes.yaml b/tests/templates/kuttl/external-access/30-listener-classes.yaml new file mode 100644 index 00000000..8b3c376e --- /dev/null +++ b/tests/templates/kuttl/external-access/30-listener-classes.yaml @@ -0,0 +1,21 @@ +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-cluster-internal +spec: + serviceType: ClusterIP +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-stable +spec: + serviceType: NodePort +--- +apiVersion: listeners.stackable.tech/v1alpha1 +kind: ListenerClass +metadata: + name: test-external-unstable +spec: + serviceType: NodePort diff --git a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 index a67a6e3a..693c1b42 100644 --- a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 @@ -38,18 +38,18 @@ spec: credentialsSecret: test-airflow-credentials webservers: config: - listenerClass: external-stable + listenerClass: test-external-stable roleGroups: default: replicas: 2 external-unstable: replicas: 1 config: - listenerClass: external-unstable + listenerClass: test-external-unstable cluster-internal: replicas: 1 config: - listenerClass: cluster-internal + listenerClass: test-cluster-internal celeryExecutors: roleGroups: default: From ea88ae4d8f2c9c4af93e6136eb0c331137f7cf3d Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 29 Apr 2025 17:21:22 +0200 Subject: [PATCH 37/39] make custom listener classes namespace-specific --- .../external-access/30-listener-classes.yaml | 25 ++++--------------- .../40-install-airflow-cluster.yaml | 8 ++++++ ...aml.j2 => install-airflow-cluster.yaml.j2} | 11 +++----- .../{30-assert.yaml => listener-classes.yaml} | 12 +++------ 4 files changed, 19 insertions(+), 37 deletions(-) create mode 100644 tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml rename tests/templates/kuttl/external-access/{40-install-airflow-cluster.yaml.j2 => install-airflow-cluster.yaml.j2} (85%) rename tests/templates/kuttl/external-access/{30-assert.yaml => listener-classes.yaml} (63%) diff --git a/tests/templates/kuttl/external-access/30-listener-classes.yaml b/tests/templates/kuttl/external-access/30-listener-classes.yaml index 8b3c376e..893032c5 100644 --- a/tests/templates/kuttl/external-access/30-listener-classes.yaml +++ b/tests/templates/kuttl/external-access/30-listener-classes.yaml @@ -1,21 +1,6 @@ --- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-cluster-internal -spec: - serviceType: ClusterIP ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-external-stable -spec: - serviceType: NodePort ---- -apiVersion: listeners.stackable.tech/v1alpha1 -kind: ListenerClass -metadata: - name: test-external-unstable -spec: - serviceType: NodePort +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: | + envsubst < listener-classes.yaml | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml new file mode 100644 index 00000000..9da44e7e --- /dev/null +++ b/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml @@ -0,0 +1,8 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +timeout: 600 +commands: + - script: > + envsubst < install-airflow-cluster.yaml | + kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 similarity index 85% rename from tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 rename to tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 index 693c1b42..d18d6c57 100644 --- a/tests/templates/kuttl/external-access/40-install-airflow-cluster.yaml.j2 +++ b/tests/templates/kuttl/external-access/install-airflow-cluster.yaml.j2 @@ -1,8 +1,3 @@ -apiVersion: kuttl.dev/v1beta1 -kind: TestStep -metadata: - name: install-airflow-db -timeout: 480 --- apiVersion: v1 kind: Secret @@ -38,18 +33,18 @@ spec: credentialsSecret: test-airflow-credentials webservers: config: - listenerClass: test-external-stable + listenerClass: test-external-stable-$NAMESPACE roleGroups: default: replicas: 2 external-unstable: replicas: 1 config: - listenerClass: test-external-unstable + listenerClass: test-external-unstable-$NAMESPACE cluster-internal: replicas: 1 config: - listenerClass: test-cluster-internal + listenerClass: test-cluster-internal-$NAMESPACE celeryExecutors: roleGroups: default: diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/listener-classes.yaml similarity index 63% rename from tests/templates/kuttl/external-access/30-assert.yaml rename to tests/templates/kuttl/external-access/listener-classes.yaml index ea3da119..4131526a 100644 --- a/tests/templates/kuttl/external-access/30-assert.yaml +++ b/tests/templates/kuttl/external-access/listener-classes.yaml @@ -1,27 +1,21 @@ --- -apiVersion: kuttl.dev/v1beta1 -kind: TestAssert -metadata: - name: test-listeners -timeout: 120 ---- apiVersion: listeners.stackable.tech/v1alpha1 kind: ListenerClass metadata: - name: test-cluster-internal + name: test-cluster-internal-$NAMESPACE spec: serviceType: ClusterIP --- apiVersion: listeners.stackable.tech/v1alpha1 kind: ListenerClass metadata: - name: test-external-stable + name: test-external-stable-$NAMESPACE spec: serviceType: NodePort --- apiVersion: listeners.stackable.tech/v1alpha1 kind: ListenerClass metadata: - name: test-external-unstable + name: test-external-unstable-$NAMESPACE spec: serviceType: NodePort From f0b5b3020037233ec34d9b7c2a046c1b5c377ce9 Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 6 May 2025 11:47:29 +0200 Subject: [PATCH 38/39] changes as per decision 51 --- .../operator-binary/src/airflow_controller.rs | 28 ++++--------------- rust/operator-binary/src/crd/mod.rs | 2 +- tests/templates/kuttl/commons/metrics.py | 10 ++++--- .../kuttl/external-access/40-assert.yaml | 6 ++-- .../kuttl/mount-dags-gitsync/dag_metrics.py | 10 ++++--- 5 files changed, 22 insertions(+), 34 deletions(-) diff --git a/rust/operator-binary/src/airflow_controller.rs b/rust/operator-binary/src/airflow_controller.rs index 6c2f1fab..e7154e9a 100644 --- a/rust/operator-binary/src/airflow_controller.rs +++ b/rust/operator-binary/src/airflow_controller.rs @@ -483,12 +483,7 @@ pub async fn reconcile_airflow( .merged_config(&airflow_role, &rolegroup) .context(FailedToResolveConfigSnafu)?; - let rg_service = build_rolegroup_service( - airflow, - &airflow_role, - &resolved_product_image, - &rolegroup, - )?; + let rg_service = build_rolegroup_service(airflow, &resolved_product_image, &rolegroup)?; cluster_resources.add(client, rg_service).await.context( ApplyRoleGroupServiceSnafu { rolegroup: rolegroup.clone(), @@ -639,15 +634,6 @@ async fn build_executor_template( Ok(()) } -fn role_ports(port: u16) -> Vec { - vec![ServicePort { - name: Some(HTTP_PORT_NAME.to_owned()), - port: port.into(), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }] -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( @@ -762,21 +748,16 @@ fn build_rolegroup_config_map( /// This is mostly useful for internal communication between peers, or for clients that perform client-side load balancing. fn build_rolegroup_service( airflow: &v1alpha1::AirflowCluster, - airflow_role: &AirflowRole, resolved_product_image: &ResolvedProductImage, rolegroup: &RoleGroupRef, ) -> Result { - let mut ports = vec![ServicePort { + let ports = vec![ServicePort { name: Some(METRICS_PORT_NAME.into()), port: METRICS_PORT.into(), protocol: Some("TCP".to_string()), ..Default::default() }]; - if let Some(http_port) = airflow_role.get_http_port() { - ports.append(&mut role_ports(http_port)); - } - let prometheus_label = Label::try_from(("prometheus.io/scrape", "true")).context(BuildLabelSnafu)?; @@ -785,6 +766,7 @@ fn build_rolegroup_service( &resolved_product_image, &rolegroup, prometheus_label, + format!("{name}-metrics", name = rolegroup.object_name()), )?; let service_selector_labels = @@ -813,10 +795,11 @@ fn build_rolegroup_metadata( resolved_product_image: &&ResolvedProductImage, rolegroup: &&RoleGroupRef, prometheus_label: Label, + name: String, ) -> Result { let metadata = ObjectMetaBuilder::new() .name_and_namespace(airflow) - .name(rolegroup.object_name()) + .name(name) .ownerreference_from_resource(airflow, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( @@ -1152,6 +1135,7 @@ fn build_server_rolegroup_statefulset( &resolved_product_image, &rolegroup_ref, restarter_label, + rolegroup_ref.object_name(), )?; let statefulset_match_labels = Labels::role_group_selector( diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index 677a8d49..0f3d30f9 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -288,7 +288,7 @@ impl v1alpha1::AirflowCluster { rolegroup: &RoleGroupRef, ) -> Option { match role { - AirflowRole::Webserver => Some(format!("{}-group", rolegroup.object_name())), + AirflowRole::Webserver => Some(rolegroup.object_name()), AirflowRole::Scheduler | AirflowRole::Worker => None, } } diff --git a/tests/templates/kuttl/commons/metrics.py b/tests/templates/kuttl/commons/metrics.py index e0fb5eff..866535f9 100755 --- a/tests/templates/kuttl/commons/metrics.py +++ b/tests/templates/kuttl/commons/metrics.py @@ -13,10 +13,12 @@ def exception_handler(exception_type, exception, traceback): def assert_metric(role, role_group, metric): - metric_response = requests.get(f"http://airflow-{role}-{role_group}:9102/metrics") - assert ( - metric_response.status_code == 200 - ), f"Metrics could not be retrieved from the {role}-{role_group}." + metric_response = requests.get( + f"http://airflow-{role}-{role_group}-metrics:9102/metrics" + ) + assert metric_response.status_code == 200, ( + f"Metrics could not be retrieved from the {role}-{role_group}." + ) return metric in metric_response.text diff --git a/tests/templates/kuttl/external-access/40-assert.yaml b/tests/templates/kuttl/external-access/40-assert.yaml index 8cad5b4a..b7a9f06f 100644 --- a/tests/templates/kuttl/external-access/40-assert.yaml +++ b/tests/templates/kuttl/external-access/40-assert.yaml @@ -95,20 +95,20 @@ status: apiVersion: v1 kind: Service metadata: - name: airflow-webserver-cluster-internal-group + name: airflow-webserver-cluster-internal spec: type: ClusterIP # cluster-internal --- apiVersion: v1 kind: Service metadata: - name: airflow-webserver-default-group + name: airflow-webserver-default spec: type: NodePort # external-stable --- apiVersion: v1 kind: Service metadata: - name: airflow-webserver-external-unstable-group + name: airflow-webserver-external-unstable spec: type: NodePort # external-unstable diff --git a/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py b/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py index bc5ff92a..670bbd53 100755 --- a/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py +++ b/tests/templates/kuttl/mount-dags-gitsync/dag_metrics.py @@ -7,10 +7,12 @@ def assert_metric(role, metric): - metric_response = requests.get(f"http://airflow-{role}-default:9102/metrics") - assert ( - metric_response.status_code == 200 - ), f"Metrics could not be retrieved from the {role}." + metric_response = requests.get( + f"http://airflow-{role}-default-metrics:9102/metrics" + ) + assert metric_response.status_code == 200, ( + f"Metrics could not be retrieved from the {role}." + ) return metric in metric_response.text From 02cc5c77c07bd7383944b93b84d01626e87bafdc Mon Sep 17 00:00:00 2001 From: Andrew Kenworthy Date: Tue, 6 May 2025 13:15:41 +0200 Subject: [PATCH 39/39] fixed tests --- tests/templates/kuttl/oidc/login.py | 2 +- tests/templates/kuttl/opa/41_check-authorization.py | 2 +- tests/templates/kuttl/orphaned-resources/50-assert.yaml | 2 +- tests/templates/kuttl/orphaned-resources/50-errors.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/templates/kuttl/oidc/login.py b/tests/templates/kuttl/oidc/login.py index 3a87f9f8..aa859803 100644 --- a/tests/templates/kuttl/oidc/login.py +++ b/tests/templates/kuttl/oidc/login.py @@ -10,7 +10,7 @@ ) session = requests.Session() -url = "http://airflow-webserver-default-group:8080" +url = "http://airflow-webserver-default:8080" # Click on "Sign In with keycloak" in Airflow login_page = session.get(f"{url}/login/keycloak?next=") diff --git a/tests/templates/kuttl/opa/41_check-authorization.py b/tests/templates/kuttl/opa/41_check-authorization.py index d956cc7d..d86502dc 100644 --- a/tests/templates/kuttl/opa/41_check-authorization.py +++ b/tests/templates/kuttl/opa/41_check-authorization.py @@ -26,7 +26,7 @@ "password": "NvfpU518", } -url = "http://airflow-webserver-default-group:8080" +url = "http://airflow-webserver-default:8080" def create_user(user): diff --git a/tests/templates/kuttl/orphaned-resources/50-assert.yaml b/tests/templates/kuttl/orphaned-resources/50-assert.yaml index 050e881d..1272ebc6 100644 --- a/tests/templates/kuttl/orphaned-resources/50-assert.yaml +++ b/tests/templates/kuttl/orphaned-resources/50-assert.yaml @@ -23,4 +23,4 @@ metadata: apiVersion: v1 kind: Service metadata: - name: airflow-worker-newrolegroup + name: airflow-worker-newrolegroup-metrics diff --git a/tests/templates/kuttl/orphaned-resources/50-errors.yaml b/tests/templates/kuttl/orphaned-resources/50-errors.yaml index 06b33ec1..1da89a7d 100644 --- a/tests/templates/kuttl/orphaned-resources/50-errors.yaml +++ b/tests/templates/kuttl/orphaned-resources/50-errors.yaml @@ -22,4 +22,4 @@ metadata: apiVersion: v1 kind: Service metadata: - name: airflow-worker-default + name: airflow-worker-default-metrics