diff --git a/CHANGELOG.md b/CHANGELOG.md index c2414611..f6a231b9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ### Added +- Added listener support for HBase ([#639]). - Adds new telemetry CLI arguments and environment variables ([#652]). - Use `--file-log-max-files` (or `FILE_LOG_MAX_FILES`) to limit the number of log files kept. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. @@ -25,6 +26,7 @@ - Use `json` file extension for log files ([#647]). - Fix a bug where changes to ConfigMaps that are referenced in the HbaseCluster spec didn't trigger a reconciliation ([#645]). +[#639]: https://github.com/stackabletech/hbase-operator/pull/639 [#640]: https://github.com/stackabletech/hbase-operator/pull/640 [#645]: https://github.com/stackabletech/hbase-operator/pull/645 [#647]: https://github.com/stackabletech/hbase-operator/pull/647 diff --git a/deploy/helm/hbase-operator/crds/crds.yaml b/deploy/helm/hbase-operator/crds/crds.yaml index 6294fe40..e8182efb 100644 --- a/deploy/helm/hbase-operator/crds/crds.yaml +++ b/deploy/helm/hbase-operator/crds/crds.yaml @@ -73,20 +73,6 @@ spec: hdfsConfigMapName: description: Name of the [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery) for an HDFS cluster. type: string - listenerClass: - default: cluster-internal - description: |- - This field controls which type of Service the Operator creates for this HbaseCluster: - - * cluster-internal: Use a ClusterIP service - - * external-unstable: Use a NodePort service - - This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - enum: - - cluster-internal - - external-unstable - type: string vectorAggregatorConfigMapName: description: Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector. nullable: true @@ -210,6 +196,14 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -460,6 +454,14 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -691,6 +693,14 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -969,6 +979,14 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -1228,6 +1246,14 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} @@ -1478,6 +1504,14 @@ spec: hbaseRootdir: nullable: true type: string + listenerClass: + description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. + enum: + - cluster-internal + - external-unstable + - external-stable + nullable: true + type: string logging: default: containers: {} diff --git a/deploy/helm/hbase-operator/templates/roles.yaml b/deploy/helm/hbase-operator/templates/roles.yaml index f13d450b..7bc43bdb 100644 --- a/deploy/helm/hbase-operator/templates/roles.yaml +++ b/deploy/helm/hbase-operator/templates/roles.yaml @@ -77,6 +77,12 @@ rules: verbs: - create - patch + - apiGroups: + - listeners.stackable.tech + resources: + - listeners + verbs: + - get - apiGroups: - {{ include "operator.name" . }}.stackable.tech resources: diff --git a/docs/modules/hbase/pages/usage-guide/listenerclass.adoc b/docs/modules/hbase/pages/usage-guide/listenerclass.adoc index 1f6d48b8..4c19e5e7 100644 --- a/docs/modules/hbase/pages/usage-guide/listenerclass.adoc +++ b/docs/modules/hbase/pages/usage-guide/listenerclass.adoc @@ -1,18 +1,37 @@ = Service exposition with ListenerClasses +:description: Configure HBase service exposure using ListenerClasses to control internal and external access for all roles. -Apache HBase offers an API. -The operator deploys a service called `` (where `` is the name of the HbaseCluster) through which HBase can be reached. +The operator deploys a xref:listener-operator:listener.adoc[Listener] for each Master, Regionserver and Restserver pod. +They all default to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.{masters,regionServers,restServers}.config.listenerClass`: -This service can have either the `cluster-internal` or `external-unstable` type. -`external-stable` is not supported for HBase at the moment. -Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level. +[source,yaml] +---- +spec: + masters: + config: + listenerClass: external-unstable # <1> + regionServers: + config: + listenerClass: external-unstable + restServers: + config: + listenerClass: external-unstable +---- +<1> Specify one of `external-stable`, `external-unstable`, `cluster-internal` (the default setting is `cluster-internal`). +This can be set separately for all three roles. -This is how the listener class is configured: +Externally-reachable endpoints (i.e. where listener-class = `external-unstable` or `external-unstable`) are written to a ConfigMap called `-ui-endpoints`, listing each rolegroup by replica: [source,yaml] ---- -spec: - clusterConfig: - listenerClass: cluster-internal # <1> +apiVersion: v1 +data: + hbase.master-0.ui: 172.19.0.3:32353 + hbase.master-1.ui: 172.19.0.5:31817 + hbase.regionserver-0.ui: 172.19.0.3:31719 + hbase.regionserver-1.ui: 172.19.0.5:30626 + hbase.restserver-0.ui: 172.19.0.3:31790 + hbase.restserver-1.ui: 172.19.0.5:32292 +kind: ConfigMap +... ---- -<1> The default `cluster-internal` setting. diff --git a/rust/operator-binary/src/crd/mod.rs b/rust/operator-binary/src/crd/mod.rs index e13e3a2c..9d0663de 100644 --- a/rust/operator-binary/src/crd/mod.rs +++ b/rust/operator-binary/src/crd/mod.rs @@ -1,5 +1,10 @@ -use std::collections::{BTreeMap, HashMap}; +use std::{ + borrow::Cow, + collections::{BTreeMap, HashMap}, + num::TryFromIntError, +}; +use futures::future::try_join_all; use product_config::types::PropertyNameKind; use security::AuthenticationConfig; use serde::{Deserialize, Serialize}; @@ -9,6 +14,7 @@ use stackable_operator::{ commons::{ affinity::StackableAffinity, cluster_operation::ClusterOperation, + listener::Listener, product_image_selection::ProductImage, resources::{ CpuLimitsFragment, MemoryLimitsFragment, NoRuntimeLimits, NoRuntimeLimitsFragment, @@ -21,7 +27,7 @@ use stackable_operator::{ }, k8s_openapi::{ DeepMerge, - api::core::v1::{EnvVar, PodTemplateSpec}, + api::core::v1::{EnvVar, Pod, PodTemplateSpec}, apimachinery::pkg::api::resource::Quantity, }, kube::{CustomResource, ResourceExt, runtime::reflector::ObjectRef}, @@ -31,6 +37,7 @@ use stackable_operator::{ schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, time::Duration, + utils::cluster_info::KubernetesClusterInfo, versioned::versioned, }; use strum::{Display, EnumIter, EnumString}; @@ -81,6 +88,11 @@ pub const HBASE_REST_UI_PORT: u16 = 8085; // Newer versions use the same port as the UI because Hbase provides it's own metrics API pub const METRICS_PORT: u16 = 9100; +pub const DEFAULT_LISTENER_CLASS: SupportedListenerClasses = + SupportedListenerClasses::ClusterInternal; +pub const LISTENER_VOLUME_NAME: &str = "listener"; +pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener"; + const DEFAULT_REGION_MOVER_TIMEOUT: Duration = Duration::from_minutes_unchecked(59); const DEFAULT_REGION_MOVER_DELTA_TO_SHUTDOWN: Duration = Duration::from_minutes_unchecked(1); @@ -106,6 +118,29 @@ pub enum Error { #[snafu(display("incompatible merge types"))] IncompatibleMergeTypes, + + #[snafu(display("object has no associated namespace"))] + NoNamespace, + + #[snafu(display("unable to get {listener} (for {pod})"))] + GetPodListener { + source: stackable_operator::client::Error, + listener: ObjectRef, + pod: ObjectRef, + }, + + #[snafu(display("{listener} (for {pod}) has no address"))] + PodListenerHasNoAddress { + listener: ObjectRef, + pod: ObjectRef, + }, + + #[snafu(display("port {port} ({port_name:?}) is out of bounds, must be within {range:?}", range = 0..=u16::MAX))] + PortOutOfBounds { + source: TryFromIntError, + port_name: String, + port: i32, + }, } #[versioned(version(name = "v1alpha1"))] @@ -175,18 +210,6 @@ pub mod versioned { /// for a ZooKeeper cluster. pub zookeeper_config_map_name: String, - /// This field controls which type of Service the Operator creates for this HbaseCluster: - /// - /// * cluster-internal: Use a ClusterIP service - /// - /// * external-unstable: Use a NodePort service - /// - /// This is a temporary solution with the goal to keep yaml manifests forward compatible. - /// In the future, this setting will control which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) - /// will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change. - #[serde(default)] - pub listener_class: CurrentlySupportedListenerClasses, - /// Settings related to user [authentication](DOCS_BASE_URL_PLACEHOLDER/usage-guide/security). pub authentication: Option, @@ -539,7 +562,7 @@ impl v1alpha1::HbaseCluster { } /// Name of the port used by the Web UI, which depends on HTTPS usage - fn ui_port_name(&self) -> String { + pub fn ui_port_name(&self) -> String { if self.has_https_enabled() { HBASE_UI_PORT_NAME_HTTPS } else { @@ -547,6 +570,269 @@ impl v1alpha1::HbaseCluster { } .to_string() } + + pub fn rolegroup_ref( + &self, + role_name: impl Into, + group_name: impl Into, + ) -> RoleGroupRef { + RoleGroupRef { + cluster: ObjectRef::from_obj(self), + role: role_name.into(), + role_group: group_name.into(), + } + } + + /// Returns rolegroup and replica information for a specific role. + /// We can't pass through the merged config for a particular role-group + /// here as we need more than the config. As this will be called by role, + /// the merged listener-class is called so that only role-group information + /// for externally-reachable services (based on their listener class) are + /// included in the collection. + pub fn rolegroup_ref_and_replicas( + &self, + role: &HbaseRole, + ) -> Vec<(RoleGroupRef, u16)> { + match role { + HbaseRole::Master => self + .spec + .masters + .iter() + .flat_map(|role| &role.role_groups) + // Order rolegroups consistently, to avoid spurious downstream rewrites + .collect::>() + .into_iter() + .filter(|(rolegroup_name, _)| { + self.resolved_listener_class_discoverable(role, rolegroup_name) + }) + .map(|(rolegroup_name, role_group)| { + ( + self.rolegroup_ref(HbaseRole::Master.to_string(), rolegroup_name), + role_group.replicas.unwrap_or_default(), + ) + }) + .collect(), + HbaseRole::RegionServer => self + .spec + .region_servers + .iter() + .flat_map(|role| &role.role_groups) + // Order rolegroups consistently, to avoid spurious downstream rewrites + .collect::>() + .into_iter() + .filter(|(rolegroup_name, _)| { + self.resolved_listener_class_discoverable(role, rolegroup_name) + }) + .map(|(rolegroup_name, role_group)| { + ( + self.rolegroup_ref(HbaseRole::RegionServer.to_string(), rolegroup_name), + role_group.replicas.unwrap_or_default(), + ) + }) + .collect(), + HbaseRole::RestServer => self + .spec + .rest_servers + .iter() + .flat_map(|role| &role.role_groups) + // Order rolegroups consistently, to avoid spurious downstream rewrites + .collect::>() + .into_iter() + .filter(|(rolegroup_name, _)| { + self.resolved_listener_class_discoverable(role, rolegroup_name) + }) + .map(|(rolegroup_name, role_group)| { + ( + self.rolegroup_ref(HbaseRole::RestServer.to_string(), rolegroup_name), + role_group.replicas.unwrap_or_default(), + ) + }) + .collect(), + } + } + + fn resolved_listener_class_discoverable( + &self, + role: &HbaseRole, + rolegroup_name: &&String, + ) -> bool { + let listener_class = self.merged_listener_class(role, rolegroup_name); + if let Some(listener_class) = listener_class { + listener_class.discoverable() + } else { + false + } + } + + pub fn pod_refs( + &self, + role: &HbaseRole, + hbase_version: &str, + ) -> Result, Error> { + let ns = self.metadata.namespace.clone().context(NoNamespaceSnafu)?; + let rolegroup_ref_and_replicas = self.rolegroup_ref_and_replicas(role); + + Ok(rolegroup_ref_and_replicas + .iter() + .flat_map(|(rolegroup_ref, replicas)| { + let ns = ns.clone(); + (0..*replicas).map(move |i| HbasePodRef { + namespace: ns.clone(), + role_group_service_name: rolegroup_ref.object_name(), + pod_name: format!("{}-{}", rolegroup_ref.object_name(), i), + ports: self + .ports(role, hbase_version) + .iter() + .map(|(n, p)| (n.clone(), *p)) + .collect(), + fqdn_override: None, + }) + }) + .collect()) + } + + pub async fn listener_refs( + &self, + client: &stackable_operator::client::Client, + role: &HbaseRole, + hbase_version: &str, + ) -> Result, Error> { + let pod_refs = self.pod_refs(role, hbase_version)?; + try_join_all(pod_refs.into_iter().map(|pod_ref| async { + // N.B. use the naming convention for persistent listener volumes as we + // have specified above that we only want externally-reachable endpoints. + let listener_name = format!("{LISTENER_VOLUME_NAME}-{}", pod_ref.pod_name); + let listener_ref = + || ObjectRef::::new(&listener_name).within(&pod_ref.namespace); + let pod_obj_ref = + || ObjectRef::::new(&pod_ref.pod_name).within(&pod_ref.namespace); + let listener = client + .get::(&listener_name, &pod_ref.namespace) + .await + .context(GetPodListenerSnafu { + listener: listener_ref(), + pod: pod_obj_ref(), + })?; + let listener_address = listener + .status + .and_then(|s| s.ingress_addresses?.into_iter().next()) + .context(PodListenerHasNoAddressSnafu { + listener: listener_ref(), + pod: pod_obj_ref(), + })?; + Ok(HbasePodRef { + fqdn_override: Some(listener_address.address), + ports: listener_address + .ports + .into_iter() + .map(|(port_name, port)| { + let port = u16::try_from(port).context(PortOutOfBoundsSnafu { + port_name: &port_name, + port, + })?; + Ok((port_name, port)) + }) + .collect::>()?, + ..pod_ref + }) + })) + .await + } + + pub fn merged_listener_class( + &self, + role: &HbaseRole, + rolegroup_name: &String, + ) -> Option { + match role { + HbaseRole::Master => { + if let Some(masters) = self.spec.masters.as_ref() { + let conf_defaults = Some(SupportedListenerClasses::ClusterInternal); + let mut conf_role = masters.config.config.listener_class.to_owned(); + let mut conf_rolegroup = masters + .role_groups + .get(rolegroup_name) + .map(|rg| rg.config.config.listener_class.clone()) + .unwrap_or_default(); + + conf_role.merge(&conf_defaults); + conf_rolegroup.merge(&conf_role); + + tracing::debug!("Merged listener-class: {:?} for {role}", conf_rolegroup); + conf_rolegroup + } else { + None + } + } + HbaseRole::RegionServer => { + if let Some(region_servers) = self.spec.region_servers.as_ref() { + let conf_defaults = Some(SupportedListenerClasses::ClusterInternal); + let mut conf_role = region_servers.config.config.listener_class.to_owned(); + let mut conf_rolegroup = region_servers + .role_groups + .get(rolegroup_name) + .map(|rg| rg.config.config.listener_class.clone()) + .unwrap_or_default(); + + conf_role.merge(&conf_defaults); + conf_rolegroup.merge(&conf_role); + + tracing::debug!("Merged listener-class: {:?} for {role}", conf_rolegroup); + conf_rolegroup + } else { + None + } + } + HbaseRole::RestServer => { + if let Some(rest_servers) = self.spec.rest_servers.as_ref() { + let conf_defaults = Some(SupportedListenerClasses::ClusterInternal); + let mut conf_role = rest_servers.config.config.listener_class.to_owned(); + let mut conf_rolegroup = rest_servers + .role_groups + .get(rolegroup_name) + .map(|rg| rg.config.config.listener_class.clone()) + .unwrap_or_default(); + + conf_role.merge(&conf_defaults); + conf_rolegroup.merge(&conf_role); + + tracing::debug!("Merged listener-class: {:?} for {role}", conf_rolegroup); + conf_rolegroup + } else { + None + } + } + } + } +} + +/// Reference to a single `Pod` that is a component of a [`HbaseCluster`] +/// +/// Used for service discovery. +#[derive(Debug)] +pub struct HbasePodRef { + pub namespace: String, + pub role_group_service_name: String, + pub pod_name: String, + pub fqdn_override: Option, + pub ports: HashMap, +} + +impl HbasePodRef { + pub fn fqdn(&self, cluster_info: &KubernetesClusterInfo) -> Cow { + self.fqdn_override.as_deref().map_or_else( + || { + Cow::Owned(format!( + "{pod_name}.{role_group_service_name}.{namespace}.svc.{cluster_domain}", + pod_name = self.pod_name, + role_group_service_name = self.role_group_service_name, + namespace = self.namespace, + cluster_domain = cluster_info.cluster_domain, + )) + }, + Cow::Borrowed, + ) + } } pub fn merged_env(rolegroup_config: Option<&BTreeMap>) -> Vec { @@ -565,27 +851,6 @@ pub fn merged_env(rolegroup_config: Option<&BTreeMap>) -> Vec String { - match self { - CurrentlySupportedListenerClasses::ClusterInternal => "ClusterIP".to_string(), - CurrentlySupportedListenerClasses::ExternalUnstable => "NodePort".to_string(), - } - } -} - #[derive(Clone, Debug, Deserialize, Eq, Hash, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct KerberosConfig { @@ -709,6 +974,7 @@ impl HbaseRole { affinity: get_affinity(cluster_name, self, hdfs_discovery_cm_name), graceful_shutdown_timeout: Some(graceful_shutdown_timeout), requested_secret_lifetime: Some(requested_secret_lifetime), + listener_class: Some(DEFAULT_LISTENER_CLASS), } } @@ -809,6 +1075,7 @@ impl AnyConfigFragment { cli_opts: None, }, requested_secret_lifetime: Some(HbaseRole::DEFAULT_REGION_SECRET_LIFETIME), + listener_class: Some(DEFAULT_LISTENER_CLASS), }) } HbaseRole::RestServer => AnyConfigFragment::RestServer(HbaseConfigFragment { @@ -820,6 +1087,7 @@ impl AnyConfigFragment { HbaseRole::DEFAULT_REST_SERVER_GRACEFUL_SHUTDOWN_TIMEOUT, ), requested_secret_lifetime: Some(HbaseRole::DEFAULT_REST_SECRET_LIFETIME), + listener_class: Some(DEFAULT_LISTENER_CLASS), }), HbaseRole::Master => AnyConfigFragment::Master(HbaseConfigFragment { hbase_rootdir: None, @@ -830,6 +1098,7 @@ impl AnyConfigFragment { HbaseRole::DEFAULT_MASTER_GRACEFUL_SHUTDOWN_TIMEOUT, ), requested_secret_lifetime: Some(HbaseRole::DEFAULT_MASTER_SECRET_LIFETIME), + listener_class: Some(DEFAULT_LISTENER_CLASS), }), } } @@ -907,6 +1176,9 @@ pub struct HbaseConfig { /// Please note that this can be shortened by the `maxCertificateLifetime` setting on the SecretClass issuing the TLS certificate. #[fragment_attrs(serde(default))] pub requested_secret_lifetime: Option, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose this rolegroup. + pub listener_class: SupportedListenerClasses, } impl Configuration for HbaseConfigFragment { @@ -1060,6 +1332,9 @@ pub struct RegionServerConfig { /// The operator will compute a timeout period for the region move that will not exceed the graceful shutdown timeout. #[fragment_attrs(serde(default))] pub region_mover: RegionMover, + + /// This field controls which [ListenerClass](DOCS_BASE_URL_PLACEHOLDER/listener-operator/listenerclass.html) is used to expose this rolegroup. + pub listener_class: SupportedListenerClasses, } impl Configuration for RegionServerConfigFragment { @@ -1131,6 +1406,35 @@ impl Configuration for RegionServerConfigFragment { } } +#[derive(Clone, Debug, Default, Display, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] +#[serde(rename_all = "PascalCase")] +pub enum SupportedListenerClasses { + #[default] + #[serde(rename = "cluster-internal")] + #[strum(serialize = "cluster-internal")] + ClusterInternal, + + #[serde(rename = "external-unstable")] + #[strum(serialize = "external-unstable")] + ExternalUnstable, + + #[serde(rename = "external-stable")] + #[strum(serialize = "external-stable")] + ExternalStable, +} + +impl Atomic for SupportedListenerClasses {} + +impl SupportedListenerClasses { + pub fn discoverable(&self) -> bool { + match self { + SupportedListenerClasses::ClusterInternal => false, + SupportedListenerClasses::ExternalUnstable => true, + SupportedListenerClasses::ExternalStable => true, + } + } +} + #[derive(Clone, Debug, Default, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] #[serde(rename_all = "camelCase")] pub struct HbaseClusterStatus { @@ -1185,6 +1489,14 @@ impl AnyServiceConfig { } } + pub fn listener_class(&self) -> SupportedListenerClasses { + match self { + AnyServiceConfig::Master(config) => config.listener_class.clone(), + AnyServiceConfig::RegionServer(config) => config.listener_class.clone(), + AnyServiceConfig::RestServer(config) => config.listener_class.clone(), + } + } + /// Returns command line arguments to pass on to the region mover tool. /// The following arguments are excluded because they are already part of the /// hbase-entrypoint.sh script. diff --git a/rust/operator-binary/src/discovery.rs b/rust/operator-binary/src/discovery.rs index ffe9c388..a708591d 100644 --- a/rust/operator-binary/src/discovery.rs +++ b/rust/operator-binary/src/discovery.rs @@ -6,12 +6,12 @@ use stackable_operator::{ builder::{configmap::ConfigMapBuilder, meta::ObjectMetaBuilder}, commons::product_image_selection::ResolvedProductImage, k8s_openapi::api::core::v1::ConfigMap, - kube::runtime::reflector::ObjectRef, + kube::{ResourceExt, runtime::reflector::ObjectRef}, utils::cluster_info::KubernetesClusterInfo, }; use crate::{ - crd::{HBASE_SITE_XML, HbaseRole, v1alpha1}, + crd::{HBASE_SITE_XML, HbasePodRef, HbaseRole, v1alpha1}, hbase_controller::build_recommended_labels, kerberos::{self, kerberos_discovery_config_properties}, zookeeper::ZookeeperConnectionInformation, @@ -84,3 +84,51 @@ pub fn build_discovery_configmap( .build() .context(BuildConfigMapSnafu) } + +pub fn build_endpoint_configmap( + hbase: &v1alpha1::HbaseCluster, + resolved_product_image: &ResolvedProductImage, + role_podrefs: BTreeMap>, +) -> Result { + let name = hbase.name_unchecked(); + let mut cm = ConfigMapBuilder::new(); + + let cmm = cm.metadata( + ObjectMetaBuilder::new() + .name_and_namespace(hbase) + .name(format!("{name}-ui-endpoints")) + .ownerreference_from_resource(hbase, None, Some(true)) + .with_context(|_| ObjectMissingMetadataForOwnerRefSnafu { + hbase: ObjectRef::from_obj(hbase), + })? + .with_recommended_labels(build_recommended_labels( + hbase, + &resolved_product_image.app_version_label, + "hbase-ui", + "discovery", + )) + .context(ObjectMetaSnafu)? + .build(), + ); + + for role_podref in role_podrefs { + for podref in role_podref.1 { + if let HbasePodRef { + fqdn_override: Some(fqdn_override), + ports, + pod_name, + .. + } = podref + { + if let Some(ui_port) = ports.get(&hbase.ui_port_name()) { + cmm.add_data( + format!("{pod_name}.http"), + format!("{fqdn_override}:{ui_port}"), + ); + } + } + } + } + + cm.build().context(BuildConfigMapSnafu) +} diff --git a/rust/operator-binary/src/hbase_controller.rs b/rust/operator-binary/src/hbase_controller.rs index 376036c6..670f9b3e 100644 --- a/rust/operator-binary/src/hbase_controller.rs +++ b/rust/operator-binary/src/hbase_controller.rs @@ -8,6 +8,7 @@ use std::{ }; use const_format::concatcp; +use indoc::formatdoc; use product_config::{ ProductConfigManager, types::PropertyNameKind, @@ -20,8 +21,14 @@ use stackable_operator::{ configmap::ConfigMapBuilder, meta::ObjectMetaBuilder, pod::{ - PodBuilder, container::ContainerBuilder, resources::ResourceRequirementsBuilder, + PodBuilder, + container::ContainerBuilder, + resources::ResourceRequirementsBuilder, security::PodSecurityContextBuilder, + volume::{ + ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError, + ListenerReference, + }, }, }, cluster_resources::{ClusterResourceApplyStrategy, ClusterResources}, @@ -71,10 +78,11 @@ use crate::{ }, crd::{ APP_NAME, AnyServiceConfig, Container, HBASE_ENV_SH, HBASE_REST_PORT_NAME_HTTP, - HBASE_REST_PORT_NAME_HTTPS, HBASE_SITE_XML, HbaseClusterStatus, HbaseRole, - JVM_SECURITY_PROPERTIES_FILE, SSL_CLIENT_XML, SSL_SERVER_XML, merged_env, v1alpha1, + HBASE_REST_PORT_NAME_HTTPS, HBASE_SITE_XML, HbaseClusterStatus, HbasePodRef, HbaseRole, + JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME, SSL_CLIENT_XML, + SSL_SERVER_XML, merged_env, v1alpha1, }, - discovery::build_discovery_configmap, + discovery::{build_discovery_configmap, build_endpoint_configmap}, kerberos::{ self, add_kerberos_pod_config, kerberos_config_properties, kerberos_ssl_client_settings, kerberos_ssl_server_settings, @@ -313,6 +321,19 @@ pub enum Error { #[snafu(display("failed to construct JVM arguments"))] ConstructJvmArgument { source: crate::config::jvm::Error }, + + #[snafu(display("failed to build Labels"))] + LabelBuild { + source: stackable_operator::kvp::LabelError, + }, + + #[snafu(display("cannot collect discovery configuration"))] + CollectDiscoveryConfig { source: crate::crd::Error }, + + #[snafu(display("failed to build listener volume"))] + BuildListenerVolume { + source: ListenerOperatorVolumeSourceBuilderError, + }, } type Result = std::result::Result; @@ -376,26 +397,6 @@ pub async fn reconcile_hbase( ) .context(CreateClusterResourcesSnafu)?; - let region_server_role_service = - build_region_server_role_service(hbase, &resolved_product_image)?; - cluster_resources - .add(client, region_server_role_service) - .await - .context(ApplyRoleServiceSnafu)?; - - // discovery config map - let discovery_cm = build_discovery_configmap( - hbase, - &client.kubernetes_cluster_info, - &zookeeper_connection_information, - &resolved_product_image, - ) - .context(BuildDiscoveryConfigMapSnafu)?; - cluster_resources - .add(client, discovery_cm) - .await - .context(ApplyDiscoveryConfigMapSnafu)?; - let (rbac_sa, rbac_rolebinding) = build_rbac_resources( hbase, APP_NAME, @@ -414,6 +415,7 @@ pub async fn reconcile_hbase( .context(ApplyRoleBindingSnafu)?; let mut ss_cond_builder = StatefulSetConditionBuilder::default(); + let mut listener_refs: BTreeMap> = BTreeMap::new(); for (role_name, group_config) in validated_config.iter() { let hbase_role = HbaseRole::from_str(role_name).context(UnidentifiedHbaseRoleSnafu { @@ -483,8 +485,56 @@ pub async fn reconcile_hbase( .await .context(FailedToCreatePdbSnafu)?; } + + // if the replicas are changed at the same time as the reconciliation + // being paused, it may be possible to have listeners that are *expected* + // (according to their replica number) but which are not yet created, so + // deactivate this action in such cases. + if hbase.spec.cluster_operation.reconciliation_paused + || hbase.spec.cluster_operation.stopped + { + tracing::info!( + "Cluster is in a transitional state so do not attempt to collect listener information that will only be active once cluster has returned to a non-transitional state." + ); + } else { + listener_refs.insert( + hbase_role.to_string(), + hbase + .listener_refs(client, &hbase_role, &resolved_product_image.product_version) + .await + .context(CollectDiscoveryConfigSnafu)?, + ); + } } + tracing::debug!( + "Listener references prepared for the ConfigMap {:#?}", + listener_refs + ); + + if !listener_refs.is_empty() { + let endpoint_cm = build_endpoint_configmap(hbase, &resolved_product_image, listener_refs) + .context(BuildDiscoveryConfigMapSnafu)?; + cluster_resources + .add(client, endpoint_cm) + .await + .context(ApplyDiscoveryConfigMapSnafu)?; + } + + // Discovery CM will fail to build until the rest of the cluster has been deployed, so do it last + // so that failure won't inhibit the rest of the cluster from booting up. + let discovery_cm = build_discovery_configmap( + hbase, + &client.kubernetes_cluster_info, + &zookeeper_connection_information, + &resolved_product_image, + ) + .context(BuildDiscoveryConfigMapSnafu)?; + cluster_resources + .add(client, discovery_cm) + .await + .context(ApplyDiscoveryConfigMapSnafu)?; + let cluster_operation_cond_builder = ClusterOperationsConditionBuilder::new(&hbase.spec.cluster_operation); @@ -504,59 +554,6 @@ pub async fn reconcile_hbase( Ok(Action::await_change()) } -/// The server-role service is the primary endpoint that should be used by clients that do not perform internal load balancing, -/// including targets outside of the cluster. -pub fn build_region_server_role_service( - hbase: &v1alpha1::HbaseCluster, - resolved_product_image: &ResolvedProductImage, -) -> Result { - let role = HbaseRole::RegionServer; - let role_name = role.to_string(); - let role_svc_name = hbase - .server_role_service_name() - .context(GlobalServiceNameNotFoundSnafu)?; - let ports = hbase - .ports(&role, &resolved_product_image.product_version) - .into_iter() - .map(|(name, value)| ServicePort { - name: Some(name), - port: i32::from(value), - protocol: Some("TCP".to_string()), - ..ServicePort::default() - }) - .collect(); - - let metadata = ObjectMetaBuilder::new() - .name_and_namespace(hbase) - .name(&role_svc_name) - .ownerreference_from_resource(hbase, None, Some(true)) - .context(ObjectMissingMetadataForOwnerRefSnafu)? - .with_recommended_labels(build_recommended_labels( - hbase, - &resolved_product_image.app_version_label, - &role_name, - "global", - )) - .context(ObjectMetaSnafu)? - .build(); - - let service_selector_labels = - Labels::role_selector(hbase, APP_NAME, &role_name).context(BuildLabelSnafu)?; - - let service_spec = ServiceSpec { - type_: Some(hbase.spec.cluster_config.listener_class.k8s_service_type()), - ports: Some(ports), - selector: Some(service_selector_labels.into()), - ..ServiceSpec::default() - }; - - Ok(Service { - metadata, - spec: Some(service_spec), - status: None, - }) -} - /// The rolegroup [`ConfigMap`] configures the rolegroup based on the configuration given by the administrator #[allow(clippy::too_many_arguments)] fn build_rolegroup_config_map( @@ -591,6 +588,30 @@ fn build_rolegroup_config_map( hbase_site_config .extend(hbase_opa_config.map_or(vec![], |config| config.hbase_site_config())); + match hbase_role { + HbaseRole::Master => { + hbase_site_config.insert( + "hbase.listener.master.hostname".to_string(), + "${HBASE_SERVICE_HOST}".to_string(), + ); + hbase_site_config.insert( + "hbase.listener.master.port".to_string(), + "${HBASE_SERVICE_PORT}".to_string(), + ) + } + HbaseRole::RegionServer => { + hbase_site_config.insert( + "hbase.listener.regionserver.hostname".to_string(), + "${HBASE_SERVICE_HOST}".to_string(), + ); + hbase_site_config.insert( + "hbase.listener.regionserver.port".to_string(), + "${HBASE_SERVICE_PORT}".to_string(), + ) + } + HbaseRole::RestServer => None, + }; + // configOverride come last hbase_site_config.extend(config.clone()); hbase_site_xml = to_hadoop_xml( @@ -868,15 +889,19 @@ fn build_rolegroup_statefulset( }, ]); + let role_name = hbase_role.cli_role_name(); let mut hbase_container = ContainerBuilder::new("hbase").expect("ContainerBuilder not created"); hbase_container .image_from_product_image(resolved_product_image) - .command(vec!["/stackable/hbase/bin/hbase-entrypoint.sh".to_string()]) - .args(vec![ - hbase_role.cli_role_name(), - hbase_service_domain_name(hbase, rolegroup_ref, cluster_info)?, - hbase.service_port(hbase_role).to_string(), - ]) + .command(command()) + .args(vec![formatdoc! {" + {entrypoint} {role} {domain} {port} {port_name}", + entrypoint = "/stackable/hbase/bin/hbase-entrypoint.sh".to_string(), + role = role_name, + domain = hbase_service_domain_name(hbase, rolegroup_ref, cluster_info)?, + port = hbase.service_port(hbase_role).to_string(), + port_name = hbase.ui_port_name(), + }]) .add_env_vars(merged_env) // Needed for the `containerdebug` process to log it's tracing information to. .add_env_var( @@ -891,6 +916,8 @@ fn build_rolegroup_statefulset( .context(AddVolumeMountSnafu)? .add_volume_mount("log", STACKABLE_LOG_DIR) .context(AddVolumeMountSnafu)? + .add_volume_mount(LISTENER_VOLUME_NAME, LISTENER_VOLUME_DIR) + .context(AddVolumeMountSnafu)? .add_container_ports(ports) .resources(merged_config.resources().clone().into()) .startup_probe(startup_probe) @@ -899,13 +926,17 @@ fn build_rolegroup_statefulset( let mut pod_builder = PodBuilder::new(); + let recommended_object_labels = build_recommended_labels( + hbase, + hbase_version, + &rolegroup_ref.role, + &rolegroup_ref.role_group, + ); + let recommended_labels = + Labels::recommended(recommended_object_labels.clone()).context(LabelBuildSnafu)?; + let pb_metadata = ObjectMetaBuilder::new() - .with_recommended_labels(build_recommended_labels( - hbase, - hbase_version, - &rolegroup_ref.role, - &rolegroup_ref.role_group, - )) + .with_recommended_labels(recommended_object_labels) .context(ObjectMetaSnafu)? .build(); @@ -947,6 +978,28 @@ fn build_rolegroup_statefulset( .build(), ); + // externally-reachable listener endpoints should use a pvc volume... + let pvcs = if merged_config.listener_class().discoverable() { + let pvc = ListenerOperatorVolumeSourceBuilder::new( + &ListenerReference::ListenerClass(merged_config.listener_class().to_string()), + &recommended_labels, + ) + .context(BuildListenerVolumeSnafu)? + .build_pvc(LISTENER_VOLUME_NAME.to_string()) + .context(BuildListenerVolumeSnafu)?; + Some(vec![pvc]) + } else { + // ...whereas others will use ephemeral volumes + pod_builder + .add_listener_volume_by_listener_class( + LISTENER_VOLUME_NAME, + &merged_config.listener_class().to_string(), + &recommended_labels, + ) + .context(AddVolumeSnafu)?; + None + }; + if let Some(ContainerLogConfig { choice: Some(ContainerLogConfigChoice::Custom(CustomContainerLogConfig { @@ -1051,6 +1104,7 @@ fn build_rolegroup_statefulset( }, service_name: rolegroup_ref.object_name(), template: pod_template, + volume_claim_templates: pvcs, ..StatefulSetSpec::default() }; @@ -1061,6 +1115,17 @@ fn build_rolegroup_statefulset( }) } +/// Returns the container command. +fn command() -> Vec { + vec![ + "/bin/bash".to_string(), + "-x".to_string(), + "-euo".to_string(), + "pipefail".to_string(), + "-c".to_string(), + ] +} + fn write_hbase_env_sh<'a, T>(properties: T) -> String where T: Iterator, @@ -1123,6 +1188,7 @@ fn build_hbase_env_sh( let role_specific_non_heap_jvm_args = construct_role_specific_non_heap_jvm_args(hbase, hbase_role, role_group, product_version) .context(ConstructJvmArgumentSnafu)?; + match hbase_role { HbaseRole::Master => { result.insert( diff --git a/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 index 4732f740..6989c892 100644 --- a/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/03-install-hbase.yaml.j2 @@ -29,6 +29,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -37,6 +38,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -45,6 +47,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 index ffc8c7c0..0431cf88 100644 --- a/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/10-pause-hbase.yaml.j2 @@ -32,6 +32,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -40,6 +41,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -48,6 +50,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 2 # ignored because reconciliation is paused diff --git a/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 index 0f4c5665..8bc4007f 100644 --- a/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/20-stop-hbase.yaml.j2 @@ -32,6 +32,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -40,6 +41,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -48,6 +50,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 # set to 0 by the operator because cluster is stopped diff --git a/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 b/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 index 388110b2..9a29aff5 100644 --- a/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 +++ b/tests/templates/kuttl/cluster-operation/30-restart-hbase.yaml.j2 @@ -32,6 +32,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -40,6 +41,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 @@ -48,6 +50,7 @@ spec: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-stable roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/external-access/00-assert.yaml.j2 b/tests/templates/kuttl/external-access/00-assert.yaml.j2 new file mode 100644 index 00000000..50b1d4c3 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-assert.yaml.j2 @@ -0,0 +1,10 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 b/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 new file mode 100644 index 00000000..2d6a0df5 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-install-vector-aggregator-discovery-configmap.yaml.j2 @@ -0,0 +1,9 @@ +{% if lookup('env', 'VECTOR_AGGREGATOR') %} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: vector-aggregator-discovery +data: + ADDRESS: {{ lookup('env', 'VECTOR_AGGREGATOR') }} +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 new file mode 100644 index 00000000..67185acf --- /dev/null +++ b/tests/templates/kuttl/external-access/00-patch-ns.yaml.j2 @@ -0,0 +1,9 @@ +{% if test_scenario['values']['openshift'] == 'true' %} +# see https://github.com/stackabletech/issues/issues/566 +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: kubectl patch namespace $NAMESPACE -p '{"metadata":{"labels":{"pod-security.kubernetes.io/enforce":"privileged"}}}' + timeout: 120 +{% endif %} diff --git a/tests/templates/kuttl/external-access/00-rbac.yaml.j2 b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 new file mode 100644 index 00000000..7ee61d23 --- /dev/null +++ b/tests/templates/kuttl/external-access/00-rbac.yaml.j2 @@ -0,0 +1,29 @@ +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-role +rules: +{% if test_scenario['values']['openshift'] == "true" %} + - apiGroups: ["security.openshift.io"] + resources: ["securitycontextconstraints"] + resourceNames: ["privileged"] + verbs: ["use"] +{% endif %} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: test-sa +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: test-rb +subjects: + - kind: ServiceAccount + name: test-sa +roleRef: + kind: Role + name: test-role + apiGroup: rbac.authorization.k8s.io diff --git a/tests/templates/kuttl/external-access/01-assert.yaml b/tests/templates/kuttl/external-access/01-assert.yaml new file mode 100644 index 00000000..e0766c49 --- /dev/null +++ b/tests/templates/kuttl/external-access/01-assert.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-zk-server-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/01-install-zookeeper.yaml.j2 b/tests/templates/kuttl/external-access/01-install-zookeeper.yaml.j2 new file mode 100644 index 00000000..0a331d50 --- /dev/null +++ b/tests/templates/kuttl/external-access/01-install-zookeeper.yaml.j2 @@ -0,0 +1,29 @@ +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperCluster +metadata: + name: test-zk +spec: + image: + productVersion: "{{ test_scenario['values']['zookeeper-latest'] }}" + pullPolicy: IfNotPresent +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + clusterConfig: + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + servers: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 +--- +apiVersion: zookeeper.stackable.tech/v1alpha1 +kind: ZookeeperZnode +metadata: + name: test-znode +spec: + clusterRef: + name: test-zk diff --git a/tests/templates/kuttl/external-access/02-assert.yaml b/tests/templates/kuttl/external-access/02-assert.yaml new file mode 100644 index 00000000..99b25f8e --- /dev/null +++ b/tests/templates/kuttl/external-access/02-assert.yaml @@ -0,0 +1,28 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hdfs-namenode-default +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hdfs-journalnode-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hdfs-datanode-default +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/02-install-hdfs.yaml.j2 b/tests/templates/kuttl/external-access/02-install-hdfs.yaml.j2 new file mode 100644 index 00000000..f9194a60 --- /dev/null +++ b/tests/templates/kuttl/external-access/02-install-hdfs.yaml.j2 @@ -0,0 +1,39 @@ +--- +apiVersion: hdfs.stackable.tech/v1alpha1 +kind: HdfsCluster +metadata: + name: test-hdfs +spec: + image: + productVersion: "{{ test_scenario['values']['hdfs-latest'] }}" + pullPolicy: IfNotPresent + clusterConfig: + dfsReplication: 1 + zookeeperConfigMapName: test-znode +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + nameNodes: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 2 + dataNodes: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 + journalNodes: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + roleGroups: + default: + replicas: 1 diff --git a/tests/templates/kuttl/external-access/03-assert.yaml b/tests/templates/kuttl/external-access/03-assert.yaml new file mode 100644 index 00000000..b9964df8 --- /dev/null +++ b/tests/templates/kuttl/external-access/03-assert.yaml @@ -0,0 +1,44 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-master-default +status: + readyReplicas: 2 + replicas: 2 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-regionserver-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-regionserver-cluster-internal +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-restserver-default +status: + readyReplicas: 1 + replicas: 1 +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: test-hbase-restserver-external-unstable +status: + readyReplicas: 1 + replicas: 1 diff --git a/tests/templates/kuttl/external-access/03-install-hbase.yaml.j2 b/tests/templates/kuttl/external-access/03-install-hbase.yaml.j2 new file mode 100644 index 00000000..249d1643 --- /dev/null +++ b/tests/templates/kuttl/external-access/03-install-hbase.yaml.j2 @@ -0,0 +1,55 @@ +--- +apiVersion: hbase.stackable.tech/v1alpha1 +kind: HbaseCluster +metadata: + name: test-hbase +spec: + image: +{% if test_scenario['values']['hbase'].find(",") > 0 %} + custom: "{{ test_scenario['values']['hbase'].split(',')[1] }}" + productVersion: "{{ test_scenario['values']['hbase'].split(',')[0] }}" +{% else %} + productVersion: "{{ test_scenario['values']['hbase'] }}" +{% endif %} + pullPolicy: IfNotPresent + clusterConfig: + hdfsConfigMapName: test-hdfs-namenode-default + zookeeperConfigMapName: test-znode +{% if lookup('env', 'VECTOR_AGGREGATOR') %} + vectorAggregatorConfigMapName: vector-aggregator-discovery +{% endif %} + masters: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-unstable + roleGroups: + default: + replicas: 2 + regionServers: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: external-unstable + roleGroups: + default: + replicas: 1 + cluster-internal: + replicas: 1 + config: + listenerClass: cluster-internal + restServers: + config: + gracefulShutdownTimeout: 1m + logging: + enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: cluster-internal + roleGroups: + default: + replicas: 1 + external-unstable: + replicas: 1 + config: + listenerClass: external-unstable diff --git a/tests/templates/kuttl/external-access/30-access-hbase.txt.j2 b/tests/templates/kuttl/external-access/30-access-hbase.txt.j2 new file mode 100644 index 00000000..ab749159 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-access-hbase.txt.j2 @@ -0,0 +1,75 @@ +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: access-hbase +spec: + template: + spec: + serviceAccountName: test-sa + containers: + - name: access-hbase +{% if test_scenario['values']['hbase'].find(",") > 0 %} + image: "{{ test_scenario['values']['hbase'].split(',')[1] }}" +{% else %} + image: oci.stackable.tech/sdp/hbase:{{ test_scenario['values']['hbase'] }}-stackable0.0.0-dev +{% endif %} + imagePullPolicy: IfNotPresent + command: + - /bin/bash + - /tmp/script/script.sh + env: + - name: MASTER_UI_0 + valueFrom: + configMapKeyRef: + name: test-hbase-ui-endpoints + key: test-hbase-master-default-0.http + - name: MASTER_UI_1 + valueFrom: + configMapKeyRef: + name: test-hbase-ui-endpoints + key: test-hbase-master-default-0.http + - name: REGIONSERVER_UI_0 + valueFrom: + configMapKeyRef: + name: test-hbase-ui-endpoints + key: test-hbase-regionserver-default-0.http + - name: RESTSERVER_UI_0 + valueFrom: + configMapKeyRef: + name: test-hbase-ui-endpoints + key: test-hbase-restserver-external-unstable-0.http + volumeMounts: + - name: script + mountPath: /tmp/script + volumes: + - name: script + configMap: + name: access-hbase-script + securityContext: + fsGroup: 1000 + runAsGroup: 1000 + runAsUser: 1000 + restartPolicy: OnFailure +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: access-hbase-script +data: + script.sh: | + set -euxo pipefail + + echo "Attempting to reach master at $MASTER_UI_0..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${MASTER_UI_0}" | grep 200 + + echo "Attempting to reach master at $MASTER_UI_1..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${MASTER_UI_1}" | grep 200 + + echo "Attempting to reach region-server at $REGIONSERVER_UI_0..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${REGIONSERVER_UI_0}" | grep 200 + + echo "Attempting to reach rest-server at $RESTSERVER_UI_0..." + curl --retry 0 -f -s -o /dev/null -w "%{http_code}" "${RESTSERVER_UI_0}" | grep 200 + + echo "All tests successful!" diff --git a/tests/templates/kuttl/external-access/30-access-hbase.yaml b/tests/templates/kuttl/external-access/30-access-hbase.yaml new file mode 100644 index 00000000..353e30ee --- /dev/null +++ b/tests/templates/kuttl/external-access/30-access-hbase.yaml @@ -0,0 +1,6 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + # We need to replace $NAMESPACE (by KUTTL) + - script: envsubst '$NAMESPACE' < 30-access-hbase.txt | kubectl apply -n $NAMESPACE -f - diff --git a/tests/templates/kuttl/external-access/30-assert.yaml b/tests/templates/kuttl/external-access/30-assert.yaml new file mode 100644 index 00000000..763b8a40 --- /dev/null +++ b/tests/templates/kuttl/external-access/30-assert.yaml @@ -0,0 +1,11 @@ +--- +apiVersion: kuttl.dev/v1beta1 +kind: TestAssert +timeout: 600 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: access-hbase +status: + succeeded: 1 diff --git a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 index 28766a48..a21046d1 100644 --- a/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/kerberos/30-install-hbase.yaml.j2 @@ -28,7 +28,6 @@ commands: clusterConfig: hdfsConfigMapName: hdfs zookeeperConfigMapName: hbase-znode - listenerClass: {{ test_scenario['values']['listener-class'] }} authentication: tlsSecretClass: tls kerberos: @@ -41,6 +40,7 @@ commands: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} resources: memory: limit: 1536Mi @@ -52,6 +52,7 @@ commands: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 2 @@ -60,6 +61,7 @@ commands: gracefulShutdownTimeout: 1m logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: replicas: 1 diff --git a/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 b/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 index 92fda28c..b2d01a8f 100644 --- a/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/opa/30-install-hbase.yaml.j2 @@ -51,7 +51,6 @@ commands: clusterConfig: hdfsConfigMapName: hdfs zookeeperConfigMapName: hbase-znode - listenerClass: 'cluster-internal' authentication: tlsSecretClass: tls kerberos: diff --git a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 index 53e9a98e..7535a3e8 100644 --- a/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 +++ b/tests/templates/kuttl/smoke/30-install-hbase.yaml.j2 @@ -15,7 +15,6 @@ spec: clusterConfig: hdfsConfigMapName: test-hdfs zookeeperConfigMapName: test-znode - listenerClass: {{ test_scenario['values']['listener-class'] }} {% if lookup('env', 'VECTOR_AGGREGATOR') %} vectorAggregatorConfigMapName: vector-aggregator-discovery {% endif %} @@ -23,6 +22,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: configOverrides: @@ -34,6 +34,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} roleGroups: default: configOverrides: @@ -45,6 +46,7 @@ spec: config: logging: enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} + listenerClass: {{ test_scenario['values']['listener-class'] }} resources: memory: limit: 1Gi diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index ba37dcd1..b7afd697 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -142,6 +142,12 @@ tests: - hdfs-latest - zookeeper-latest - openshift + - name: external-access + dimensions: + - hbase + - hdfs-latest + - zookeeper-latest + - openshift suites: - name: nightly patch: