Skip to content

Commit 6898b3f

Browse files
soenkeliebaufhennigsbernauer
authored
chore(deps): update all dependencies to latest versions (#528)
* A whole lot of dependency updates, the major ones are: - built -> 0.7.2 - strum -> 0.26 - snafu -> 0.8 - operator-rs -> 0.67 (breaking changes: https://github.com/stackabletech/operator-rs/blob/main/crates/stackable-operator/CHANGELOG.md) * Fix syntax errors in tests * Regenerated charts * Update Cargo.toml Co-authored-by: Felix Hennig <fhennig@users.noreply.github.com> * Update Cargo.toml Co-authored-by: Sebastian Bernauer <sebastian.bernauer@stackable.de> * Update rust/crd/src/lib.rs Co-authored-by: Sebastian Bernauer <sebastian.bernauer@stackable.de> --------- Co-authored-by: Felix Hennig <fhennig@users.noreply.github.com> Co-authored-by: Sebastian Bernauer <sebastian.bernauer@stackable.de>
1 parent 260484d commit 6898b3f

File tree

15 files changed

+1807
-1131
lines changed

15 files changed

+1807
-1131
lines changed

‎Cargo.lock

Lines changed: 616 additions & 582 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -11,20 +11,20 @@ repository = "https://github.com/stackabletech/hdfs-operator"
1111

1212
[workspace.dependencies]
1313
anyhow = "1.0"
14-
built = { version = "0.6", features = ["chrono", "git2"] }
14+
built = { version = "0.7", features = ["chrono", "git2"] }
1515
clap = "4.3"
1616
futures = { version = "0.3", features = ["compat"] }
1717
indoc = "2.0"
18-
rstest = "0.18"
18+
rstest = "0.19"
1919
semver = "1.0"
2020
serde = { version = "1.0", features = ["derive"] }
2121
serde_json = "1.0"
2222
serde_yaml = "0.9"
23-
snafu = "0.7"
24-
stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "0.64.0" }
23+
snafu = "0.8"
24+
stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.67.0" }
2525
product-config = { git = "https://github.com/stackabletech/product-config.git", tag = "0.6.0" }
26-
strum = { version = "0.25", features = ["derive"] }
27-
tokio = { version = "1.29", features = ["full"] }
26+
strum = { version = "0.26", features = ["derive"] }
27+
tokio = { version = "1.37", features = ["full"] }
2828
tracing = "0.1"
2929
tracing-futures = { version = "0.2", features = ["futures-03"] }
3030

‎deploy/helm/hdfs-operator/crds/crds.yaml

Lines changed: 1105 additions & 469 deletions
Large diffs are not rendered by default.

‎rust/crd/src/affinity.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -93,6 +93,7 @@ spec:
9393
namespace_selector: None,
9494
namespaces: None,
9595
topology_key: "kubernetes.io/hostname".to_string(),
96+
..PodAffinityTerm::default()
9697
},
9798
weight: 20
9899
}
@@ -120,6 +121,7 @@ spec:
120121
namespace_selector: None,
121122
namespaces: None,
122123
topology_key: "kubernetes.io/hostname".to_string(),
124+
..PodAffinityTerm::default()
123125
},
124126
weight: 70
125127
}

‎rust/crd/src/lib.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ use stackable_operator::{
3333
},
3434
kube::{runtime::reflector::ObjectRef, CustomResource, ResourceExt},
3535
kvp::{LabelError, Labels},
36-
product_config_utils::{ConfigError, Configuration},
36+
product_config_utils::{Configuration, Error as ConfigError},
3737
product_logging,
3838
product_logging::spec::{ContainerLogConfig, Logging},
3939
role_utils::{GenericRoleConfig, Role, RoleGroup, RoleGroupRef},
@@ -79,7 +79,7 @@ pub enum Error {
7979

8080
#[snafu(display("unable to get {listener} (for {pod})"))]
8181
GetPodListener {
82-
source: stackable_operator::error::Error,
82+
source: stackable_operator::client::Error,
8383
listener: ObjectRef<Listener>,
8484
pod: ObjectRef<Pod>,
8585
},

‎rust/crd/src/storage.rs

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -182,10 +182,11 @@ impl HdfsStorageType {
182182
mod test {
183183
use std::collections::BTreeMap;
184184

185+
use stackable_operator::k8s_openapi::api::core::v1::VolumeResourceRequirements;
185186
use stackable_operator::{
186187
commons::resources::PvcConfig,
187188
k8s_openapi::{
188-
api::core::v1::{PersistentVolumeClaimSpec, ResourceRequirements},
189+
api::core::v1::PersistentVolumeClaimSpec,
189190
apimachinery::pkg::{api::resource::Quantity, apis::meta::v1::LabelSelector},
190191
},
191192
};
@@ -217,12 +218,12 @@ mod test {
217218
assert_eq!(
218219
pvcs[0].spec,
219220
Some(PersistentVolumeClaimSpec {
220-
resources: Some(ResourceRequirements {
221+
resources: Some(VolumeResourceRequirements {
221222
requests: Some(BTreeMap::from([(
222223
"storage".to_string(),
223224
Quantity("5Gi".to_string())
224225
)])),
225-
..ResourceRequirements::default()
226+
..VolumeResourceRequirements::default()
226227
}),
227228
access_modes: Some(vec!["ReadWriteOnce".to_string()]),
228229
storage_class_name: None,
@@ -276,12 +277,12 @@ mod test {
276277
assert_eq!(
277278
pvcs[0].spec,
278279
Some(PersistentVolumeClaimSpec {
279-
resources: Some(ResourceRequirements {
280+
resources: Some(VolumeResourceRequirements {
280281
requests: Some(BTreeMap::from([(
281282
"storage".to_string(),
282283
Quantity("12Ti".to_string())
283284
)])),
284-
..ResourceRequirements::default()
285+
..VolumeResourceRequirements::default()
285286
}),
286287
access_modes: Some(vec!["ReadWriteOnce".to_string()]),
287288
storage_class_name: Some("hdd-storage-class".to_string()),

‎rust/operator-binary/build.rs

Lines changed: 1 addition & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,3 @@
1-
use std::path::PathBuf;
2-
31
fn main() {
4-
let out_dir = PathBuf::from(std::env::var("OUT_DIR").expect("OUT_DIR is required"));
5-
built::write_built_file_with_opts(
6-
// built's env module depends on a whole bunch of variables that crate2nix doesn't provide
7-
// so we grab the specific env variables that we care about out ourselves instead.
8-
built::Options::default().set_env(false),
9-
"Cargo.toml".as_ref(),
10-
&out_dir.join("built.rs"),
11-
)
12-
.unwrap();
2+
built::write_built_file().unwrap();
133
}

‎rust/operator-binary/src/container.rs

Lines changed: 28 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -9,30 +9,24 @@
99
//! - Set resources
1010
//! - Add tcp probes and container ports (to the main containers)
1111
//!
12+
use crate::DATANODE_ROOT_DATA_DIR_PREFIX;
13+
use crate::JVM_SECURITY_PROPERTIES_FILE;
14+
use crate::LOG4J_PROPERTIES;
15+
use stackable_operator::utils::COMMON_BASH_TRAP_FUNCTIONS;
1216
use std::{collections::BTreeMap, str::FromStr};
1317

1418
use indoc::formatdoc;
1519
use snafu::{OptionExt, ResultExt, Snafu};
16-
use stackable_hdfs_crd::{
17-
constants::{
18-
DATANODE_ROOT_DATA_DIR_PREFIX, DEFAULT_DATA_NODE_METRICS_PORT,
19-
DEFAULT_JOURNAL_NODE_METRICS_PORT, DEFAULT_NAME_NODE_METRICS_PORT,
20-
JVM_SECURITY_PROPERTIES_FILE, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME,
21-
LIVENESS_PROBE_FAILURE_THRESHOLD, LIVENESS_PROBE_INITIAL_DELAY_SECONDS,
22-
LIVENESS_PROBE_PERIOD_SECONDS, LOG4J_PROPERTIES, NAMENODE_ROOT_DATA_DIR,
23-
READINESS_PROBE_FAILURE_THRESHOLD, READINESS_PROBE_INITIAL_DELAY_SECONDS,
24-
READINESS_PROBE_PERIOD_SECONDS, SERVICE_PORT_NAME_HTTP, SERVICE_PORT_NAME_HTTPS,
25-
SERVICE_PORT_NAME_IPC, SERVICE_PORT_NAME_RPC, STACKABLE_ROOT_DATA_DIR,
26-
},
27-
storage::DataNodeStorageConfig,
28-
AnyNodeConfig, DataNodeContainer, HdfsCluster, HdfsPodRef, HdfsRole, NameNodeContainer,
29-
};
3020
use stackable_operator::{
3121
builder::{
32-
resources::ResourceRequirementsBuilder, ContainerBuilder,
33-
ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError,
34-
ListenerReference, PodBuilder, SecretFormat, SecretOperatorVolumeSourceBuilder,
35-
SecretOperatorVolumeSourceBuilderError, VolumeBuilder, VolumeMountBuilder,
22+
pod::container::ContainerBuilder,
23+
pod::resources::ResourceRequirementsBuilder,
24+
pod::volume::{
25+
ListenerOperatorVolumeSourceBuilder, ListenerOperatorVolumeSourceBuilderError,
26+
ListenerReference, SecretFormat, SecretOperatorVolumeSourceBuilder,
27+
SecretOperatorVolumeSourceBuilderError, VolumeBuilder, VolumeMountBuilder,
28+
},
29+
pod::PodBuilder,
3630
},
3731
commons::product_image_selection::ResolvedProductImage,
3832
k8s_openapi::{
@@ -56,10 +50,23 @@ use stackable_operator::{
5650
CustomContainerLogConfig,
5751
},
5852
},
59-
utils::COMMON_BASH_TRAP_FUNCTIONS,
6053
};
6154
use strum::{Display, EnumDiscriminants, IntoStaticStr};
6255

56+
use stackable_hdfs_crd::{
57+
constants::{
58+
DEFAULT_DATA_NODE_METRICS_PORT, DEFAULT_JOURNAL_NODE_METRICS_PORT,
59+
DEFAULT_NAME_NODE_METRICS_PORT, LISTENER_VOLUME_DIR, LISTENER_VOLUME_NAME,
60+
LIVENESS_PROBE_FAILURE_THRESHOLD, LIVENESS_PROBE_INITIAL_DELAY_SECONDS,
61+
LIVENESS_PROBE_PERIOD_SECONDS, NAMENODE_ROOT_DATA_DIR, READINESS_PROBE_FAILURE_THRESHOLD,
62+
READINESS_PROBE_INITIAL_DELAY_SECONDS, READINESS_PROBE_PERIOD_SECONDS,
63+
SERVICE_PORT_NAME_HTTP, SERVICE_PORT_NAME_HTTPS, SERVICE_PORT_NAME_IPC,
64+
SERVICE_PORT_NAME_RPC, STACKABLE_ROOT_DATA_DIR,
65+
},
66+
storage::DataNodeStorageConfig,
67+
AnyNodeConfig, DataNodeContainer, HdfsCluster, HdfsPodRef, HdfsRole, NameNodeContainer,
68+
};
69+
6370
use crate::product_logging::{
6471
FORMAT_NAMENODES_LOG4J_CONFIG_FILE, FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, HDFS_LOG4J_CONFIG_FILE,
6572
MAX_FORMAT_NAMENODE_LOG_FILE_SIZE, MAX_FORMAT_ZOOKEEPER_LOG_FILE_SIZE, MAX_HDFS_LOG_FILE_SIZE,
@@ -82,7 +89,7 @@ pub enum Error {
8289

8390
#[snafu(display("invalid java heap config for {role:?}"))]
8491
InvalidJavaHeapConfig {
85-
source: stackable_operator::error::Error,
92+
source: stackable_operator::memory::Error,
8693
role: String,
8794
},
8895

@@ -91,7 +98,7 @@ pub enum Error {
9198

9299
#[snafu(display("invalid container name {name:?}"))]
93100
InvalidContainerName {
94-
source: stackable_operator::error::Error,
101+
source: stackable_operator::builder::pod::container::Error,
95102
name: String,
96103
},
97104

‎rust/operator-binary/src/discovery.rs

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ use stackable_hdfs_crd::{
44
HdfsCluster, HdfsPodRef, HdfsRole,
55
};
66
use stackable_operator::{
7-
builder::{ConfigMapBuilder, ObjectMetaBuilder, ObjectMetaBuilderError},
7+
builder::configmap::ConfigMapBuilder,
8+
builder::meta::ObjectMetaBuilder,
89
commons::product_image_selection::ResolvedProductImage,
910
k8s_openapi::api::core::v1::ConfigMap,
1011
kube::{runtime::reflector::ObjectRef, ResourceExt},
@@ -23,17 +24,19 @@ type Result<T, E = Error> = std::result::Result<T, E>;
2324
pub enum Error {
2425
#[snafu(display("object {hdfs} is missing metadata to build owner reference"))]
2526
ObjectMissingMetadataForOwnerRef {
26-
source: stackable_operator::error::Error,
27+
source: stackable_operator::builder::meta::Error,
2728
hdfs: ObjectRef<HdfsCluster>,
2829
},
2930

3031
#[snafu(display("failed to build ConfigMap"))]
3132
BuildConfigMap {
32-
source: stackable_operator::error::Error,
33+
source: stackable_operator::builder::configmap::Error,
3334
},
3435

3536
#[snafu(display("failed to build object meta data"))]
36-
ObjectMeta { source: ObjectMetaBuilderError },
37+
ObjectMeta {
38+
source: stackable_operator::builder::meta::Error,
39+
},
3740

3841
#[snafu(display("failed to build security discovery config map"))]
3942
BuildSecurityDiscoveryConfigMap { source: kerberos::Error },

‎rust/operator-binary/src/hdfs_controller.rs

Lines changed: 25 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -10,13 +10,11 @@ use product_config::{
1010
ProductConfigManager,
1111
};
1212
use snafu::{OptionExt, ResultExt, Snafu};
13-
use stackable_hdfs_crd::{
14-
constants::*, AnyNodeConfig, HdfsCluster, HdfsClusterStatus, HdfsPodRef, HdfsRole,
15-
};
1613
use stackable_operator::{
1714
builder::{
18-
ConfigMapBuilder, ObjectMetaBuilder, ObjectMetaBuilderError, PodBuilder,
19-
PodSecurityContextBuilder,
15+
configmap::ConfigMapBuilder,
16+
meta::ObjectMetaBuilder,
17+
pod::{security::PodSecurityContextBuilder, PodBuilder},
2018
},
2119
client::Client,
2220
cluster_resources::{ClusterResourceApplyStrategy, ClusterResources},
@@ -49,6 +47,10 @@ use stackable_operator::{
4947
};
5048
use strum::{EnumDiscriminants, IntoStaticStr};
5149

50+
use stackable_hdfs_crd::{
51+
constants::*, AnyNodeConfig, HdfsCluster, HdfsClusterStatus, HdfsPodRef, HdfsRole,
52+
};
53+
5254
use crate::{
5355
build_recommended_labels,
5456
config::{CoreSiteConfigBuilder, HdfsSiteConfigBuilder},
@@ -73,41 +75,41 @@ const DOCKER_IMAGE_BASE_NAME: &str = "hadoop";
7375
pub enum Error {
7476
#[snafu(display("invalid role configuration"))]
7577
InvalidRoleConfig {
76-
source: stackable_operator::product_config_utils::ConfigError,
78+
source: stackable_operator::product_config_utils::Error,
7779
},
7880

7981
#[snafu(display("invalid product configuration"))]
8082
InvalidProductConfig {
81-
source: stackable_operator::error::Error,
83+
source: stackable_operator::product_config_utils::Error,
8284
},
8385

8486
#[snafu(display("cannot create rolegroup service {name:?}"))]
8587
ApplyRoleGroupService {
86-
source: stackable_operator::error::Error,
88+
source: stackable_operator::cluster_resources::Error,
8789
name: String,
8890
},
8991

9092
#[snafu(display("cannot create role group config map {name:?}"))]
9193
ApplyRoleGroupConfigMap {
92-
source: stackable_operator::error::Error,
94+
source: stackable_operator::cluster_resources::Error,
9395
name: String,
9496
},
9597

9698
#[snafu(display("cannot create role group stateful set {name:?}"))]
9799
ApplyRoleGroupStatefulSet {
98-
source: stackable_operator::error::Error,
100+
source: stackable_operator::cluster_resources::Error,
99101
name: String,
100102
},
101103

102104
#[snafu(display("cannot create discovery config map {name:?}"))]
103105
ApplyDiscoveryConfigMap {
104-
source: stackable_operator::error::Error,
106+
source: stackable_operator::client::Error,
105107
name: String,
106108
},
107109

108110
#[snafu(display("no metadata for {obj_ref:?}"))]
109111
ObjectMissingMetadataForOwnerRef {
110-
source: stackable_operator::error::Error,
112+
source: stackable_operator::builder::meta::Error,
111113
obj_ref: ObjectRef<HdfsCluster>,
112114
},
113115

@@ -122,7 +124,7 @@ pub enum Error {
122124

123125
#[snafu(display("cannot build config map for role {role:?} and role group {role_group:?}"))]
124126
BuildRoleGroupConfigMap {
125-
source: stackable_operator::error::Error,
127+
source: stackable_operator::builder::configmap::Error,
126128
role: String,
127129
role_group: String,
128130
},
@@ -135,22 +137,22 @@ pub enum Error {
135137

136138
#[snafu(display("failed to patch service account"))]
137139
ApplyServiceAccount {
138-
source: stackable_operator::error::Error,
140+
source: stackable_operator::cluster_resources::Error,
139141
},
140142

141143
#[snafu(display("failed to patch role binding"))]
142144
ApplyRoleBinding {
143-
source: stackable_operator::error::Error,
145+
source: stackable_operator::cluster_resources::Error,
144146
},
145147

146148
#[snafu(display("failed to create cluster resources"))]
147149
CreateClusterResources {
148-
source: stackable_operator::error::Error,
150+
source: stackable_operator::cluster_resources::Error,
149151
},
150152

151153
#[snafu(display("failed to delete orphaned resources"))]
152154
DeleteOrphanedResources {
153-
source: stackable_operator::error::Error,
155+
source: stackable_operator::cluster_resources::Error,
154156
},
155157

156158
#[snafu(display("failed to create pod references"))]
@@ -186,12 +188,12 @@ pub enum Error {
186188

187189
#[snafu(display("failed to update status"))]
188190
ApplyStatus {
189-
source: stackable_operator::error::Error,
191+
source: stackable_operator::client::Error,
190192
},
191193

192194
#[snafu(display("failed to build RBAC resources"))]
193195
BuildRbacResources {
194-
source: stackable_operator::error::Error,
196+
source: stackable_operator::commons::rbac::Error,
195197
},
196198

197199
#[snafu(display(
@@ -222,7 +224,9 @@ pub enum Error {
222224
BuildRoleGroupVolumeClaimTemplates { source: container::Error },
223225

224226
#[snafu(display("failed to build object meta data"))]
225-
ObjectMeta { source: ObjectMetaBuilderError },
227+
ObjectMeta {
228+
source: stackable_operator::builder::meta::Error,
229+
},
226230

227231
#[snafu(display("failed to build security config"))]
228232
BuildSecurityConfig { source: kerberos::Error },
@@ -251,7 +255,7 @@ pub async fn reconcile_hdfs(hdfs: Arc<HdfsCluster>, ctx: Arc<Ctx>) -> HdfsOperat
251255
let resolved_product_image = hdfs
252256
.spec
253257
.image
254-
.resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::CARGO_PKG_VERSION);
258+
.resolve(DOCKER_IMAGE_BASE_NAME, crate::built_info::PKG_VERSION);
255259

256260
let vector_aggregator_address = resolve_vector_aggregator_address(&hdfs, client)
257261
.await

0 commit comments

Comments
 (0)