Skip to content

Commit fddc463

Browse files
committed
fix: HADOOP_HEAPSIZE has no suffix
1 parent 7d756f7 commit fddc463

File tree

3 files changed

+26
-23
lines changed

3 files changed

+26
-23
lines changed

docs/modules/hive/pages/usage-guide/overrides.adoc

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,3 +108,4 @@ Stackable operators automatically determine the set of needed JVM arguments, suc
108108
Using JVM argument overrides you can configure the JVM arguments xref:concepts:overrides.adoc#jvm-argument-overrides[according to the concepts page].
109109

110110
One thing that is different for Hive metastores, is that all heap-related arguments will be passed in via the env variable `HADOOP_HEAPSIZE`, all the other ones via `HADOOP_OPTS`.
111+
`HADOOP_HEAPSIZE` can *not* have a unit suffix, it will always be an integer representing the number of megabytes heap available.

rust/operator-binary/src/config/jvm.rs

Lines changed: 23 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -86,18 +86,22 @@ pub fn construct_non_heap_jvm_args(
8686
Ok(jvm_args.join(" "))
8787
}
8888

89-
/// Arguments that go into `HADOOP_HEAPSIZE`.
90-
/// You can get the normal JVM arguments using [`construct_non_heap_jvm_args`].
91-
pub fn construct_heap_jvm_args(
92-
hive: &HiveCluster,
93-
merged_config: &MetaStoreConfig,
94-
role: &Role<MetaStoreConfigFragment, GenericRoleConfig, JavaCommonConfig>,
95-
role_group: &str,
96-
) -> Result<String, Error> {
97-
let mut jvm_args = construct_jvm_args(hive, merged_config, role, role_group)?;
98-
jvm_args.retain(|arg| is_heap_jvm_argument(arg));
89+
/// This will be put into `HADOOP_HEAPSIZE`, which is just the heap size in megabytes (*without* the `m`
90+
/// unit prepended).
91+
pub fn construct_hadoop_heapsize_env(merged_config: &MetaStoreConfig) -> Result<String, Error> {
92+
let heap_size_in_mb = (MemoryQuantity::try_from(
93+
merged_config
94+
.resources
95+
.memory
96+
.limit
97+
.as_ref()
98+
.context(MissingMemoryResourceConfigSnafu)?,
99+
)
100+
.context(InvalidMemoryConfigSnafu)?
101+
* JAVA_HEAP_FACTOR)
102+
.scale_to(BinaryMultiple::Mebi);
99103

100-
Ok(jvm_args.join(" "))
104+
Ok((heap_size_in_mb.value.floor() as u32).to_string())
101105
}
102106

103107
fn is_heap_jvm_argument(jvm_argument: &str) -> bool {
@@ -131,11 +135,10 @@ mod tests {
131135
default:
132136
replicas: 1
133137
"#;
134-
let (hive, hive_role, role, merged_config) = construct_boilerplate(input);
138+
let (hive, merged_config, role, rolegroup) = construct_boilerplate(input);
135139
let non_heap_jvm_args =
136-
construct_non_heap_jvm_args(&hive, &hive_role, &role, &merged_config).unwrap();
137-
let heap_jvm_args =
138-
construct_heap_jvm_args(&hive, &hive_role, &role, &merged_config).unwrap();
140+
construct_non_heap_jvm_args(&hive, &merged_config, &role, &rolegroup).unwrap();
141+
let hadoop_heapsize_env = construct_hadoop_heapsize_env(&merged_config).unwrap();
139142

140143
assert_eq!(
141144
non_heap_jvm_args,
@@ -145,7 +148,7 @@ mod tests {
145148
-Djavax.net.ssl.trustStorePassword=changeit \
146149
-Djavax.net.ssl.trustStoreType=pkcs12"
147150
);
148-
assert_eq!(heap_jvm_args, "-Xmx409m -Xms409m");
151+
assert_eq!(hadoop_heapsize_env, "409");
149152
}
150153

151154
#[test]
@@ -185,11 +188,10 @@ mod tests {
185188
- -Xmx40000m
186189
- -Dhttps.proxyPort=1234
187190
"#;
188-
let (hive, merged_config, role, role_group) = construct_boilerplate(input);
191+
let (hive, merged_config, role, rolegroup) = construct_boilerplate(input);
189192
let non_heap_jvm_args =
190-
construct_non_heap_jvm_args(&hive, &merged_config, &role, &role_group).unwrap();
191-
let heap_jvm_args =
192-
construct_heap_jvm_args(&hive, &merged_config, &role, &role_group).unwrap();
193+
construct_non_heap_jvm_args(&hive, &merged_config, &role, &rolegroup).unwrap();
194+
let hadoop_heapsize_env = construct_hadoop_heapsize_env(&merged_config).unwrap();
193195

194196
assert_eq!(
195197
non_heap_jvm_args,
@@ -202,7 +204,7 @@ mod tests {
202204
-Djava.net.preferIPv4Stack=true \
203205
-Dhttps.proxyPort=1234"
204206
);
205-
assert_eq!(heap_jvm_args, "-Xms34406m -Xmx40000m");
207+
assert_eq!(hadoop_heapsize_env, "34406");
206208
}
207209

208210
fn construct_boilerplate(

rust/operator-binary/src/controller.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,7 +79,7 @@ use tracing::warn;
7979

8080
use crate::{
8181
command::build_container_command_args,
82-
config::jvm::{construct_heap_jvm_args, construct_non_heap_jvm_args},
82+
config::jvm::{construct_hadoop_heapsize_env, construct_non_heap_jvm_args},
8383
crd::{
8484
v1alpha1, Container, HiveClusterStatus, HiveRole, MetaStoreConfig, MetaStoreConfigFragment,
8585
APP_NAME, CORE_SITE_XML, DB_PASSWORD_ENV, DB_USERNAME_ENV, HIVE_ENV_SH, HIVE_PORT,
@@ -621,7 +621,7 @@ fn build_metastore_rolegroup_config_map(
621621
let mut data = BTreeMap::from([
622622
(
623623
"HADOOP_HEAPSIZE".to_string(),
624-
construct_heap_jvm_args(hive, merged_config, role, &rolegroup.role_group)
624+
construct_hadoop_heapsize_env(merged_config)
625625
.context(ConstructJvmArgumentsSnafu)?,
626626
),
627627
(

0 commit comments

Comments
 (0)