From 59d0ad24cf837193c187f9f314304811d28222b2 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Fri, 27 Dec 2024 11:47:29 +0100 Subject: [PATCH 01/12] refactor --- Cargo.lock | 186 +++++++-------- Cargo.nix | 211 +++++++++--------- .../src/config/jvm_arguments.rs | 75 +++++++ .../src/{config.rs => config/mod.rs} | 2 + rust/operator-binary/src/container.rs | 83 +++---- 5 files changed, 306 insertions(+), 251 deletions(-) create mode 100644 rust/operator-binary/src/config/jvm_arguments.rs rename rust/operator-binary/src/{config.rs => config/mod.rs} (99%) diff --git a/Cargo.lock b/Cargo.lock index 87471160..28992fea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,15 +111,15 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.94" +version = "1.0.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1fd03a028ef38ba2276dce7e33fcd6369c158a1bca17946c4b1b701891c1ff7" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" [[package]] name = "async-broadcast" -version = "0.7.1" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20cd0e2e25ea8e5f7e9df04578dc6cf5c83577fd09b1a46aaf5c85e1c33f2a7e" +checksum = "435a87a52755b8f27fcf321ac4f04b2802e337c8c4872923137471ec39c37532" dependencies = [ "event-listener", "event-listener-strategy", @@ -146,7 +146,7 @@ checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -157,7 +157,7 @@ checksum = "721cae7de5c34fbb2acd27e21e6d2cf7b886dce0c27388d46c4e6c47ea4318dd" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -270,9 +270,9 @@ checksum = "325918d6fe32f23b19878fe4b34794ae41fc19ddbe53b10571a4874d44ffd39b" [[package]] name = "cc" -version = "1.2.3" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27f657647bcff5394bf56c7317665bbf790a137a50eaaa5c6bfbb9e27a518f2d" +checksum = "8d6dbb628b8f8555f86d0323c2eb39e3ec81901f4b83e091db8a6a76d316a333" dependencies = [ "jobserver", "libc", @@ -329,7 +329,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -410,18 +410,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.20" +version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22ec99545bb0ed0ea7bb9b8e1e9122ea386ff8a48c0922e43f36d45ab09e0e80" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crypto-common" @@ -454,7 +454,7 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -465,7 +465,7 @@ checksum = "d336a2a514f6ccccaa3e09b02d41d35330c07ddf03a62165fcec10bb561c7806" dependencies = [ "darling_core", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -476,7 +476,7 @@ checksum = "bc2323e10c92e1cf4d86e11538512e6dc03ceb586842970b6332af3d4046a046" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -506,7 +506,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -544,7 +544,7 @@ dependencies = [ "enum-ordinalize", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -588,7 +588,7 @@ checksum = "0d28318a75d4aead5c4db25382e8ef717932d0346600cacae6357eb5941bc5ff" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -715,7 +715,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -855,11 +855,11 @@ checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" [[package]] name = "home" -version = "0.5.9" +version = "0.5.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3d1354bf6b7235cb4a0576c2619fd4ed18183f689b12b006a0ee7329eeff9a5" +checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" dependencies = [ - "windows-sys 0.52.0", + "windows-sys 0.59.0", ] [[package]] @@ -910,9 +910,9 @@ checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" [[package]] name = "hyper" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97818827ef4f364230e16705d4706e2897df2bb60617d6ca15d598025a3c481f" +checksum = "256fb8d4bd6413123cc9d91832d78325c48ff41677595be797d90f42969beae0" dependencies = [ "bytes", "futures-channel", @@ -949,9 +949,9 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.3" +version = "0.27.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08afdbb5c31130e3034af566421053ab03787c640246a446327f550d11bcb333" +checksum = "2d191583f3da1305256f22463b9bb0471acad48a4e534a5218b9963e9c1f59b2" dependencies = [ "futures-util", "http", @@ -1136,7 +1136,7 @@ checksum = "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -1370,7 +1370,7 @@ dependencies = [ "proc-macro2", "quote", "serde_json", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -1409,9 +1409,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "libc" -version = "0.2.168" +version = "0.2.169" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aaeb2981e0606ca11d79718f8bb01164f1d6ed75080182d3abf017e6d244b6d" +checksum = "b5aba8db14291edd000dfcc4d620c7ebfb122c613afb886ca8803fa4e128a20a" [[package]] name = "libgit2-sys" @@ -1482,9 +1482,9 @@ checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" [[package]] name = "miniz_oxide" -version = "0.8.0" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +checksum = "4ffbe83022cedc1d264172192511ae958937694cd57ce297164951b8b3568394" dependencies = [ "adler2", ] @@ -1537,9 +1537,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.5" +version = "0.36.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aedf0a2d09c573ed1d8d85b30c119153926a2b36dce0ab28322c09a117a4683e" +checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" dependencies = [ "memchr", ] @@ -1606,7 +1606,7 @@ dependencies = [ "lazy_static", "once_cell", "opentelemetry", - "ordered-float 4.5.0", + "ordered-float 4.6.0", "percent-encoding", "rand", "thiserror 1.0.69", @@ -1625,9 +1625,9 @@ dependencies = [ [[package]] name = "ordered-float" -version = "4.5.0" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c65ee1f9701bf938026630b455d5315f490640234259037edb259798b3bcf85e" +checksum = "7bb71e1b3fa6ca1c61f383464aaf2bb0e2f8e772a1f01d486832464de363b951" dependencies = [ "num-traits", ] @@ -1690,7 +1690,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b7cafe60d6cf8e62e1b9b2ea516a089c008945bb5a275416789e7db0bc199dc" dependencies = [ "memchr", - "thiserror 2.0.6", + "thiserror 2.0.9", "ucd-trie", ] @@ -1714,7 +1714,7 @@ dependencies = [ "pest_meta", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -1745,7 +1745,7 @@ checksum = "3c0f5fad0874fc7abcd4d750e76917eaebbecaa2c20bde22e1dbeeba8beb758c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -1817,9 +1817,9 @@ dependencies = [ [[package]] name = "quote" -version = "1.0.37" +version = "1.0.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5b9d34b8991d19d98081b46eacdd8eb58c6f2b201139f7c5f643cc155a633af" +checksum = "0e4dccaaaf89514f546c693ddc140f729f958c247918a13380cccc6078391acc" dependencies = [ "proc-macro2", ] @@ -1856,9 +1856,9 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b6dfecf2c74bce2466cabf93f6664d6998a69eb21e39f4207930065b27b771f" +checksum = "03a862b389f93e68874fbf580b9de08dd02facb9a788ebadaf4a3fd33cf58834" dependencies = [ "bitflags 2.6.0", ] @@ -1954,7 +1954,7 @@ dependencies = [ "regex", "relative-path", "rustc_version", - "syn 2.0.90", + "syn 2.0.92", "unicode-ident", ] @@ -1975,9 +1975,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.19" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "934b404430bb06b3fae2cba809eb45a1ab1aecd64491213d7c3301b88393f8d1" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "log", "once_cell", @@ -2010,7 +2010,7 @@ dependencies = [ "openssl-probe", "rustls-pki-types", "schannel", - "security-framework 3.0.1", + "security-framework 3.1.0", ] [[package]] @@ -2024,9 +2024,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.10.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "16f1201b3c9a7ee8039bcadc17b7e605e2945b27eee7631788c1bd2b0643674b" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" [[package]] name = "rustls-webpki" @@ -2041,9 +2041,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.18" +version = "1.0.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e819f2bc632f285be6d7cd36e25940d45b2391dd6d9b939e79de557f7014248" +checksum = "f7c45b9784283f1b2e7fb61b42047c2fd678ef0960d4f6f1eba131594cc369d4" [[package]] name = "ryu" @@ -2082,7 +2082,7 @@ dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2115,9 +2115,9 @@ dependencies = [ [[package]] name = "security-framework" -version = "3.0.1" +version = "3.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1415a607e92bec364ea2cf9264646dcce0f91e6d65281bd6f2819cca3bf39c8" +checksum = "81d3f8c9bfcc3cbb6b0179eb57042d75b1582bdc65c3cb95f3fa999509c03cbc" dependencies = [ "bitflags 2.6.0", "core-foundation 0.10.0", @@ -2128,9 +2128,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.12.1" +version = "2.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa39c7303dc58b5543c94d22c1766b0d31f2ee58306363ea622b10bbc075eaa2" +checksum = "1863fd3768cd83c56a7f60faa4dc0d403f1b6df0a38c3c25f44b7894e45370d5" dependencies = [ "core-foundation-sys", "libc", @@ -2138,15 +2138,15 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.23" +version = "1.0.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" +checksum = "3cb6eb87a131f756572d7fb904f6e7b68633f09cca868c5df1c4b8d1a694bbba" [[package]] name = "serde" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6513c1ad0b11a9376da888e3e0baa0077f1aed55c17f50e7b2397136129fb88f" +checksum = "0b9781016e935a97e8beecf0c933758c97a5520d32930e460142b4cd80c6338e" dependencies = [ "serde_derive", ] @@ -2163,13 +2163,13 @@ dependencies = [ [[package]] name = "serde_derive" -version = "1.0.215" +version = "1.0.216" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad1e866f866923f252f05c889987993144fb74e722403468a4ebd70c3cd756c0" +checksum = "46f859dbbf73865c6627ed570e78961cd3ac92407a2d117204c49232485da55e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2180,14 +2180,14 @@ checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] name = "serde_json" -version = "1.0.133" +version = "1.0.134" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7fceb2473b9166b2294ef05efcb65a3db80803f0b03ef86a5fc88a2b85ee377" +checksum = "d00f4175c42ee48b15416f6193a959ba3a0d67fc699a0db9ad12df9f83991c7d" dependencies = [ "itoa", "memchr", @@ -2308,7 +2308,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2419,7 +2419,7 @@ dependencies = [ "darling", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2459,7 +2459,7 @@ dependencies = [ "proc-macro2", "quote", "rustversion", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2481,9 +2481,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.90" +version = "2.0.92" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "919d3b74a5dd0ccd15aeb8f93e7006bd9e14c295087c9896a110f490752bcf31" +checksum = "70ae51629bf965c5c098cc9e87908a3df5301051a9e087d6f9bef5c9771ed126" dependencies = [ "proc-macro2", "quote", @@ -2492,9 +2492,9 @@ dependencies = [ [[package]] name = "sync_wrapper" -version = "0.1.2" +version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" [[package]] name = "synstructure" @@ -2504,7 +2504,7 @@ checksum = "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2518,11 +2518,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.6" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec2a1820ebd077e2b90c4df007bebf344cd394098a13c563957d0afc83ea47" +checksum = "f072643fd0190df67a8bab670c20ef5d8737177d6ac6b2e9a236cb096206b2cc" dependencies = [ - "thiserror-impl 2.0.6", + "thiserror-impl 2.0.9", ] [[package]] @@ -2533,18 +2533,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] name = "thiserror-impl" -version = "2.0.6" +version = "2.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65750cab40f4ff1929fb1ba509e9914eb756131cef4210da8d5d700d26f6312" +checksum = "7b50fa271071aae2e6ee85f842e2e28ba8cd2c5fb67f11fcb1fd70b276f9e7d4" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2646,7 +2646,7 @@ checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2703,9 +2703,9 @@ dependencies = [ [[package]] name = "tower" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2873938d487c3cfb9aed7546dc9f2711d867c9f90c46b889989a2cb84eba6b4f" +checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9" dependencies = [ "futures-core", "futures-util", @@ -2780,7 +2780,7 @@ checksum = "395ae124c09f9e6918a2310af6038fba074bcf474ac352496d5910dd59a2226d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -2991,7 +2991,7 @@ dependencies = [ "log", "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", "wasm-bindgen-shared", ] @@ -3013,7 +3013,7 @@ checksum = "30d7a95b763d3c45903ed6c81f156801839e5ee968bb07e534c44df0fcd330c2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", "wasm-bindgen-backend", "wasm-bindgen-shared", ] @@ -3194,7 +3194,7 @@ checksum = "2380878cad4ac9aac1e2435f3eb4020e8374b5f13c296cb75b4620ff8e229154" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", "synstructure", ] @@ -3216,7 +3216,7 @@ checksum = "fa4f8080344d4671fb4e831a13ad1e68092748387dfc4f55e356242fae12ce3e" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] [[package]] @@ -3236,7 +3236,7 @@ checksum = "595eed982f7d355beb85837f651fa22e90b3c044842dc7f2c2842c086f295808" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", "synstructure", ] @@ -3265,5 +3265,5 @@ checksum = "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.90", + "syn 2.0.92", ] diff --git a/Cargo.nix b/Cargo.nix index 2a320592..87bbbe7b 100644 --- a/Cargo.nix +++ b/Cargo.nix @@ -359,9 +359,9 @@ rec { }; "anyhow" = rec { crateName = "anyhow"; - version = "1.0.94"; + version = "1.0.95"; edition = "2018"; - sha256 = "1xqz3j4h3dxiqi37k8dwl5cc2sb3rlzy7rywfqiblf7g52h07zf1"; + sha256 = "010vd1ki8w84dzgx6c81sc8qm9n02fxic1gkpv52zp4nwrn0kb1l"; authors = [ "David Tolnay " ]; @@ -373,9 +373,9 @@ rec { }; "async-broadcast" = rec { crateName = "async-broadcast"; - version = "0.7.1"; - edition = "2018"; - sha256 = "0zia7z1y31awmxma9c89zmvkbj7mdkf7highkmz5z3pa4lp0xk90"; + version = "0.7.2"; + edition = "2021"; + sha256 = "0ckmqcwyqwbl2cijk1y4r0vy60i89gqc86ijrxzz5f2m4yjqfnj3"; libName = "async_broadcast"; authors = [ "Stjepan Glavina " @@ -448,7 +448,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" "visit-mut" ]; } ]; @@ -475,7 +475,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "proc-macro" "visit-mut" ]; } @@ -775,9 +775,9 @@ rec { }; "cc" = rec { crateName = "cc"; - version = "1.2.3"; + version = "1.2.6"; edition = "2018"; - sha256 = "0bcga5xf5fgvddfamsjhg89hlydzbdk1fwvcym5kkxfggdj5gxi7"; + sha256 = "0cx32v9pcslavf8y10sb3y883v7377mw48q3dpw5b1cgidibnvcd"; authors = [ "Alex Crichton " ]; @@ -982,7 +982,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" ]; } ]; @@ -1206,9 +1206,9 @@ rec { }; "crossbeam-channel" = rec { crateName = "crossbeam-channel"; - version = "0.5.13"; + version = "0.5.14"; edition = "2021"; - sha256 = "1wkx45r34v7g3wyi3lg2wz536lrrrab4h4hh741shfhr8rlhsj1k"; + sha256 = "0wa41qybq5w8s70anb472myh4fid4aw6v65vws6wn528w9l6vfh6"; libName = "crossbeam_channel"; dependencies = [ { @@ -1225,9 +1225,9 @@ rec { }; "crossbeam-utils" = rec { crateName = "crossbeam-utils"; - version = "0.8.20"; + version = "0.8.21"; edition = "2021"; - sha256 = "100fksq5mm1n7zj242cclkw6yf7a4a8ix3lvpfkhxvdhbda9kv12"; + sha256 = "0a3aa2bmc8q35fb67432w16wvi54sfmb69rk9h5bhd18vw0c99fh"; libName = "crossbeam_utils"; features = { "default" = [ "std" ]; @@ -1318,7 +1318,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" "extra-traits" ]; } ]; @@ -1348,7 +1348,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; @@ -1374,7 +1374,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" "visit-mut" ]; } ]; @@ -1461,7 +1461,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; features = { @@ -1554,13 +1554,13 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; devDependencies = [ { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" ]; } ]; @@ -1662,7 +1662,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; features = { @@ -2007,7 +2007,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" ]; } ]; @@ -2423,16 +2423,16 @@ rec { }; "home" = rec { crateName = "home"; - version = "0.5.9"; + version = "0.5.11"; edition = "2021"; - sha256 = "19grxyg35rqfd802pcc9ys1q3lafzlcjcv2pl2s5q8xpyr5kblg3"; + sha256 = "1kxb4k87a9sayr8jipr7nq9wpgmjk4hk4047hmf9kc24692k75aq"; authors = [ "Brian Anderson " ]; dependencies = [ { name = "windows-sys"; - packageId = "windows-sys 0.52.0"; + packageId = "windows-sys 0.59.0"; target = { target, features }: (target."windows" or false); features = [ "Win32_Foundation" "Win32_UI_Shell" "Win32_System_Com" ]; } @@ -2552,9 +2552,9 @@ rec { }; "hyper" = rec { crateName = "hyper"; - version = "1.5.1"; + version = "1.5.2"; edition = "2021"; - sha256 = "07s87id0566m2p5dc5q6nqmxz5r8drqd81b7w4q44djgxwkqi0cp"; + sha256 = "1q7akfb443yrjzkmnnbp2vs8zi15hgbk466rr4y144v4ppabhvr5"; authors = [ "Sean McArthur " ]; @@ -2626,6 +2626,10 @@ rec { usesDefaultFeatures = false; features = [ "alloc" "sink" ]; } + { + name = "pin-project-lite"; + packageId = "pin-project-lite"; + } { name = "tokio"; packageId = "tokio"; @@ -2743,9 +2747,9 @@ rec { }; "hyper-rustls" = rec { crateName = "hyper-rustls"; - version = "0.27.3"; + version = "0.27.5"; edition = "2021"; - sha256 = "0cxkph8hsmbz693a8ih2ciy7h0xbac844rpm981y6c0iqfsxpbq8"; + sha256 = "1cjr3yf3x5mr3194llsfibacl6j7n2dknii2dwjha4ysyf1ia69d"; libName = "hyper_rustls"; dependencies = [ { @@ -3450,7 +3454,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; @@ -4280,7 +4284,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "extra-traits" ]; } ]; @@ -4424,9 +4428,9 @@ rec { }; "libc" = rec { crateName = "libc"; - version = "0.2.168"; + version = "0.2.169"; edition = "2021"; - sha256 = "0vab4inpw0dz78nii02hsxp1skqn06xzh64psw8wl1h63scb5bjs"; + sha256 = "02m253hs8gw0m1n8iyrsc4n15yzbqwhddi7w1l0ds7i92kdsiaxm"; authors = [ "The Rust Project Developers" ]; @@ -4641,9 +4645,9 @@ rec { }; "miniz_oxide" = rec { crateName = "miniz_oxide"; - version = "0.8.0"; + version = "0.8.2"; edition = "2021"; - sha256 = "1wadxkg6a6z4lr7kskapj5d8pxlx7cp1ifw4daqnkzqjxych5n72"; + sha256 = "1543asrvhla92sby4z6m9ilkg2cmmq8ja6bj84k1vp6f48qfiysg"; authors = [ "Frommi " "oyvindln " @@ -4795,9 +4799,9 @@ rec { }; "object" = rec { crateName = "object"; - version = "0.36.5"; + version = "0.36.7"; edition = "2018"; - sha256 = "0gk8lhbs229c68lapq6w6qmnm4jkj48hrcw5ilfyswy514nhmpxf"; + sha256 = "11vv97djn9nc5n6w1gc6bd96d2qk2c8cg1kw5km9bsi3v4a8x532"; dependencies = [ { name = "memchr"; @@ -5045,7 +5049,7 @@ rec { } { name = "ordered-float"; - packageId = "ordered-float 4.5.0"; + packageId = "ordered-float 4.6.0"; } { name = "percent-encoding"; @@ -5133,11 +5137,11 @@ rec { }; resolvedDefaultFeatures = [ "default" "std" ]; }; - "ordered-float 4.5.0" = rec { + "ordered-float 4.6.0" = rec { crateName = "ordered-float"; - version = "4.5.0"; + version = "4.6.0"; edition = "2021"; - sha256 = "0ppqpjrri5r5vdz06na24d00cjaz67ambd1hcq13iy8vf3wy2pn6"; + sha256 = "0ldrcgilsiijd141vw51fbkziqmh5fpllil3ydhirjm67wdixdvv"; libName = "ordered_float"; authors = [ "Jonathan Reem " @@ -5156,6 +5160,7 @@ rec { "bytemuck" = [ "dep:bytemuck" ]; "default" = [ "std" ]; "derive-visitor" = [ "dep:derive-visitor" ]; + "libm" = [ "num-traits/libm" ]; "num-cmp" = [ "dep:num-cmp" ]; "proptest" = [ "dep:proptest" ]; "rand" = [ "dep:rand" ]; @@ -5322,7 +5327,7 @@ rec { } { name = "thiserror"; - packageId = "thiserror 2.0.6"; + packageId = "thiserror 2.0.9"; optional = true; } { @@ -5397,7 +5402,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; features = { @@ -5470,7 +5475,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; usesDefaultFeatures = false; features = [ "parsing" "printing" "clone-impls" "proc-macro" "full" "visit-mut" ]; } @@ -5637,9 +5642,9 @@ rec { }; "quote" = rec { crateName = "quote"; - version = "1.0.37"; + version = "1.0.38"; edition = "2018"; - sha256 = "1brklraw2g34bxy9y4q1nbrccn7bv36ylihv12c9vlcii55x7fdm"; + sha256 = "1k0s75w61k6ch0rs263r4j69b7vj1wadqgb9dia4ylc9mymcqk8f"; authors = [ "David Tolnay " ]; @@ -5756,9 +5761,9 @@ rec { }; "redox_syscall" = rec { crateName = "redox_syscall"; - version = "0.5.7"; + version = "0.5.8"; edition = "2021"; - sha256 = "07vpgfr6a04k0x19zqr1xdlqm6fncik3zydbdi3f5g3l5k7zwvcv"; + sha256 = "0d48ylyd6gsamynyp257p6n2zl4dw2fhnn5z9y3nhgpri6rn5a03"; libName = "syscall"; authors = [ "Jeremy Soller " @@ -6110,7 +6115,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" "parsing" "extra-traits" "visit" "visit-mut" ]; } { @@ -6160,9 +6165,9 @@ rec { }; "rustls" = rec { crateName = "rustls"; - version = "0.23.19"; + version = "0.23.20"; edition = "2021"; - sha256 = "1lgqjf1vh09kghyj34a4svn1max18pmhka6bwbxb61mv61240jwk"; + sha256 = "06rvj13ia4hx0kba454vcm3p4f2jz907rrabi76k5lyba3rc6rah"; dependencies = [ { name = "log"; @@ -6285,7 +6290,7 @@ rec { } { name = "security-framework"; - packageId = "security-framework 3.0.1"; + packageId = "security-framework 3.1.0"; target = { target, features }: ("macos" == target."os" or null); } ]; @@ -6312,9 +6317,9 @@ rec { }; "rustls-pki-types" = rec { crateName = "rustls-pki-types"; - version = "1.10.0"; + version = "1.10.1"; edition = "2021"; - sha256 = "0jv78c32pgf1i0bn7rzf4xdr9qh5wsvigp6akc1yhzls7hdj1w8n"; + sha256 = "0dqb3d0cbld1yrp084wyzgw6yk3qzzic8l5pbs1b6bcjzzk4ggyj"; libName = "rustls_pki_types"; features = { "default" = [ "alloc" ]; @@ -6359,9 +6364,9 @@ rec { }; "rustversion" = rec { crateName = "rustversion"; - version = "1.0.18"; + version = "1.0.19"; edition = "2018"; - sha256 = "0j2207vmgrcxwwwvknfn3lwv4i8djhjnxlvwdnz8bwijqqmrz08f"; + sha256 = "1m39qd65jcd1xgqzdm3017ppimiggh2446xngwp1ngr8hjbmpi7p"; procMacro = true; build = "build/build.rs"; authors = [ @@ -6496,7 +6501,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "extra-traits" ]; } ]; @@ -6581,11 +6586,11 @@ rec { }; resolvedDefaultFeatures = [ "OSX_10_10" "OSX_10_11" "OSX_10_12" "OSX_10_9" "default" ]; }; - "security-framework 3.0.1" = rec { + "security-framework 3.1.0" = rec { crateName = "security-framework"; - version = "3.0.1"; + version = "3.1.0"; edition = "2021"; - sha256 = "1j1rpyiwq698dyyq2lnnws8hzknw8r32dy9cx9jc7gljgrh5lhg1"; + sha256 = "1g1wq04rb6gsyfawphv5vhmmicbm5l25gsvr05mvng6cpz4zilw1"; libName = "security_framework"; authors = [ "Steven Fackler " @@ -6626,9 +6631,9 @@ rec { }; "security-framework-sys" = rec { crateName = "security-framework-sys"; - version = "2.12.1"; + version = "2.13.0"; edition = "2021"; - sha256 = "18pafp0bn41bcbm66qrhb3pg4c8dddvc28jdr51mb2y57lqcffgs"; + sha256 = "1mbhagj98y2byhjkr353y1nings01pfa9yk0gxmcb0ydd0vzsqqq"; libName = "security_framework_sys"; authors = [ "Steven Fackler " @@ -6657,9 +6662,9 @@ rec { }; "semver" = rec { crateName = "semver"; - version = "1.0.23"; + version = "1.0.24"; edition = "2018"; - sha256 = "12wqpxfflclbq4dv8sa6gchdh92ahhwn4ci1ls22wlby3h57wsb1"; + sha256 = "1fmvjjkd3f64y5fqr1nakkq371mnwzv09fbz5mbmdxril63ypdiw"; authors = [ "David Tolnay " ]; @@ -6671,9 +6676,9 @@ rec { }; "serde" = rec { crateName = "serde"; - version = "1.0.215"; + version = "1.0.216"; edition = "2018"; - sha256 = "13xqkw93cw9rnbkm0zy1apnilzq7l2xf1qw8m1nkga8i1fnw24v5"; + sha256 = "13ikqs0cvd220530x4rj1m9ab5wcflrwkw7cpvl9fnlkdq0q35qb"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -6726,9 +6731,9 @@ rec { }; "serde_derive" = rec { crateName = "serde_derive"; - version = "1.0.215"; + version = "1.0.216"; edition = "2015"; - sha256 = "1h2nswy0rmzblil38h12wxsgni1ik63rk22wy19g48v9hrpqc7md"; + sha256 = "0pm5bm4354n40ir12bbs829arlqwjrw0wmzd4xk5r1kkpzdmky26"; procMacro = true; authors = [ "Erick Tryzelaar " @@ -6749,7 +6754,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" "proc-macro" ]; } @@ -6781,7 +6786,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; usesDefaultFeatures = false; features = [ "clone-impls" "derive" "parsing" "printing" ]; } @@ -6790,9 +6795,9 @@ rec { }; "serde_json" = rec { crateName = "serde_json"; - version = "1.0.133"; + version = "1.0.134"; edition = "2021"; - sha256 = "0xz3bswa527wln3fy0qb7y081nx3cp5yy1ggjhi6n5mrfcjfpz67"; + sha256 = "0z8wk61rzpqjmnwhv6k9zikhsfmsb6lr6qbg84aqpr1fqisl23yh"; authors = [ "Erick Tryzelaar " "David Tolnay " @@ -7153,7 +7158,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" ]; } ]; @@ -7575,7 +7580,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; @@ -7689,7 +7694,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "parsing" "extra-traits" ]; } ]; @@ -7742,11 +7747,11 @@ rec { }; resolvedDefaultFeatures = [ "clone-impls" "default" "derive" "full" "parsing" "printing" "proc-macro" "quote" ]; }; - "syn 2.0.90" = rec { + "syn 2.0.92" = rec { crateName = "syn"; - version = "2.0.90"; + version = "2.0.92"; edition = "2021"; - sha256 = "0cfg5dsr1x0hl6b9hz08jp1197mx0rq3xydqmqaws36xlms3p7ci"; + sha256 = "09ni3rvwkxdyz7b8gq59a4831x9xia88g7nck30cargrkdi53bkh"; authors = [ "David Tolnay " ]; @@ -7777,9 +7782,9 @@ rec { }; "sync_wrapper" = rec { crateName = "sync_wrapper"; - version = "0.1.2"; - edition = "2018"; - sha256 = "0q01lyj0gr9a93n10nxsn8lwbzq97jqd6b768x17c8f7v7gccir0"; + version = "1.0.2"; + edition = "2021"; + sha256 = "0qvjyasd6w18mjg5xlaq5jgy84jsjfsvmnn12c13gypxbv75dwhb"; authors = [ "Actyx AG " ]; @@ -7809,7 +7814,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; usesDefaultFeatures = false; features = [ "derive" "parsing" "printing" "clone-impls" "visit" "extra-traits" ]; } @@ -7836,18 +7841,18 @@ rec { ]; }; - "thiserror 2.0.6" = rec { + "thiserror 2.0.9" = rec { crateName = "thiserror"; - version = "2.0.6"; + version = "2.0.9"; edition = "2021"; - sha256 = "0izahgy0lzcmcg2i72h9jk9lqd5zpq3z0k8cp7i7gl7b40c2mv4g"; + sha256 = "1k5j0ri0kjrnlblv5ikaglbkg1sxxwh0qrxbidxgc38rs0zn8wph"; authors = [ "David Tolnay " ]; dependencies = [ { name = "thiserror-impl"; - packageId = "thiserror-impl 2.0.6"; + packageId = "thiserror-impl 2.0.9"; } ]; features = { @@ -7876,16 +7881,16 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; }; - "thiserror-impl 2.0.6" = rec { + "thiserror-impl 2.0.9" = rec { crateName = "thiserror-impl"; - version = "2.0.6"; + version = "2.0.9"; edition = "2021"; - sha256 = "04k3dz901mymm06j3x6f65hpbsqlk6g51fmiky9g2kqgnk550myn"; + sha256 = "1m77z5vb4w7xn7y12zxnbwncva4bwbi45y45xvkf5aki20kzll3v"; procMacro = true; libName = "thiserror_impl"; authors = [ @@ -7902,7 +7907,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; @@ -8257,7 +8262,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" ]; } ]; @@ -8450,9 +8455,9 @@ rec { }; "tower" = rec { crateName = "tower"; - version = "0.5.1"; + version = "0.5.2"; edition = "2018"; - sha256 = "0kvbp97bhb4sk24vhihcz74ngn0i4ygxqikmxndgng3w926r6wr8"; + sha256 = "1ybmd59nm4abl9bsvy6rx31m4zvzp5rja2slzpn712y9b68ssffh"; authors = [ "Tower Maintainers " ]; @@ -8792,7 +8797,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; usesDefaultFeatures = false; features = [ "full" "parsing" "printing" "visit-mut" "clone-impls" "extra-traits" "proc-macro" ]; } @@ -9463,7 +9468,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "full" ]; } { @@ -9526,7 +9531,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "visit" "visit-mut" "full" ]; } { @@ -9894,7 +9899,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "Win32_UI" "Win32_UI_Shell" "default" ]; + resolvedDefaultFeatures = [ "Wdk" "Wdk_Foundation" "Wdk_Storage" "Wdk_Storage_FileSystem" "Wdk_System" "Wdk_System_IO" "Win32" "Win32_Foundation" "Win32_Networking" "Win32_Networking_WinSock" "Win32_Security" "Win32_Storage" "Win32_Storage_FileSystem" "Win32_System" "Win32_System_Console" "Win32_System_IO" "Win32_System_Pipes" "Win32_System_SystemServices" "Win32_System_Threading" "Win32_System_WindowsProgramming" "default" ]; }; "windows-sys 0.59.0" = rec { crateName = "windows-sys"; @@ -10153,7 +10158,7 @@ rec { "Win32_Web" = [ "Win32" ]; "Win32_Web_InternetExplorer" = [ "Win32_Web" ]; }; - resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_System" "Win32_System_Console" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_SystemInformation" "default" ]; + resolvedDefaultFeatures = [ "Win32" "Win32_Foundation" "Win32_Security" "Win32_Security_Authentication" "Win32_Security_Authentication_Identity" "Win32_Security_Credentials" "Win32_Security_Cryptography" "Win32_System" "Win32_System_Com" "Win32_System_Console" "Win32_System_LibraryLoader" "Win32_System_Memory" "Win32_System_SystemInformation" "Win32_UI" "Win32_UI_Shell" "default" ]; }; "windows-targets" = rec { crateName = "windows-targets"; @@ -10415,7 +10420,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "fold" ]; } { @@ -10488,7 +10493,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; } ]; @@ -10536,7 +10541,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "fold" ]; } { @@ -10620,7 +10625,7 @@ rec { } { name = "syn"; - packageId = "syn 2.0.90"; + packageId = "syn 2.0.92"; features = [ "extra-traits" ]; } ]; diff --git a/rust/operator-binary/src/config/jvm_arguments.rs b/rust/operator-binary/src/config/jvm_arguments.rs new file mode 100644 index 00000000..0c7c6635 --- /dev/null +++ b/rust/operator-binary/src/config/jvm_arguments.rs @@ -0,0 +1,75 @@ +use snafu::{ResultExt, Snafu}; +use stackable_hdfs_crd::{constants::JVM_SECURITY_PROPERTIES_FILE, HdfsRole}; +use stackable_operator::{ + k8s_openapi::api::core::v1::ResourceRequirements, + memory::{BinaryMultiple, MemoryQuantity}, +}; + +const JVM_HEAP_FACTOR: f32 = 0.8; + +#[derive(Snafu, Debug)] +pub enum Error { + #[snafu(display("invalid java heap config for {role:?}"))] + InvalidJavaHeapConfig { + source: stackable_operator::memory::Error, + role: String, + }, +} + +// Additionally, any other init or sidecar container must have access to the following settings. +// As the Prometheus metric emitter is not part of this config it's safe to use for hdfs cli tools as well. +// This will not only enable the init containers to work, but also the user to run e.g. +// `bin/hdfs dfs -ls /` without getting `Caused by: java.lang.IllegalArgumentException: KrbException: Cannot locate default realm` +// because the `-Djava.security.krb5.conf` setting is missing +pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { + let mut jvm_args = Vec::new(); + + if kerberos_enabled { + jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_owned()); + } + + // TODO: Handle user input + jvm_args.join(" ") +} + +pub fn construct_role_specific_jvm_args( + role: &HdfsRole, + kerberos_enabled: bool, + resources: Option<&ResourceRequirements>, + config_dir: &str, + metrics_port: u16, +) -> Result { + let mut jvm_args = Vec::new(); + + if let Some(memory_limit) = resources.and_then(|r| r.limits.as_ref()?.get("memory")) { + let memory_limit = MemoryQuantity::try_from(memory_limit).with_context(|_| { + InvalidJavaHeapConfigSnafu { + role: role.to_string(), + } + })?; + jvm_args.push(format!( + "-Xmx{}", + (memory_limit * JVM_HEAP_FACTOR) + .scale_to(BinaryMultiple::Kibi) + .format_for_java() + .with_context(|_| { + InvalidJavaHeapConfigSnafu { + role: role.to_string(), + } + })? + )); + } + + jvm_args.extend([format!( + "-Djava.security.properties={config_dir}/{JVM_SECURITY_PROPERTIES_FILE}", + ),format!( + "-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={metrics_port}:/stackable/jmx/{role}.yaml", + )]); + + if kerberos_enabled { + jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string()); + } + + // TODO: Handle user input + Ok(jvm_args.join(" ")) +} diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config/mod.rs similarity index 99% rename from rust/operator-binary/src/config.rs rename to rust/operator-binary/src/config/mod.rs index 584d7ec6..55e3b47e 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -13,6 +13,8 @@ use stackable_hdfs_crd::{HdfsCluster, HdfsPodRef}; use stackable_operator::utils::cluster_info::KubernetesClusterInfo; use std::collections::BTreeMap; +pub mod jvm_arguments; + #[derive(Clone)] pub struct HdfsSiteConfigBuilder { config: BTreeMap, diff --git a/rust/operator-binary/src/container.rs b/rust/operator-binary/src/container.rs index d854b19e..9c83578e 100644 --- a/rust/operator-binary/src/container.rs +++ b/rust/operator-binary/src/container.rs @@ -53,7 +53,6 @@ use stackable_operator::{ }, kube::{core::ObjectMeta, ResourceExt}, kvp::Labels, - memory::{BinaryMultiple, MemoryQuantity}, product_logging::{ self, framework::{ @@ -69,6 +68,10 @@ use stackable_operator::{ use strum::{Display, EnumDiscriminants, IntoStaticStr}; use crate::{ + config::{ + self, + jvm_arguments::{construct_global_jvm_args, construct_role_specific_jvm_args}, + }, product_logging::{ FORMAT_NAMENODES_LOG4J_CONFIG_FILE, FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, HDFS_LOG4J_CONFIG_FILE, MAX_FORMAT_NAMENODE_LOG_FILE_SIZE, @@ -76,7 +79,7 @@ use crate::{ MAX_WAIT_NAMENODES_LOG_FILE_SIZE, MAX_ZKFC_LOG_FILE_SIZE, STACKABLE_LOG_DIR, WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, ZKFC_LOG4J_CONFIG_FILE, }, - DATANODE_ROOT_DATA_DIR_PREFIX, JVM_SECURITY_PROPERTIES_FILE, LOG4J_PROPERTIES, + DATANODE_ROOT_DATA_DIR_PREFIX, LOG4J_PROPERTIES, }; pub(crate) const TLS_STORE_DIR: &str = "/stackable/tls"; @@ -95,9 +98,9 @@ pub enum Error { #[snafu(display("object has no namespace"))] ObjectHasNoNamespace, - #[snafu(display("invalid java heap config for {role:?}"))] - InvalidJavaHeapConfig { - source: stackable_operator::memory::Error, + #[snafu(display("failed to construct JVM arguments fro role {role:?}"))] + ConstructJvmArguments { + source: config::jvm_arguments::Error, role: String, }, @@ -212,7 +215,6 @@ impl ContainerConfig { const WAIT_FOR_NAMENODES_CONFIG_VOLUME_MOUNT_NAME: &'static str = "wait-for-namenodes-config"; const WAIT_FOR_NAMENODES_LOG_VOLUME_MOUNT_NAME: &'static str = "wait-for-namenodes-log-config"; - const JVM_HEAP_FACTOR: f32 = 0.8; const HADOOP_HOME: &'static str = "/stackable/hadoop"; /// Add all main, side and init containers as well as required volumes to the pod builder. @@ -870,23 +872,16 @@ wait_for_termination $! }, ); } - // Additionally, any other init or sidecar container must have access to the following settings. - // As the Prometheus metric emitter is not part of this config it's safe to use for hdfs cli tools as well. - // This will not only enable the init containers to work, but also the user to run e.g. - // `bin/hdfs dfs -ls /` without getting `Caused by: java.lang.IllegalArgumentException: KrbException: Cannot locate default realm` - // because the `-Djava.security.krb5.conf` setting is missing - if hdfs.has_kerberos_enabled() { - env.insert( - "HADOOP_OPTS".to_string(), - EnvVar { - name: "HADOOP_OPTS".to_string(), - value: Some( - "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string(), - ), - ..EnvVar::default() - }, - ); + env.insert( + "HADOOP_OPTS".to_string(), + EnvVar { + name: "HADOOP_OPTS".to_string(), + value: Some(construct_global_jvm_args(hdfs.has_kerberos_enabled())), + ..EnvVar::default() + }, + ); + if hdfs.has_kerberos_enabled() { env.insert( "KRB5_CONFIG".to_string(), EnvVar { @@ -904,6 +899,7 @@ wait_for_termination $! }, ); } + // Needed for the `containerdebug` process to log it's tracing information to. env.insert( "CONTAINERDEBUG_LOG_DIRECTORY".to_string(), @@ -1221,39 +1217,16 @@ wait_for_termination $! } => { let cvd = ContainerVolumeDirs::from(role); let config_dir = cvd.final_config(); - let mut jvm_args = vec![ - format!( - "-Djava.security.properties={config_dir}/{JVM_SECURITY_PROPERTIES_FILE} -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={metrics_port}:/stackable/jmx/{role}.yaml", - )]; - - if hdfs.has_kerberos_enabled() { - jvm_args.push( - "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string(), - ); - } - - if let Some(memory_limit) = resources.and_then(|r| r.limits.as_ref()?.get("memory")) - { - let memory_limit = - MemoryQuantity::try_from(memory_limit).with_context(|_| { - InvalidJavaHeapConfigSnafu { - role: role.to_string(), - } - })?; - jvm_args.push(format!( - "-Xmx{}", - (memory_limit * Self::JVM_HEAP_FACTOR) - .scale_to(BinaryMultiple::Kibi) - .format_for_java() - .with_context(|_| { - InvalidJavaHeapConfigSnafu { - role: role.to_string(), - } - })? - )); - } - - Ok(jvm_args.join(" ").trim().to_string()) + construct_role_specific_jvm_args( + role, + hdfs.has_kerberos_enabled(), + resources, + config_dir, + *metrics_port, + ) + .with_context(|_| ConstructJvmArgumentsSnafu { + role: role.to_string(), + }) } _ => Ok("".to_string()), } From 9a9b4dc639f585aa9026beacc8ef9960711b0409 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 09:57:20 +0100 Subject: [PATCH 02/12] Rename --- rust/operator-binary/src/config/{jvm_arguments.rs => jvm.rs} | 0 rust/operator-binary/src/config/mod.rs | 2 +- rust/operator-binary/src/container.rs | 4 ++-- 3 files changed, 3 insertions(+), 3 deletions(-) rename rust/operator-binary/src/config/{jvm_arguments.rs => jvm.rs} (100%) diff --git a/rust/operator-binary/src/config/jvm_arguments.rs b/rust/operator-binary/src/config/jvm.rs similarity index 100% rename from rust/operator-binary/src/config/jvm_arguments.rs rename to rust/operator-binary/src/config/jvm.rs diff --git a/rust/operator-binary/src/config/mod.rs b/rust/operator-binary/src/config/mod.rs index 55e3b47e..26406d36 100644 --- a/rust/operator-binary/src/config/mod.rs +++ b/rust/operator-binary/src/config/mod.rs @@ -13,7 +13,7 @@ use stackable_hdfs_crd::{HdfsCluster, HdfsPodRef}; use stackable_operator::utils::cluster_info::KubernetesClusterInfo; use std::collections::BTreeMap; -pub mod jvm_arguments; +pub mod jvm; #[derive(Clone)] pub struct HdfsSiteConfigBuilder { diff --git a/rust/operator-binary/src/container.rs b/rust/operator-binary/src/container.rs index 60acfbb3..3741ef6e 100644 --- a/rust/operator-binary/src/container.rs +++ b/rust/operator-binary/src/container.rs @@ -69,7 +69,7 @@ use strum::{Display, EnumDiscriminants, IntoStaticStr}; use crate::{ config::{ self, - jvm_arguments::{construct_global_jvm_args, construct_role_specific_jvm_args}, + jvm::{construct_global_jvm_args, construct_role_specific_jvm_args}, }, product_logging::{ FORMAT_NAMENODES_LOG4J_CONFIG_FILE, FORMAT_ZOOKEEPER_LOG4J_CONFIG_FILE, @@ -99,7 +99,7 @@ pub enum Error { #[snafu(display("failed to construct JVM arguments fro role {role:?}"))] ConstructJvmArguments { - source: config::jvm_arguments::Error, + source: config::jvm::Error, role: String, }, From 8ff284fc079851f5b12c6a090a5fcd7ae261fd4a Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 10:11:51 +0100 Subject: [PATCH 03/12] Fix error handling --- rust/operator-binary/src/config/jvm.rs | 2 +- rust/operator-binary/src/container.rs | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 0c7c6635..a8368f0c 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -16,7 +16,7 @@ pub enum Error { }, } -// Additionally, any other init or sidecar container must have access to the following settings. +// All init or sidecar containers must have access to the following settings. // As the Prometheus metric emitter is not part of this config it's safe to use for hdfs cli tools as well. // This will not only enable the init containers to work, but also the user to run e.g. // `bin/hdfs dfs -ls /` without getting `Caused by: java.lang.IllegalArgumentException: KrbException: Cannot locate default realm` diff --git a/rust/operator-binary/src/container.rs b/rust/operator-binary/src/container.rs index 3741ef6e..276c0820 100644 --- a/rust/operator-binary/src/container.rs +++ b/rust/operator-binary/src/container.rs @@ -468,7 +468,7 @@ impl ContainerConfig { zookeeper_config_map_name, env_overrides, resources.as_ref(), - )) + )?) .add_volume_mounts(self.volume_mounts(hdfs, merged_config, labels)?) .context(AddVolumeMountSnafu)? .add_container_ports(self.container_ports(hdfs)); @@ -519,7 +519,7 @@ impl ContainerConfig { cb.image_from_product_image(resolved_product_image) .command(Self::command()) .args(self.args(hdfs, cluster_info, role, merged_config, namenode_podrefs)?) - .add_env_vars(self.env(hdfs, zookeeper_config_map_name, env_overrides, None)) + .add_env_vars(self.env(hdfs, zookeeper_config_map_name, env_overrides, None)?) .add_volume_mounts(self.volume_mounts(hdfs, merged_config, labels)?) .context(AddVolumeMountSnafu)?; @@ -828,7 +828,7 @@ wait_for_termination $! zookeeper_config_map_name: &str, env_overrides: Option<&BTreeMap>, resources: Option<&ResourceRequirements>, - ) -> Vec { + ) -> Result, Error> { // Maps env var name to env var object. This allows env_overrides to work // as expected (i.e. users can override the env var value). let mut env: BTreeMap = BTreeMap::new(); @@ -857,7 +857,7 @@ wait_for_termination $! role_opts_name.clone(), EnvVar { name: role_opts_name, - value: self.build_hadoop_opts(hdfs, resources).ok(), + value: Some(self.build_hadoop_opts(hdfs, resources)?), ..EnvVar::default() }, ); @@ -909,7 +909,7 @@ wait_for_termination $! env.append(&mut env_override_vars); - env.into_values().collect() + Ok(env.into_values().collect()) } /// Returns the container resources. From 87a4772f3edfeb21d0d69caa1a8747f094fe7522 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 11:40:52 +0100 Subject: [PATCH 04/12] Add (failing test) -> TDD :) --- rust/operator-binary/src/config/jvm.rs | 133 +++++++++++++++++++++++-- rust/operator-binary/src/container.rs | 2 +- 2 files changed, 123 insertions(+), 12 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index a8368f0c..7a2b873a 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -47,17 +47,15 @@ pub fn construct_role_specific_jvm_args( role: role.to_string(), } })?; - jvm_args.push(format!( - "-Xmx{}", - (memory_limit * JVM_HEAP_FACTOR) - .scale_to(BinaryMultiple::Kibi) - .format_for_java() - .with_context(|_| { - InvalidJavaHeapConfigSnafu { - role: role.to_string(), - } - })? - )); + let heap = memory_limit.scale_to(BinaryMultiple::Mebi) * JVM_HEAP_FACTOR; + let heap = heap + .format_for_java() + .with_context(|_| InvalidJavaHeapConfigSnafu { + role: role.to_string(), + })?; + + jvm_args.push(format!("-Xms{heap}")); + jvm_args.push(format!("-Xmx{heap}")); } jvm_args.extend([format!( @@ -73,3 +71,116 @@ pub fn construct_role_specific_jvm_args( // TODO: Handle user input Ok(jvm_args.join(" ")) } + +#[cfg(test)] +mod tests { + use stackable_hdfs_crd::{constants::DEFAULT_NAME_NODE_METRICS_PORT, HdfsCluster}; + + use crate::container::ContainerConfig; + + use super::*; + + #[test] + fn test_global_jvm_args() { + assert_eq!(construct_global_jvm_args(false), ""); + assert_eq!( + construct_global_jvm_args(true), + "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf" + ); + } + + #[test] + fn test_jvm_config_defaults_without_kerberos() { + let input = r#" + apiVersion: hdfs.stackable.tech/v1alpha1 + kind: HdfsCluster + metadata: + name: hdfs + spec: + image: + productVersion: 3.4.0 + clusterConfig: + zookeeperConfigMapName: hdfs-zk + nameNodes: + roleGroups: + default: + replicas: 1 + "#; + let jvm_config = construct_test_role_specific_jvm_args(input, false); + + assert_eq!( + jvm_config, + "-Xms819m \ + -Xmx819m \ + -Djava.security.properties=/stackable/config/security.properties \ + -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8183:/stackable/jmx/namenode.yaml" + ); + } + + #[test] + fn test_jvm_config_jvm_argument_overrides() { + let input = r#" + apiVersion: hdfs.stackable.tech/v1alpha1 + kind: HdfsCluster + metadata: + name: hdfs + spec: + image: + productVersion: 3.4.0 + clusterConfig: + zookeeperConfigMapName: hdfs-zk + nameNodes: + config: + resources: + memory: + limit: 42Gi + jvmArgumentOverrides: + add: + - -Dhttps.proxyHost=proxy.my.corp + - -Dhttps.proxyPort=8080 + - -Djava.net.preferIPv4Stack=true + roleGroups: + default: + replicas: 1 + jvmArgumentOverrides: + # We need more memory! + removeRegex: + - -Xmx.* + - -Dhttps.proxyPort=.* + add: + - -Xmx40000m + - -Dhttps.proxyPort=1234 + "#; + let jvm_config = construct_test_role_specific_jvm_args(input, true); + + assert_eq!( + jvm_config, + "-Xms34406m \ + -Djava.security.properties=/stackable/config/security.properties \ + -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8183:/stackable/jmx/namenode.yaml \ + -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf \ + -Dhttps.proxyHost=proxy.my.corp \ + -Djava.net.preferIPv4Stack=true \ + -Xmx40000m \ + -Dhttps.proxyPort=1234" + ); + } + + fn construct_test_role_specific_jvm_args(hdfs_cluster: &str, kerberos_enabled: bool) -> String { + let hdfs: HdfsCluster = serde_yaml::from_str(hdfs_cluster).expect("illegal test input"); + + let role = HdfsRole::NameNode; + let merged_config = role.merged_config(&hdfs, "default").unwrap(); + let container_config = ContainerConfig::from(role); + let resources = container_config.resources(&merged_config); + + construct_role_specific_jvm_args( + &role, + kerberos_enabled, + resources.as_ref(), + "/stackable/config", + DEFAULT_NAME_NODE_METRICS_PORT, + ) + .unwrap() + } +} diff --git a/rust/operator-binary/src/container.rs b/rust/operator-binary/src/container.rs index 276c0820..26825a66 100644 --- a/rust/operator-binary/src/container.rs +++ b/rust/operator-binary/src/container.rs @@ -913,7 +913,7 @@ wait_for_termination $! } /// Returns the container resources. - fn resources(&self, merged_config: &AnyNodeConfig) -> Option { + pub fn resources(&self, merged_config: &AnyNodeConfig) -> Option { match self { // Namenode sidecar containers ContainerConfig::Zkfc { .. } => Some( From 38320b0f4bbd6dab2086f447df107eee32faed1c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 15:04:20 +0100 Subject: [PATCH 05/12] Actually merge stuff --- deploy/helm/hdfs-operator/crds/crds.yaml | 156 ++++++++++++++++++++ rust/crd/src/lib.rs | 65 ++++++-- rust/operator-binary/src/config/jvm.rs | 35 +++-- rust/operator-binary/src/container.rs | 23 ++- rust/operator-binary/src/hdfs_controller.rs | 2 + 5 files changed, 257 insertions(+), 24 deletions(-) diff --git a/deploy/helm/hdfs-operator/crds/crds.yaml b/deploy/helm/hdfs-operator/crds/crds.yaml index d1fccfc8..87731606 100644 --- a/deploy/helm/hdfs-operator/crds/crds.yaml +++ b/deploy/helm/hdfs-operator/crds/crds.yaml @@ -373,6 +373,32 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. @@ -654,6 +680,32 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. @@ -951,6 +1003,32 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. @@ -1219,6 +1297,32 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. @@ -1472,6 +1576,32 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. @@ -1744,6 +1874,32 @@ spec: default: {} description: '`envOverrides` configure environment variables to be set in the Pods. It is a map from strings to strings - environment variables and the value to set. Read the [environment variable overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#env-overrides) for more information and consult the operator specific usage guide to find out about the product specific environment variables that are available.' type: object + jvmArgumentOverrides: + default: + add: [] + remove: [] + removeRegex: [] + description: Allows overriding JVM arguments. Please read on the [JVM argument overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#jvm-argument-overrides) for details on the usage. + properties: + add: + default: [] + description: JVM arguments to be added + items: + type: string + type: array + remove: + default: [] + description: JVM arguments to be removed by exact match + items: + type: string + type: array + removeRegex: + default: [] + description: JVM arguments matching any of this regexes will be removed + items: + type: string + type: array + type: object podOverrides: default: {} description: In the `podOverrides` property you can define a [PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.27/#podtemplatespec-v1-core) to override any property that can be set on a Kubernetes Pod. Read the [Pod overrides documentation](https://docs.stackable.tech/home/nightly/concepts/overrides#pod-overrides) for more information. diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index fcf67775..f3e0458c 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -38,7 +38,8 @@ use stackable_operator::{ spec::{ContainerLogConfig, Logging}, }, role_utils::{ - GenericProductSpecificCommonConfig, GenericRoleConfig, Role, RoleGroup, RoleGroupRef, + self, GenericRoleConfig, JavaCommonConfig, JvmArgumentOverrides, Role, RoleGroup, + RoleGroupRef, }, schemars::{self, JsonSchema}, status::condition::{ClusterCondition, HasStatusCondition}, @@ -72,7 +73,7 @@ pub enum Error { #[snafu(display("object has no associated namespace"))] NoNamespace, - #[snafu(display("missing node role {role:?}"))] + #[snafu(display("missing role {role:?}"))] MissingRole { role: String }, #[snafu(display("missing role group {role_group:?} for role {role:?}"))] @@ -103,6 +104,9 @@ pub enum Error { #[snafu(display("failed to build role-group selector label"))] BuildRoleGroupSelectorLabel { source: LabelError }, + + #[snafu(display("failed to merge jvm argument overrides"))] + MergeJvmArgumentOverrides { source: role_utils::Error }, } /// An HDFS cluster stacklet. This resource is managed by the Stackable operator for Apache Hadoop HDFS. @@ -140,15 +144,15 @@ pub struct HdfsClusterSpec { // no doc string - See Role struct #[serde(default, skip_serializing_if = "Option::is_none")] - pub name_nodes: Option>, + pub name_nodes: Option>, // no doc string - See Role struct #[serde(default, skip_serializing_if = "Option::is_none")] - pub data_nodes: Option>, + pub data_nodes: Option>, // no doc string - See Role struct #[serde(default, skip_serializing_if = "Option::is_none")] - pub journal_nodes: Option>, + pub journal_nodes: Option>, } #[derive(Clone, Debug, Deserialize, Eq, JsonSchema, PartialEq, Serialize)] @@ -541,7 +545,7 @@ impl HdfsCluster { pub fn namenode_rolegroup( &self, role_group: &str, - ) -> Option<&RoleGroup> { + ) -> Option<&RoleGroup> { self.spec.name_nodes.as_ref()?.role_groups.get(role_group) } @@ -549,7 +553,7 @@ impl HdfsCluster { pub fn datanode_rolegroup( &self, role_group: &str, - ) -> Option<&RoleGroup> { + ) -> Option<&RoleGroup> { self.spec.data_nodes.as_ref()?.role_groups.get(role_group) } @@ -557,7 +561,7 @@ impl HdfsCluster { pub fn journalnode_rolegroup( &self, role_group: &str, - ) -> Option<&RoleGroup> { + ) -> Option<&RoleGroup> { self.spec .journal_nodes .as_ref()? @@ -565,14 +569,49 @@ impl HdfsCluster { .get(role_group) } - pub fn role_config(&self, role: &HdfsRole) -> Option<&GenericRoleConfig> { - match role { + pub fn role_config(&self, hdfs_role: &HdfsRole) -> Option<&GenericRoleConfig> { + match hdfs_role { HdfsRole::NameNode => self.spec.name_nodes.as_ref().map(|nn| &nn.role_config), HdfsRole::DataNode => self.spec.data_nodes.as_ref().map(|dn| &dn.role_config), HdfsRole::JournalNode => self.spec.journal_nodes.as_ref().map(|jn| &jn.role_config), } } + pub fn get_merged_jvm_argument_overrides( + &self, + hdfs_role: &HdfsRole, + role_group: &str, + operator_generated: &JvmArgumentOverrides, + ) -> Result { + match hdfs_role { + HdfsRole::JournalNode => self + .spec + .journal_nodes + .as_ref() + .with_context(|| MissingRoleSnafu { + role: HdfsRole::JournalNode.to_string(), + })? + .get_merged_jvm_argument_overrides(role_group, operator_generated), + HdfsRole::NameNode => self + .spec + .name_nodes + .as_ref() + .with_context(|| MissingRoleSnafu { + role: HdfsRole::NameNode.to_string(), + })? + .get_merged_jvm_argument_overrides(role_group, operator_generated), + HdfsRole::DataNode => self + .spec + .data_nodes + .as_ref() + .with_context(|| MissingRoleSnafu { + role: HdfsRole::DataNode.to_string(), + })? + .get_merged_jvm_argument_overrides(role_group, operator_generated), + } + .context(MergeJvmArgumentOverridesSnafu) + } + pub fn pod_overrides_for_role(&self, role: &HdfsRole) -> Option<&PodTemplateSpec> { match role { HdfsRole::NameNode => self @@ -768,7 +807,11 @@ impl HdfsCluster { String, ( Vec, - Role>, + Role< + impl Configuration, + GenericRoleConfig, + JavaCommonConfig, + >, ), >, Error, diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index 7a2b873a..fc7d45b9 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -1,8 +1,9 @@ use snafu::{ResultExt, Snafu}; -use stackable_hdfs_crd::{constants::JVM_SECURITY_PROPERTIES_FILE, HdfsRole}; +use stackable_hdfs_crd::{constants::JVM_SECURITY_PROPERTIES_FILE, HdfsCluster, HdfsRole}; use stackable_operator::{ k8s_openapi::api::core::v1::ResourceRequirements, memory::{BinaryMultiple, MemoryQuantity}, + role_utils::JvmArgumentOverrides, }; const JVM_HEAP_FACTOR: f32 = 0.8; @@ -14,6 +15,9 @@ pub enum Error { source: stackable_operator::memory::Error, role: String, }, + + #[snafu(display("failed to merge jvm argument overrides"))] + MergeJvmArgumentOverrides { source: stackable_hdfs_crd::Error }, } // All init or sidecar containers must have access to the following settings. @@ -33,7 +37,9 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { } pub fn construct_role_specific_jvm_args( - role: &HdfsRole, + hdfs: &HdfsCluster, + hdfs_role: &HdfsRole, + role_group: &str, kerberos_enabled: bool, resources: Option<&ResourceRequirements>, config_dir: &str, @@ -44,32 +50,37 @@ pub fn construct_role_specific_jvm_args( if let Some(memory_limit) = resources.and_then(|r| r.limits.as_ref()?.get("memory")) { let memory_limit = MemoryQuantity::try_from(memory_limit).with_context(|_| { InvalidJavaHeapConfigSnafu { - role: role.to_string(), + role: hdfs_role.to_string(), } })?; let heap = memory_limit.scale_to(BinaryMultiple::Mebi) * JVM_HEAP_FACTOR; let heap = heap .format_for_java() .with_context(|_| InvalidJavaHeapConfigSnafu { - role: role.to_string(), + role: hdfs_role.to_string(), })?; jvm_args.push(format!("-Xms{heap}")); jvm_args.push(format!("-Xmx{heap}")); } - jvm_args.extend([format!( - "-Djava.security.properties={config_dir}/{JVM_SECURITY_PROPERTIES_FILE}", - ),format!( - "-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={metrics_port}:/stackable/jmx/{role}.yaml", - )]); - + jvm_args.extend([ + format!("-Djava.security.properties={config_dir}/{JVM_SECURITY_PROPERTIES_FILE}"), + format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={metrics_port}:/stackable/jmx/{hdfs_role}.yaml") + ]); if kerberos_enabled { jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string()); } // TODO: Handle user input - Ok(jvm_args.join(" ")) + let operator_generated = JvmArgumentOverrides::new_with_only_additions(jvm_args); + let merged_jvm_args = hdfs + .get_merged_jvm_argument_overrides(hdfs_role, role_group, &operator_generated) + .context(MergeJvmArgumentOverridesSnafu)?; + + Ok(merged_jvm_args + .effective_jvm_config_after_merging() + .join(" ")) } #[cfg(test)] @@ -175,7 +186,9 @@ mod tests { let resources = container_config.resources(&merged_config); construct_role_specific_jvm_args( + &hdfs, &role, + "default", kerberos_enabled, resources.as_ref(), "/stackable/config", diff --git a/rust/operator-binary/src/container.rs b/rust/operator-binary/src/container.rs index 26825a66..33f13f2f 100644 --- a/rust/operator-binary/src/container.rs +++ b/rust/operator-binary/src/container.rs @@ -214,6 +214,7 @@ impl ContainerConfig { hdfs: &HdfsCluster, cluster_info: &KubernetesClusterInfo, role: &HdfsRole, + role_group: &str, resolved_product_image: &ResolvedProductImage, merged_config: &AnyNodeConfig, env_overrides: Option<&BTreeMap>, @@ -230,6 +231,7 @@ impl ContainerConfig { hdfs, cluster_info, role, + role_group, resolved_product_image, zk_config_map_name, env_overrides, @@ -315,6 +317,7 @@ impl ContainerConfig { hdfs, cluster_info, role, + role_group, resolved_product_image, zk_config_map_name, env_overrides, @@ -335,6 +338,7 @@ impl ContainerConfig { hdfs, cluster_info, role, + role_group, resolved_product_image, zk_config_map_name, env_overrides, @@ -356,6 +360,7 @@ impl ContainerConfig { hdfs, cluster_info, role, + role_group, resolved_product_image, zk_config_map_name, env_overrides, @@ -378,6 +383,7 @@ impl ContainerConfig { hdfs, cluster_info, role, + role_group, resolved_product_image, zk_config_map_name, env_overrides, @@ -447,6 +453,7 @@ impl ContainerConfig { hdfs: &HdfsCluster, cluster_info: &KubernetesClusterInfo, role: &HdfsRole, + role_group: &str, resolved_product_image: &ResolvedProductImage, zookeeper_config_map_name: &str, env_overrides: Option<&BTreeMap>, @@ -465,6 +472,7 @@ impl ContainerConfig { .args(self.args(hdfs, cluster_info, role, merged_config, &[])?) .add_env_vars(self.env( hdfs, + role_group, zookeeper_config_map_name, env_overrides, resources.as_ref(), @@ -506,6 +514,7 @@ impl ContainerConfig { hdfs: &HdfsCluster, cluster_info: &KubernetesClusterInfo, role: &HdfsRole, + role_group: &str, resolved_product_image: &ResolvedProductImage, zookeeper_config_map_name: &str, env_overrides: Option<&BTreeMap>, @@ -519,7 +528,13 @@ impl ContainerConfig { cb.image_from_product_image(resolved_product_image) .command(Self::command()) .args(self.args(hdfs, cluster_info, role, merged_config, namenode_podrefs)?) - .add_env_vars(self.env(hdfs, zookeeper_config_map_name, env_overrides, None)?) + .add_env_vars(self.env( + hdfs, + role_group, + zookeeper_config_map_name, + env_overrides, + None, + )?) .add_volume_mounts(self.volume_mounts(hdfs, merged_config, labels)?) .context(AddVolumeMountSnafu)?; @@ -825,6 +840,7 @@ wait_for_termination $! fn env( &self, hdfs: &HdfsCluster, + role_group: &str, zookeeper_config_map_name: &str, env_overrides: Option<&BTreeMap>, resources: Option<&ResourceRequirements>, @@ -857,7 +873,7 @@ wait_for_termination $! role_opts_name.clone(), EnvVar { name: role_opts_name, - value: Some(self.build_hadoop_opts(hdfs, resources)?), + value: Some(self.build_hadoop_opts(hdfs, role_group, resources)?), ..EnvVar::default() }, ); @@ -1194,6 +1210,7 @@ wait_for_termination $! fn build_hadoop_opts( &self, hdfs: &HdfsCluster, + role_group: &str, resources: Option<&ResourceRequirements>, ) -> Result { match self { @@ -1203,7 +1220,9 @@ wait_for_termination $! let cvd = ContainerVolumeDirs::from(role); let config_dir = cvd.final_config(); construct_role_specific_jvm_args( + hdfs, role, + role_group, hdfs.has_kerberos_enabled(), resources, config_dir, diff --git a/rust/operator-binary/src/hdfs_controller.rs b/rust/operator-binary/src/hdfs_controller.rs index 31054adb..234c63e6 100644 --- a/rust/operator-binary/src/hdfs_controller.rs +++ b/rust/operator-binary/src/hdfs_controller.rs @@ -861,6 +861,7 @@ fn rolegroup_statefulset( hdfs, cluster_info, role, + &rolegroup_ref.role_group, resolved_product_image, merged_config, env_overrides, @@ -1000,6 +1001,7 @@ properties: [] cluster_domain: DomainName::try_from("cluster.local").unwrap(), }, &role, + "default", &resolved_product_image, &merged_config, env_overrides, From ebf83d55b63bbc58bd575f350d92963cbb70b97c Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 15:06:24 +0100 Subject: [PATCH 06/12] changelog --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 72ac4d0f..7bc90639 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ All notable changes to this project will be documented in this file. - The lifetime of auto generated TLS certificates is now configurable with the role and roleGroup config property `requestedSecretLifetime`. This helps reducing frequent Pod restarts ([#619]). - Run a `containerdebug` process in the background of each HDFS container to collect debugging information ([#629]). +- Support configuring JVM arguments ([#636]). - Aggregate emitted Kubernetes events on the CustomResources ([#643]). ### Changed @@ -19,6 +20,7 @@ All notable changes to this project will be documented in this file. [#619]: https://github.com/stackabletech/hdfs-operator/pull/619 [#629]: https://github.com/stackabletech/hdfs-operator/pull/629 +[#636]: https://github.com/stackabletech/hdfs-operator/pull/636 [#643]: https://github.com/stackabletech/hdfs-operator/pull/643 ## [24.11.1] - 2025-01-10 From 4b90e216db2d12a3094307dc756356dd627c6720 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 15:10:06 +0100 Subject: [PATCH 07/12] Add some docs --- rust/operator-binary/src/config/jvm.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index fc7d45b9..d58cd2bd 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -32,7 +32,14 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_owned()); } - // TODO: Handle user input + // We do *not* add user overrides to the global JVM args, but only the role specific JVm arguments. + // This allows users to configure stuff for the server (probably what they want to do), without + // also influencing e.g. startup scripts. + // + // However, this is just an assumptions. If it is wrong users can still envOverride the global + // JVM args. + // + // Please feel absolutely free to change this behavior! jvm_args.join(" ") } @@ -72,7 +79,6 @@ pub fn construct_role_specific_jvm_args( jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string()); } - // TODO: Handle user input let operator_generated = JvmArgumentOverrides::new_with_only_additions(jvm_args); let merged_jvm_args = hdfs .get_merged_jvm_argument_overrides(hdfs_role, role_group, &operator_generated) From abe242ffff643dbb57302957f0a8917c27f4d138 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 15:11:57 +0100 Subject: [PATCH 08/12] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7bc90639..f9f07c6b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,7 @@ All notable changes to this project will be documented in this file. - Switch the WebUI liveness probe from `httpGet` to checking the tcp socket. This helps with setups where configOverrides are used to enable security on the HTTP interfaces. As this results in `401` HTTP responses (instead of `200`), this previously failed the liveness checks. +- Set the JVM argument `-Xms` in addition to `-Xmx` (with the same value). This ensure consistent JVM configs across our products. [#619]: https://github.com/stackabletech/hdfs-operator/pull/619 [#629]: https://github.com/stackabletech/hdfs-operator/pull/629 From 4063f833a5e16dfc48d779ee62aececcf8cddb12 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 15:13:01 +0100 Subject: [PATCH 09/12] clippy --- rust/crd/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/rust/crd/src/lib.rs b/rust/crd/src/lib.rs index f3e0458c..64655370 100644 --- a/rust/crd/src/lib.rs +++ b/rust/crd/src/lib.rs @@ -800,6 +800,7 @@ impl HdfsCluster { } } + #[allow(clippy::type_complexity)] pub fn build_role_properties( &self, ) -> Result< From 4f420c0fe01c1e5949f594a022c4a4a2f75215bf Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 16:37:39 +0100 Subject: [PATCH 10/12] Update CHANGELOG.md Co-authored-by: Malte Sander --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9f07c6b..86c8a611 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,7 +17,7 @@ All notable changes to this project will be documented in this file. - Switch the WebUI liveness probe from `httpGet` to checking the tcp socket. This helps with setups where configOverrides are used to enable security on the HTTP interfaces. As this results in `401` HTTP responses (instead of `200`), this previously failed the liveness checks. -- Set the JVM argument `-Xms` in addition to `-Xmx` (with the same value). This ensure consistent JVM configs across our products. +- Set the JVM argument `-Xms` in addition to `-Xmx` (with the same value). This ensure consistent JVM configs across our products ([#636]). [#619]: https://github.com/stackabletech/hdfs-operator/pull/619 [#629]: https://github.com/stackabletech/hdfs-operator/pull/629 From 882fb32ba2d81acbc31ef7fc6d3cc4a3cf02ba9f Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 16:38:03 +0100 Subject: [PATCH 11/12] Update rust/operator-binary/src/config/jvm.rs Co-authored-by: Malte Sander --- rust/operator-binary/src/config/jvm.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index d58cd2bd..b7f4e77a 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -32,11 +32,11 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_owned()); } - // We do *not* add user overrides to the global JVM args, but only the role specific JVm arguments. + // We do *not* add user overrides to the global JVM args, but only the role specific JVM arguments. // This allows users to configure stuff for the server (probably what they want to do), without // also influencing e.g. startup scripts. // - // However, this is just an assumptions. If it is wrong users can still envOverride the global + // However, this is just an assumption. If it is wrong users can still envOverride the global // JVM args. // // Please feel absolutely free to change this behavior! From 61345c49abba1546f0284219ded64c6e3e3c0ae5 Mon Sep 17 00:00:00 2001 From: Sebastian Bernauer Date: Tue, 28 Jan 2025 16:54:43 +0100 Subject: [PATCH 12/12] constants. constants everywhere. --- rust/operator-binary/src/config/jvm.rs | 29 ++++++++++++------- rust/operator-binary/src/container.rs | 15 +++++----- rust/operator-binary/src/security/kerberos.rs | 17 +++++++++-- 3 files changed, 40 insertions(+), 21 deletions(-) diff --git a/rust/operator-binary/src/config/jvm.rs b/rust/operator-binary/src/config/jvm.rs index b7f4e77a..9976b95e 100644 --- a/rust/operator-binary/src/config/jvm.rs +++ b/rust/operator-binary/src/config/jvm.rs @@ -6,6 +6,8 @@ use stackable_operator::{ role_utils::JvmArgumentOverrides, }; +use crate::security::kerberos::KERBEROS_CONTAINER_PATH; + const JVM_HEAP_FACTOR: f32 = 0.8; #[derive(Snafu, Debug)] @@ -29,7 +31,9 @@ pub fn construct_global_jvm_args(kerberos_enabled: bool) -> String { let mut jvm_args = Vec::new(); if kerberos_enabled { - jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_owned()); + jvm_args.push(format!( + "-Djava.security.krb5.conf={KERBEROS_CONTAINER_PATH}/krb5.conf" + )); } // We do *not* add user overrides to the global JVM args, but only the role specific JVM arguments. @@ -76,7 +80,9 @@ pub fn construct_role_specific_jvm_args( format!("-javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar={metrics_port}:/stackable/jmx/{hdfs_role}.yaml") ]); if kerberos_enabled { - jvm_args.push("-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf".to_string()); + jvm_args.push(format!( + "-Djava.security.krb5.conf={KERBEROS_CONTAINER_PATH}/krb5.conf" + )); } let operator_generated = JvmArgumentOverrides::new_with_only_additions(jvm_args); @@ -102,7 +108,7 @@ mod tests { assert_eq!(construct_global_jvm_args(false), ""); assert_eq!( construct_global_jvm_args(true), - "-Djava.security.krb5.conf=/stackable/kerberos/krb5.conf" + format!("-Djava.security.krb5.conf={KERBEROS_CONTAINER_PATH}/krb5.conf") ); } @@ -172,14 +178,15 @@ mod tests { assert_eq!( jvm_config, - "-Xms34406m \ - -Djava.security.properties=/stackable/config/security.properties \ - -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8183:/stackable/jmx/namenode.yaml \ - -Djava.security.krb5.conf=/stackable/kerberos/krb5.conf \ - -Dhttps.proxyHost=proxy.my.corp \ - -Djava.net.preferIPv4Stack=true \ - -Xmx40000m \ - -Dhttps.proxyPort=1234" + format!( + "-Xms34406m \ + -Djava.security.properties=/stackable/config/security.properties \ + -javaagent:/stackable/jmx/jmx_prometheus_javaagent.jar=8183:/stackable/jmx/namenode.yaml \ + -Djava.security.krb5.conf={KERBEROS_CONTAINER_PATH}/krb5.conf \ + -Dhttps.proxyHost=proxy.my.corp \ + -Djava.net.preferIPv4Stack=true \ + -Xmx40000m \ + -Dhttps.proxyPort=1234") ); } diff --git a/rust/operator-binary/src/container.rs b/rust/operator-binary/src/container.rs index 33f13f2f..929ef118 100644 --- a/rust/operator-binary/src/container.rs +++ b/rust/operator-binary/src/container.rs @@ -78,6 +78,7 @@ use crate::{ MAX_WAIT_NAMENODES_LOG_FILE_SIZE, MAX_ZKFC_LOG_FILE_SIZE, STACKABLE_LOG_DIR, WAIT_FOR_NAMENODES_LOG4J_CONFIG_FILE, ZKFC_LOG4J_CONFIG_FILE, }, + security::kerberos::KERBEROS_CONTAINER_PATH, DATANODE_ROOT_DATA_DIR_PREFIX, LOG4J_PROPERTIES, }; @@ -801,8 +802,7 @@ wait_for_termination $! // Command to export `KERBEROS_REALM` env var to default real from krb5.conf, e.g. `CLUSTER.LOCAL` fn export_kerberos_real_env_var_command() -> String { - "export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' /stackable/kerberos/krb5.conf)\n" - .to_string() + format!("export KERBEROS_REALM=$(grep -oP 'default_realm = \\K.*' {KERBEROS_CONTAINER_PATH}/krb5.conf)\n") } /// Command to `kinit` a ticket using the principal created for the specified hdfs role @@ -822,8 +822,8 @@ wait_for_termination $! ); Ok(formatdoc!( r###" - echo "Getting ticket for {principal}" from /stackable/kerberos/keytab - kinit "{principal}" -kt /stackable/kerberos/keytab + echo "Getting ticket for {principal}" from {KERBEROS_CONTAINER_PATH}/keytab + kinit "{principal}" -kt {KERBEROS_CONTAINER_PATH}/keytab "###, )) } @@ -892,7 +892,7 @@ wait_for_termination $! "KRB5_CONFIG".to_string(), EnvVar { name: "KRB5_CONFIG".to_string(), - value: Some("/stackable/kerberos/krb5.conf".to_string()), + value: Some(format!("{KERBEROS_CONTAINER_PATH}/krb5.conf")), ..EnvVar::default() }, ); @@ -900,7 +900,7 @@ wait_for_termination $! "KRB5_CLIENT_KTNAME".to_string(), EnvVar { name: "KRB5_CLIENT_KTNAME".to_string(), - value: Some("/stackable/kerberos/keytab".to_string()), + value: Some(format!("{KERBEROS_CONTAINER_PATH}/keytab")), ..EnvVar::default() }, ); @@ -1107,7 +1107,8 @@ wait_for_termination $! // Adding this for all containers, as not only the main container needs Kerberos or TLS if hdfs.has_kerberos_enabled() { - volume_mounts.push(VolumeMountBuilder::new("kerberos", "/stackable/kerberos").build()); + volume_mounts + .push(VolumeMountBuilder::new("kerberos", KERBEROS_CONTAINER_PATH).build()); } if hdfs.has_https_enabled() { // This volume will be propagated by the create-tls-cert-bundle container diff --git a/rust/operator-binary/src/security/kerberos.rs b/rust/operator-binary/src/security/kerberos.rs index 9b44748c..5bb85f5a 100644 --- a/rust/operator-binary/src/security/kerberos.rs +++ b/rust/operator-binary/src/security/kerberos.rs @@ -10,6 +10,8 @@ use stackable_operator::{ use crate::config::{CoreSiteConfigBuilder, HdfsSiteConfigBuilder}; +pub const KERBEROS_CONTAINER_PATH: &str = "/stackable/kerberos"; + type Result = std::result::Result; #[derive(Snafu, Debug)] @@ -85,9 +87,18 @@ impl CoreSiteConfigBuilder { "dfs.web.authentication.kerberos.principal", format!("HTTP/{principal_host_part}"), ) - .add("dfs.journalnode.keytab.file", "/stackable/kerberos/keytab") - .add("dfs.namenode.keytab.file", "/stackable/kerberos/keytab") - .add("dfs.datanode.keytab.file", "/stackable/kerberos/keytab") + .add( + "dfs.journalnode.keytab.file", + format!("{KERBEROS_CONTAINER_PATH}/keytab"), + ) + .add( + "dfs.namenode.keytab.file", + format!("{KERBEROS_CONTAINER_PATH}/keytab"), + ) + .add( + "dfs.datanode.keytab.file", + format!("{KERBEROS_CONTAINER_PATH}/keytab"), + ) .add( "dfs.journalnode.kerberos.principal.pattern", format!("jn/{principal_host_part}"),