fix(config): prohibit multiline interpolation [WIP]

.github/actions/spelling/allow.txt

@@ -528,3 +528,4 @@ meme
 simdutf
 vhosts
 miniz
+envsubst

Cargo.toml

@@ -145,8 +145,9 @@ futures = { version = "0.3.31", default-features = false, features = ["compat",
 glob = { version = "0.3.2", default-features = false }
 hickory-proto = { version = "0.25.1", default-features = false, features = ["dnssec-ring"] }
 indexmap = { version = "2.9.0", default-features = false, features = ["serde", "std"] }
-inventory = { version = "0.3" }
 indoc = { version = "2.0.6" }
+inventory = { version = "0.3" }
+lexical-core = { version = "1.0.5", default-features = false, features = ["parse-integers", "parse-floats"] }
 metrics = "0.24.2"
 metrics-tracing-context = { version = "0.17.0", default-features = false }
 metrics-util = { version = "0.18.0", default-features = false, features = ["registry"] }
@@ -180,6 +181,7 @@ vrl = { git = "https://github.com/vectordotdev/vrl", branch = "main", features =
 cfg-if.workspace = true
 clap.workspace = true
 indoc.workspace = true
+lexical-core.workspace = true
 paste.workspace = true
 pin-project.workspace = true
 proptest = { workspace = true, optional = true }

LICENSE-3rdparty.csv

@@ -351,6 +351,10 @@ kube,https://github.com/kube-rs/kube,Apache-2.0,"clux <sszynrae@gmail.com>, Nata
 lalrpop-util,https://github.com/lalrpop/lalrpop,Apache-2.0 OR MIT,Niko Matsakis <niko@alum.mit.edu>
 lapin,https://github.com/amqp-rs/lapin,MIT,"Geoffroy Couprie <geo.couprie@gmail.com>, Marc-Antoine Perennou <Marc-Antoine@Perennou.com>"
 lazy_static,https://github.com/rust-lang-nursery/lazy-static.rs,MIT OR Apache-2.0,Marvin Löbel <loebel.marvin@gmail.com>
+lexical-core,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh <ahuszagh@gmail.com>
+lexical-parse-float,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh <ahuszagh@gmail.com>
+lexical-parse-integer,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh <ahuszagh@gmail.com>
+lexical-util,https://github.com/Alexhuszagh/rust-lexical,MIT OR Apache-2.0,Alex Huszagh <ahuszagh@gmail.com>
 libc,https://github.com/rust-lang/libc,MIT OR Apache-2.0,The Rust Project Developers
 libflate,https://github.com/sile/libflate,MIT,Takeru Ohta <phjgt308@gmail.com>
 libm,https://github.com/rust-lang/libm,MIT OR Apache-2.0,Jorge Aparicio <jorge@japaric.io>

changelog.d/multiline_interpolation_change.breaking.md

@@ -0,0 +1,37 @@
+The env var and secrets resolution now happens after the config string is parsed into a TOML table.
+As a side effect, this fixes a bug where comment lines referring to env vars or secrets that don't exist caused a config build error.
+
+This change breaks existing behavior. Injecting whole blocks now results in error e.g.
+
+A block:
+
+```shell
+export SOURCES_BLOCK="sources:\"
+  demo:
+    type: demo_logs
+    format: json
+    interval: 1
+```
+
+Config snippet:
+
+```yaml
+${SOURCES_BLOCK}
+```
+
+The config above will fail to load.
+
+Here is a potential workaround:
+
+```shell
+envsubst < snippet_in.yaml > snippet_out.yaml
+
+cat snippet_out.yaml
+sources:"
+  demo:
+    type: demo_logs
+    format: json
+    interval: 1
+```
+
+authors: pront

src/config/cmd.rs

@@ -205,9 +205,9 @@ mod tests {
         SinkDescription, SourceDescription, TransformDescription,
     };
 
-    use crate::config::Format;
+    use crate::config::{interpolate, Format};
     use crate::{
-        config::{cmd::serialize_to_json, vars, ConfigBuilder},
+        config::{cmd::serialize_to_json, ConfigBuilder},
         generate,
         generate::{generate_example, TransformInputsStrategy},
     };
@@ -250,7 +250,7 @@ mod tests {
         "#,
             env_var, env_var_in_arr
         );
-        let interpolated_config_source = vars::interpolate(
+        let interpolated_config_source = interpolate(
             config_source.as_ref(),
             &HashMap::from([
                 (env_var.to_string(), "syslog".to_string()),

src/config/loading/config_builder.rs

@@ -1,9 +1,11 @@
-use std::{collections::HashMap, io::Read};
+use std::collections::HashMap;
 
 use indexmap::IndexMap;
 use toml::value::Table;
 
-use super::{deserialize_table, loader, prepare_input, secret};
+use super::{
+    deserialize_table, interpolate_toml_table_with_secrets, loader, resolve_environment_variables,
+};
 use super::{ComponentHint, Process};
 use crate::config::{
     ComponentKey, ConfigBuilder, EnrichmentTableOuter, SinkOuter, SourceOuter, TestDefinition,
@@ -33,14 +35,12 @@ impl ConfigBuilderLoader {
 
 impl Process for ConfigBuilderLoader {
     /// Prepares input for a `ConfigBuilder` by interpolating environment variables.
-    fn prepare<R: Read>(&mut self, input: R) -> Result<String, Vec<String>> {
-        let prepared_input = prepare_input(input)?;
-        let prepared_input = self
-            .secrets
+    fn postprocess(&mut self, table: Table) -> Result<Table, Vec<String>> {
+        let table = resolve_environment_variables(table)?;
+        self.secrets
             .as_ref()
-            .map(|s| secret::interpolate(&prepared_input, s))
-            .unwrap_or(Ok(prepared_input))?;
-        Ok(prepared_input)
+            .map(|secrets_map| interpolate_toml_table_with_secrets(&table, secrets_map))
+            .unwrap_or(Ok(table))
     }
 
     /// Merge a TOML `Table` with a `ConfigBuilder`. Component types extend specific keys.