fixed config attach for Yaml/Jason pipelines - when there is no variables section, config is not attached to pipeline

qwshen · qwshen · commit e6f9767140c1 · 2022-12-01T19:25:09.000-05:00
diff --git a/src/main/scala/com/qwshen/etl/pipeline/builder/JsonPipelineBuilder.scala b/src/main/scala/com/qwshen/etl/pipeline/builder/JsonPipelineBuilder.scala
@@ -131,6 +131,7 @@ class JsonPipelineBuilder extends PipelineBuilder with Loggable {
         def withKV = (k: String, v: String) => {
           val data = this.evaluate(this.resolve(ConfigurationManager.quote(v))(newConfig))(session)
           vars.put(k, data)
+          //update the config
           newConfig = ConfigurationManager.mergeVariables(newConfig, Map(k -> data))
         }
         value.foreach(v => withKV(n, v))

Original file line number	Diff line number	Diff line change
`@@ -131,6 +131,7 @@ class JsonPipelineBuilder extends PipelineBuilder with Loggable {`
`131`	`131`	`def withKV = (k: String, v: String) => {`
`132`	`132`	`val data = this.evaluate(this.resolve(ConfigurationManager.quote(v))(newConfig))(session)`
`133`	`133`	`vars.put(k, data)`
	`134`	`+ //update the config`
`134`	`135`	`newConfig = ConfigurationManager.mergeVariables(newConfig, Map(k -> data))`
`135`	`136`	`}`
`136`	`137`	`value.foreach(v => withKV(n, v))`