Add secrets

aggregator/task_engine.go

@@ -25,7 +25,8 @@ func (agg *Aggregator) startTaskEngine(ctx context.Context) {
 	})
 	agg.worker = apqueue.NewWorker(agg.queue, agg.db)
 	taskExecutor := taskengine.NewExecutor(agg.db, agg.logger)
-	taskengine.SetMacro(agg.config.Macros)
+	taskengine.SetMacroVars(agg.config.MacroVars)
+	taskengine.SetMacroSecrets(agg.config.MacroSecrets)
 	taskengine.SetCache(agg.cache)
 	macros.SetRpc(agg.config.SmartWallet.EthRpcUrl)
 

core/config/config.go

@@ -55,9 +55,10 @@ type Config struct {
 	SocketPath  string
 	Environment sdklogging.LogLevel
 
-	Macros map[string]string
+	MacroVars    map[string]string
+	MacroSecrets map[string]string
 
-	MetricsReg       *prometheus.Registry
+	MetricsReg *prometheus.Registry
 }
 
 type SmartWalletConfig struct {
@@ -96,7 +97,7 @@ type ConfigRaw struct {
 
 	SocketPath string `yaml:"socket_path"`
 
-	Macros map[string]string `yaml:"macros"`
+	Macros map[string]map[string]string `yaml:"macros"`
 }
 
 // These are read from CredibleSquaringDeploymentFileFlag
@@ -202,9 +203,10 @@ func NewConfig(configFilePath string) (*Config, error) {
 			ControllerPrivateKey: controllerPrivateKey,
 		},
 
-		SocketPath: configRaw.SocketPath,
-		Macros:     configRaw.Macros,
-		MetricsReg: reg,
+		SocketPath:   configRaw.SocketPath,
+		MacroVars:    configRaw.Macros["vars"],
+		MacroSecrets: configRaw.Macros["secrets"],
+		MetricsReg:   reg,
 	}
 
 	if config.SocketPath == "" {

core/taskengine/engine.go

@@ -44,8 +44,9 @@ var (
 
 	// a global variable that we expose to our tasks. User can use `{{name}}` to access them
 	// These macro are define in our aggregator yaml config file under `macros`
-	macroEnvs map[string]string
-	cache     *bigcache.BigCache
+	macroVars    map[string]string
+	macroSecrets map[string]string
+	cache        *bigcache.BigCache
 
 	defaultSalt = big.NewInt(0)
 )
@@ -56,8 +57,12 @@ func SetLogger(mylogger sdklogging.Logger) {
 }
 
 // Set the global macro system. macros are static, immutable and available to  all tasks at runtime
-func SetMacro(v map[string]string) {
-	macroEnvs = v
+func SetMacroVars(v map[string]string) {
+	macroVars = v
+}
+
+func SetMacroSecrets(v map[string]string) {
+	macroSecrets = v
 }
 
 func SetCache(c *bigcache.BigCache) {

core/taskengine/macros/vars.go

@@ -9,11 +9,21 @@ func Render(text []byte, vars map[string]string) string {
 	return RenderString(string(text), vars)
 }
 
-// TODO: Add more variable and coument these macros
+// TODO: Add more variable and documents these macros
 func RenderString(text string, vars map[string]string) string {
 	for k, v := range vars {
 		text = strings.ReplaceAll(text, fmt.Sprintf("{{%s}}", k), v)
 	}
 
 	return text
 }
+
+// TODO: document all of our available secrets
+// There is a certain operation we let use use it, but don't let user see it. Example to setup email or notifiction, behind the scene, they require an API key. So they can use their API key to send notification and craft the message the way they want, but they cannot see it.
+func RenderSecrets(text string, vars map[string]string) string {
+	for k, v := range vars {
+		text = strings.ReplaceAll(text, fmt.Sprintf("${{secrets.%s}}", k), v)
+	}
+
+	return text
+}

core/taskengine/macros/vars_test.go

@@ -0,0 +1,15 @@
+package macros
+
+import (
+	"testing"
+)
+
+func TestRenderSecret(t *testing.T) {
+	text := RenderSecrets("this has ${{secrets.foo_token}}", map[string]string{
+		"foo_token": "123abc",
+	})
+
+	if text != "this has 123abc" {
+		t.Errorf("render secret doesn't render final text that contains the secrets. expect `this has 123abc` but got %s", text)
+	}
+}

core/taskengine/vm.go

@@ -351,7 +351,7 @@ func (v *VM) runRestApi(stepID string, nodeValue *avsproto.RestAPINode) (*avspro
 	// only evaluate string when there is string interpolation
 	if nodeValue.Body != "" && (strings.Contains(nodeValue.Body, "$") || strings.Contains(nodeValue.Body, "`")) {
 		nodeValue2 := &avsproto.RestAPINode{
-			Url:     macros.RenderString(nodeValue.Url, macroEnvs),
+			Url:     macros.RenderSecrets(nodeValue.Url, macroSecrets),
 			Headers: nodeValue.Headers,
 			Method:  nodeValue.Method,
 			Body:    strings.Clone(nodeValue.Body),