jsonpath (#118)

* feat: add keyValToMap and MapToKeyVal

* feat: add jsonpath and jmespath functions

* fix: nil handling of jq/jsonpath/jmespath

* chore: test fixes

* chore: fix typo

Author: moshloop

coll/coll.go

@@ -6,9 +6,12 @@
 package coll
 
 import (
+	"bytes"
+	"errors"
 	"fmt"
 	"reflect"
 	"sort"
+	"strings"
 
 	"github.com/flanksource/gomplate/v3/conv"
 	iconv "github.com/flanksource/gomplate/v3/internal/conv"
@@ -340,3 +343,30 @@ func Flatten(list interface{}, depth int) ([]interface{}, error) {
 	}
 	return out, nil
 }
+
+func MapToKeyVal[T string | any | interface{}](m map[string]T) string {
+	var buf bytes.Buffer
+	for k, v := range m {
+		if buf.Len() > 0 {
+			buf.WriteByte(',')
+
+		}
+		buf.WriteString(k)
+		buf.WriteByte('=')
+		buf.WriteString(fmt.Sprintf("%v", v))
+	}
+	return buf.String()
+}
+
+func KeyValToMap(s string) (map[string]string, error) {
+	m := make(map[string]string)
+	for _, kv := range strings.Split(s, ",") {
+		kv = strings.TrimSpace(kv)
+		parts := strings.Split(kv, "=")
+		if len(parts) != 2 {
+			return nil, errors.New("invalid input string")
+		}
+		m[parts[0]] = parts[1]
+	}
+	return m, nil
+}

coll/coll_test.go

@@ -616,3 +616,17 @@ func TestPick(t *testing.T) {
 
 	assert.EqualValues(t, in, Pick(in, "foo", "bar", ""))
 }
+
+func TestKeyValToMap(t *testing.T) {
+	in := "foo=bar,bar=true"
+	expected := map[string]string{
+		"foo": "bar",
+		"bar": "true",
+	}
+	result, err := KeyValToMap(in)
+	assert.NoError(t, err)
+	assert.EqualValues(t, expected, result)
+	out := MapToKeyVal(expected)
+	assert.EqualValues(t, in, out)
+
+}

coll/jq.go renamed to coll/json.go

@@ -7,8 +7,12 @@ import (
 	"reflect"
 
 	"github.com/itchyny/gojq"
+	"github.com/jmespath/go-jmespath"
+	"github.com/ohler55/ojg/jp"
 )
 
+const NullValue = "NULL_VALUE"
+
 // JQ -
 func JQ(ctx context.Context, jqExpr string, in interface{}) (interface{}, error) {
 	query, err := gojq.Parse(jqExpr)
@@ -52,6 +56,63 @@ func JQ(ctx context.Context, jqExpr string, in interface{}) (interface{}, error)
 	return out, nil
 }
 
+func JMESPath(jmesPath string, in interface{}) (interface{}, error) {
+	// convert input to a supported type, if necessary
+	in, err := jqConvertType(in)
+	if err != nil {
+		return nil, fmt.Errorf("type conversion: %w", err)
+	}
+
+	if inString, ok := in.(string); ok {
+		var v map[string]any
+		if err := json.Unmarshal([]byte(inString), &v); err == nil {
+			in = v
+		}
+	}
+	out, err := jmespath.Search(jmesPath, in)
+
+	if err != nil {
+		return nil, fmt.Errorf("%+w", err)
+	}
+	if out == nil || out == NullValue || out == "" {
+		out = ""
+	}
+
+	return out, nil
+}
+
+func JSONPath(jsonPath string, in interface{}) (interface{}, error) {
+	// convert input to a supported type, if necessary
+	in, err := jqConvertType(in)
+	if err != nil {
+		return nil, fmt.Errorf("type conversion: %w", err)
+	}
+
+	if inString, ok := in.(string); ok {
+		var v map[string]any
+		if err := json.Unmarshal([]byte(inString), &v); err == nil {
+			in = v
+		}
+	}
+
+	x, err := jp.ParseString(jsonPath)
+	if err != nil {
+		return nil, err
+	}
+	out := x.Get(in)
+
+	if len(out) == 1 {
+		if out[0] == NullValue || out[0] == nil {
+			return "", nil
+		}
+		return out[0], nil
+	}
+	if len(out) == 0 {
+		return "", nil
+	}
+	return out, nil
+}
+
 func isSupportableType(in interface{}) bool {
 	switch in.(type) {
 	case map[string]interface{},

coll/jq_test.go renamed to coll/json_test.go

@@ -43,6 +43,10 @@ func CreateCollFuncs(ctx context.Context) map[string]interface{} {
 	f["sort"] = ns.Sort
 	f["jq"] = ns.JQ
 	f["flatten"] = ns.Flatten
+	f["mapToKeyVal"] = coll.MapToKeyVal[any]
+	f["keyValToMap"] = coll.KeyValToMap
+	f["jsonpath"] = coll.JSONPath
+	f["jmespath"] = coll.JMESPath
 	return f
 }
 

funcs/cel_exports.go

@@ -67,6 +67,7 @@ require (
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/itchyny/timefmt-go v0.1.6 // indirect
 	github.com/jeremywohl/flatten v0.0.0-20180923035001-588fe0d4c603 // indirect
+	github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/kr/text v0.2.0 // indirect

funcs/coll.go

@@ -84,6 +84,9 @@ github.com/itchyny/timefmt-go v0.1.6 h1:ia3s54iciXDdzWzwaVKXZPbiXzxxnv1SPGFfM/my
 github.com/itchyny/timefmt-go v0.1.6/go.mod h1:RRDZYC5s9ErkjQvTvvU7keJjxUYzIISJGxm9/mAERQg=
 github.com/jeremywohl/flatten v0.0.0-20180923035001-588fe0d4c603 h1:gSech9iGLFCosfl/DC7BWnpSSh/tQClWnKS2I2vdPww=
 github.com/jeremywohl/flatten v0.0.0-20180923035001-588fe0d4c603/go.mod h1:4AmD/VxjWcI5SRB0n6szE2A6s2fsNHDLO0nAlMHgfLQ=
+github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY=
+github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
+github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=

funcs/coll_gen.go

@@ -23,6 +23,7 @@ import (
 	"github.com/robertkrimen/otto/registry"
 	_ "github.com/robertkrimen/otto/underscore"
 	"github.com/samber/oops"
+	"google.golang.org/protobuf/types/known/structpb"
 )
 
 var funcMap gotemplate.FuncMap
@@ -265,6 +266,9 @@ func RunTemplateContext(ctx commonsContext.Context, environment map[string]any,
 		if err != nil {
 			return "", err
 		}
+		if _, ok := out.(structpb.NullValue); ok || out == nil {
+			return "", nil
+		}
 		return fmt.Sprintf("%v", out), nil
 	}
 

go.mod

@@ -117,6 +117,7 @@ func TestCelColl(t *testing.T) {
 		{nil, `Dict(['a','b', 'c', 'd']).c`, "d"},
 		{nil, `Has(['a','b', 'c'], 'a')`, "true"},
 		{nil, `Has(['a','b', 'c'], 'e')`, "false"},
+		{map[string]interface{}{"kv": "a=b,c=d"}, "keyValToMap(kv).a", "b"},
 	})
 }
 
@@ -170,10 +171,6 @@ func TestCelFilePath(t *testing.T) {
 }
 
 func TestCelJSON(t *testing.T) {
-	person := Person{
-		Name:    "Aditya",
-		Address: &Address{City: "Kathmandu"},
-	}
 
 	personJSONString, _ := json.Marshal(person)
 
@@ -183,19 +180,28 @@ func TestCelJSON(t *testing.T) {
 		{nil, `dyn({'name': 'John'}).toJSON()`, `{"name":"John"}`},
 		{nil, `{'name': 'John'}.toJSON()`, `{"name":"John"}`},
 		{nil, `1.toJSON()`, `1`},
-		{map[string]interface{}{"i": person}, "i.toJSON().JSON().name", "Aditya"},
+		{map[string]interface{}{"i": person}, "i.toJSON().JSON().name", "John Doe"},
 		{map[string]interface{}{"i": person}, `'["1", "2"]'.JSONArray()[0]`, "1"},
 		{map[string]interface{}{"i": map[string]string{"name": "aditya"}}, `i.toJSON()`, `{"name":"aditya"}`},
 
 		{nil, `'{"name": "John"}'.JSON().name`, `John`},
 		{nil, `'{"name": "Alice", "age": 30}'.JSON().name`, `Alice`},
 		{nil, `'[1, 2, 3, 4, 5]'.JSONArray()[0]`, `1`},
-		{map[string]interface{}{"i": person}, "i.toJSONPretty('\t')", "{\n\t\"Address\": {\n\t\t\"city_name\": \"Kathmandu\"\n\t},\n\t\"name\": \"Aditya\"\n}"},
-		{nil, "[\"Alice\", 30].toJSONPretty('\t')", "[\n\t\"Alice\",\n\t30\n]"},
+		{map[string]interface{}{"i": person}, "i.toJSONPretty('\t').JSON().addresses[0].country", "Nepal"},
 		{nil, "{'name': 'aditya'}.toJSONPretty('\t')", "{\n\t\"name\": \"aditya\"\n}"},
 
-		// JQ
+		{map[string]interface{}{"i": person}, "jsonpath('$.addresses[-1:].city_name', i)", "New York"},
+		{map[string]interface{}{"i": person}, "jmespath('addresses[*].city_name | [0]', i)", "Kathmandu"},
+		//FIXME: jmespath function return a parse error
+		{map[string]interface{}{"i": person}, "jmespath('length(addresses)', i)", "3"},
+		{map[string]interface{}{"i": person}, "jmespath('ceil(`1.2`)', i)", "2"},
+		//FIXME: jmespath always returns a list
+		{map[string]interface{}{"i": person}, "jmespath('name', i)", "John Doe"},
+		{map[string]interface{}{"i": person}, "jmespath('Address.country', i)", ""},
+		{map[string]interface{}{"i": person}, "jsonpath('Address.country', i)", ""},
+
 		{map[string]interface{}{"i": person}, "jq('.Address.city_name', i)", "Kathmandu"},
+		{map[string]interface{}{"i": person}, "jq('.Address.country', i)", ""},
 		{map[string]interface{}{"i": personJSONString}, "jq('.Address.city_name', i)", "Kathmandu"},
 	})
 }
@@ -313,9 +319,9 @@ func TestCelMaps(t *testing.T) {
 		{m, "x.a", "b"},
 		{m, "x.c", "1"},
 		{m, "x.d", "true"},
-		{m, "x.?e", "<nil>"},
+		{m, "x.?e", ""},
 		{m, "x.?f.?a", "5"},
-		{m, "x.?f.?b", "<nil>"},
+		{m, "x.?f.?b", ""},
 		{nil, "{'a': 'c'}.merge({'b': 'd'}).keys().join(',')", "a,b"},
 		{nil, "{'a': '1', 'b': '2', 'c': '3'}.pick(['a', 'c']).keys()", "[a c]"},
 		{nil, "{'a': '1', 'b': '2', 'c': '3'}.omit(['b']).keys()", "[a c]"},