Skip to content

Update json payload query support #26

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 10 commits into from
Oct 21, 2024
Merged

Update json payload query support #26

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
87753a3
wip
fimac Oct 21, 2024
84d50fc
update payload and queries
fimac Oct 21, 2024
c4ae699
update naming of query functions
fimac Oct 21, 2024
4da9810
make serializeQuery private
fimac Oct 21, 2024
cd9dc75
add goeql tests for new query formats
fimac Oct 21, 2024
9366f2e
point to central release folder for eql dsl sql file
fimac Oct 21, 2024
43dc457
Document functions idiomatically
auxesis Oct 21, 2024
ff650ca
Simplify the function
auxesis Oct 21, 2024
bdc63ee
Remove unnecessary assignment
auxesis Oct 21, 2024
b76f6ef
Make Q a concrete type, per the spec
auxesis Oct 21, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 28 additions & 6 deletions languages/go/goeql/goeql.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ type EncryptedColumn struct {
P string `json:"p"`
I TableColumn `json:"i"`
V int `json:"v"`
Q string `json:"q"`
}

// EncryptedText is a string value to be encrypted
Expand Down Expand Up @@ -149,9 +150,29 @@ func (eb *EncryptedBool) Deserialize(data []byte) (EncryptedBool, error) {
return false, fmt.Errorf("invalid format: missing 'p' field")
}

// SerializeQuery produces a jsonb payload used by EQL query functions to perform search operations like equality checks, range queries, and unique constraints.
func SerializeQuery(value any, table string, column string) ([]byte, error) {
query, err := ToEncryptedColumn(value, table, column)
// MatchQuery serializes a plaintext value used in a match query
func MatchQuery(value any, table string, column string) ([]byte, error) {
return serializeQuery(value, table, column, "match")
}

// OreQuery serializes a plaintext value used in an ore query
func OreQuery(value any, table string, column string) ([]byte, error) {
return serializeQuery(value, table, column, "ore")
}

// UniqueQuery serializes a plaintext value used in a unique query
func UniqueQuery(value any, table string, column string) ([]byte, error) {
return serializeQuery(value, table, column, "unique")
}

// JsonbQuery serializes a plaintext value used in a jsonb query
func JsonbQuery(value any, table string, column string) ([]byte, error) {
return serializeQuery(value, table, column, "ste_vec")
}

// serializeQuery produces a jsonb payload used by EQL query functions to perform search operations like equality checks, range queries, and unique constraints.
func serializeQuery(value any, table string, column string, queryType string) ([]byte, error) {
query, err := ToEncryptedColumn(value, table, column, queryType)
if err != nil {
return nil, fmt.Errorf("error converting to EncryptedColumn: %v", err)
}
Expand All @@ -165,14 +186,15 @@ func SerializeQuery(value any, table string, column string) ([]byte, error) {
}

// ToEncryptedColumn converts a plaintext value to a string, and returns the EncryptedColumn struct for inserting into a database.
func ToEncryptedColumn(value any, table string, column string) (EncryptedColumn, error) {
func ToEncryptedColumn(value any, table string, column string, queryType ...string) (EncryptedColumn, error) {
str, err := convertToString(value)
if err != nil {
return EncryptedColumn{}, fmt.Errorf("error: %v", err)
}

data := EncryptedColumn{K: "pt", P: str, I: TableColumn{T: table, C: column}, V: 1}

if queryType != nil {
data.Q = queryType[0]
}
return data, nil
}

Expand Down
119 changes: 95 additions & 24 deletions languages/go/goeql/goeql_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -210,34 +210,105 @@ func TestEncryptedBool_Deserialize(t *testing.T) {
}
}

// Test SerializeQuery Function
func TestSerializeQuery(t *testing.T) {
tests := []struct {
value interface{}
table string
column string
expectedP string
}{
{value: "test_string", table: "table1", column: "column1", expectedP: "test_string"},
{value: 123, table: "table2", column: "column2", expectedP: "123"},
{value: true, table: "table3", column: "column3", expectedP: "true"},
{value: map[string]interface{}{"key": "value"}, table: "table4", column: "column4", expectedP: `{"key":"value"}`},
func TestMatchQuerySerialization(t *testing.T) {
value := "test_string"
table := "table1"
column := "column1"
expectedP := "test_string"
expectedQ := "match"

serializedData, err := MatchQuery(value, table, column)
if err != nil {
t.Fatalf("SerializeQuery returned error: %v", err)
}

for _, tt := range tests {
serializedData, err := SerializeQuery(tt.value, tt.table, tt.column)
if err != nil {
t.Fatalf("SerializeQuery returned error: %v", err)
}
var ec EncryptedColumn
if err := json.Unmarshal(serializedData, &ec); err != nil {
t.Fatalf("Error unmarshaling serialized data: %v", err)
}

var ec EncryptedColumn
if err := json.Unmarshal(serializedData, &ec); err != nil {
t.Fatalf("Error unmarshaling serialized data: %v", err)
}
if ec.P != expectedP {
t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P)
}

if ec.P != tt.expectedP {
t.Errorf("Expected P to be '%s', got '%s'", tt.expectedP, ec.P)
}
if ec.Q != expectedQ {
t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q)
}
}
func TestOreQuerySerialization(t *testing.T) {
value := 123
table := "table1"
column := "column1"
expectedP := "123"
expectedQ := "ore"

serializedData, err := OreQuery(value, table, column)
if err != nil {
t.Fatalf("SerializeQuery returned error: %v", err)
}

var ec EncryptedColumn
if err := json.Unmarshal(serializedData, &ec); err != nil {
t.Fatalf("Error unmarshaling serialized data: %v", err)
}

if ec.P != expectedP {
t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P)
}

if ec.Q != expectedQ {
t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q)
}
}

func TestUniqueQuerySerialization(t *testing.T) {
value := true
table := "table1"
column := "column1"
expectedP := "true"
expectedQ := "unique"

serializedData, err := UniqueQuery(value, table, column)
if err != nil {
t.Fatalf("SerializeQuery returned error: %v", err)
}

var ec EncryptedColumn
if err := json.Unmarshal(serializedData, &ec); err != nil {
t.Fatalf("Error unmarshaling serialized data: %v", err)
}

if ec.P != expectedP {
t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P)
}

if ec.Q != expectedQ {
t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q)
}
}

func TestJsonbQuerySerialization(t *testing.T) {
value := map[string]interface{}{"key": "value"}
table := "table1"
column := "column1"
expectedP := `{"key":"value"}`
expectedQ := "ste_vec"

serializedData, err := JsonbQuery(value, table, column)
if err != nil {
t.Fatalf("SerializeQuery returned error: %v", err)
}

var ec EncryptedColumn
if err := json.Unmarshal(serializedData, &ec); err != nil {
t.Fatalf("Error unmarshaling serialized data: %v", err)
}

if ec.P != expectedP {
t.Errorf("Expected P to be '%s', got '%s'", expectedP, ec.P)
}
if ec.Q != expectedQ {
t.Errorf("Expected Q to be '%s', got '%s'", expectedQ, ec.Q)
}
}

Expand Down
52 changes: 29 additions & 23 deletions languages/go/xorm/e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,10 @@ func TestWhereQueryOnUnencryptedColumn(t *testing.T) {
t.Fatalf("Could not insert new example: %v", err)
}

var example Example
var returnedExample Example
text := "sydney"

has, err := engine.Where("non_encrypted_field = ?", text).Get(&example)
has, err := engine.Where("non_encrypted_field = ?", text).Get(&returnedExample)
if err != nil {
t.Fatalf("Could not retrieve example: %v", err)
}
Expand All @@ -63,10 +63,10 @@ func TestWhereQueryOnUnencryptedColumn(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, newExample.NonEncryptedField, example.NonEncryptedField, "NonEncryptedField does not match")
assert.Equal(t, newExample.EncryptedIntField, example.EncryptedIntField, "EncryptedIntField does not match")
assert.Equal(t, newExample.EncryptedTextField, example.EncryptedTextField, "EncryptedTextField does not match")
assert.Equal(t, newExample.EncryptedJsonbField, example.EncryptedJsonbField, "EncryptedJsonbField does not match")
assert.Equal(t, newExample.NonEncryptedField, returnedExample.NonEncryptedField, "NonEncryptedField does not match")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Updated these assertions as the expected value should be the first value in the assertion.

func assert.Equal(t assert.TestingT, expected interface{}, actual interface{}, msgAndArgs ...interface{}) bool

assert.Equal(t, newExample.EncryptedIntField, returnedExample.EncryptedIntField, "EncryptedIntField does not match")
assert.Equal(t, newExample.EncryptedTextField, returnedExample.EncryptedTextField, "EncryptedTextField does not match")
assert.Equal(t, newExample.EncryptedJsonbField, returnedExample.EncryptedJsonbField, "EncryptedJsonbField does not match")
}

func TestMatchQueryLongString(t *testing.T) {
Expand All @@ -82,6 +82,12 @@ func TestMatchQueryLongString(t *testing.T) {
}

examples := []Example{
{
NonEncryptedField: "brisbane",
EncryptedIntField: 23,
EncryptedTextField: "another string that shouldn't be returned",
EncryptedJsonbField: jsonData,
},
Copy link
Contributor Author

@fimac fimac Oct 21, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Added another example to insert to ensure failures are picked up

https://cipherstash.slack.com/archives/C05CX3G2YQY/p1729475070543969

{
NonEncryptedField: "sydney",
EncryptedIntField: 23,
Expand All @@ -102,9 +108,9 @@ func TestMatchQueryLongString(t *testing.T) {
t.Errorf("Error inserting examples: %v", err)
}

assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows")
assert.Equal(t, int64(3), inserted, "Expected to insert 2 rows")

query, err := goeql.SerializeQuery("this", "examples", "encrypted_text_field")
query, err := goeql.MatchQuery("this", "examples", "encrypted_text_field")
if err != nil {
log.Fatalf("Error marshaling encrypted_text_field query: %v", err)
}
Expand All @@ -119,7 +125,7 @@ func TestMatchQueryLongString(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedTextField, EncryptedTextField("this is a long string"), "EncryptedTextField should match")
assert.Equal(t, EncryptedTextField("this is a long string"), returnedExample.EncryptedTextField, "EncryptedTextField should match")
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

same as above, expected should be first in the assertion

}

func TestMatchQueryEmail(t *testing.T) {
Expand Down Expand Up @@ -157,7 +163,7 @@ func TestMatchQueryEmail(t *testing.T) {

assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows")

query, err := goeql.SerializeQuery("test", "examples", "encrypted_text_field")
query, err := goeql.MatchQuery("test", "examples", "encrypted_text_field")
if err != nil {
log.Fatalf("Error marshaling encrypted_text_field query: %v", err)
}
Expand All @@ -172,7 +178,7 @@ func TestMatchQueryEmail(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedTextField, EncryptedTextField("testemail@test.com"), "EncryptedTextField should match")
assert.Equal(t, EncryptedTextField("testemail@test.com"), returnedExample.EncryptedTextField, "EncryptedTextField should match")
}

func TestJsonbQuerySimple(t *testing.T) {
Expand Down Expand Up @@ -218,7 +224,7 @@ func TestJsonbQuerySimple(t *testing.T) {
},
}

query, errTwo := goeql.SerializeQuery(jsonbQuery, "examples", "encrypted_jsonb_field")
query, errTwo := goeql.JsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field")
if errTwo != nil {
log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo)
}
Expand All @@ -233,7 +239,7 @@ func TestJsonbQuerySimple(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedJsonbField, EncryptedJsonbField(expectedJson), "EncryptedJsonb field should match")
assert.Equal(t, EncryptedJsonbField(expectedJson), returnedExample.EncryptedJsonbField, "EncryptedJsonb field should match")
}

func TestJsonbQueryNested(t *testing.T) {
Expand Down Expand Up @@ -284,7 +290,7 @@ func TestJsonbQueryNested(t *testing.T) {
},
}

query, errTwo := goeql.SerializeQuery(jsonbQuery, "examples", "encrypted_jsonb_field")
query, errTwo := goeql.JsonbQuery(jsonbQuery, "examples", "encrypted_jsonb_field")
if errTwo != nil {
log.Fatalf("Error marshaling encrypted_jsonb_field: %v", errTwo)
}
Expand All @@ -299,7 +305,7 @@ func TestJsonbQueryNested(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedJsonbField, EncryptedJsonbField(expectedJson), "EncryptedJsonb field should match")
assert.Equal(t, EncryptedJsonbField(expectedJson), returnedExample.EncryptedJsonbField, "EncryptedJsonb field should match")
}

func TestOreStringRangeQuery(t *testing.T) {
Expand Down Expand Up @@ -331,7 +337,7 @@ func TestOreStringRangeQuery(t *testing.T) {
assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows")

// Query
query, errQuery := goeql.SerializeQuery("tree", "examples", "encrypted_text_field")
query, errQuery := goeql.OreQuery("tree", "examples", "encrypted_text_field")
if errQuery != nil {
log.Fatalf("err: %v", errQuery)
}
Expand All @@ -346,7 +352,7 @@ func TestOreStringRangeQuery(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedTextField, expected, "EncryptedText field should match")
assert.Equal(t, expected, returnedExample.EncryptedTextField, "EncryptedText field should match")
}

func TestOreIntRangeQuery(t *testing.T) {
Expand Down Expand Up @@ -378,7 +384,7 @@ func TestOreIntRangeQuery(t *testing.T) {
assert.Equal(t, int64(2), inserted, "Expected to insert 2 rows")

// Query
query, errQuery := goeql.SerializeQuery(32, "examples", "encrypted_int_field")
query, errQuery := goeql.OreQuery(32, "examples", "encrypted_int_field")
if errQuery != nil {
log.Fatalf("err: %v", errQuery)
}
Expand All @@ -393,7 +399,7 @@ func TestOreIntRangeQuery(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedIntField, expected, "EncryptedInt field should match")
assert.Equal(t, expected, returnedExample.EncryptedIntField, "EncryptedInt field should match")
}

func TestOreBoolRangeQuery(t *testing.T) {
Expand Down Expand Up @@ -434,7 +440,7 @@ func TestOreBoolRangeQuery(t *testing.T) {
assert.Equal(t, int64(3), inserted, "Expected to insert 3 rows")

// Query
query, errQuery := goeql.SerializeQuery(false, "examples", "encrypted_bool_field")
query, errQuery := goeql.OreQuery(false, "examples", "encrypted_bool_field")
if errQuery != nil {
log.Fatalf("err: %v", errQuery)
}
Expand All @@ -449,7 +455,7 @@ func TestOreBoolRangeQuery(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedBoolField, expected, "EncryptedBool field should match")
assert.Equal(t, expected, returnedExample.EncryptedBoolField, "EncryptedBool field should match")
}

func TestUniqueStringQuery(t *testing.T) {
Expand Down Expand Up @@ -490,7 +496,7 @@ func TestUniqueStringQuery(t *testing.T) {
assert.Equal(t, int64(3), inserted, "Expected to insert 3 rows")

// Query
query, errQuery := goeql.SerializeQuery("testing two", "examples", "encrypted_text_field")
query, errQuery := goeql.UniqueQuery("testing two", "examples", "encrypted_text_field")
if errQuery != nil {
log.Fatalf("err: %v", errQuery)
}
Expand All @@ -505,5 +511,5 @@ func TestUniqueStringQuery(t *testing.T) {
t.Errorf("Expected has to equal true, got: %v", has)
}

assert.Equal(t, returnedExample.EncryptedTextField, expected, "EncryptedText field should match")
assert.Equal(t, expected, returnedExample.EncryptedTextField, "EncryptedText field should match")
}
Loading