Skip to content

Commit 43ab83f

Browse files
KKouldKKould
authored
feat: impl function strip_null_value for Variant (#17640)
* feat: impl function `strip_null_value` for Variant Signed-off-by: Kould <kould2333@gmail.com> * test: add unit test for `strip_null_value` on variant.rs Signed-off-by: Kould <kould2333@gmail.com> --------- Signed-off-by: Kould <kould2333@gmail.com>
1 parent 3981c46 commit 43ab83f

File tree

5 files changed

+135
-0
lines changed

5 files changed

+135
-0
lines changed

‎src/query/functions/src/scalars/variant.rs

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1869,6 +1869,19 @@ pub fn register(registry: &mut FunctionRegistry) {
18691869
},
18701870
}))
18711871
});
1872+
1873+
registry.register_1_arg_core::<NullableType<VariantType>, NullableType<VariantType>, _, _>(
1874+
"strip_null_value",
1875+
|_, _| FunctionDomain::Full,
1876+
vectorize_1_arg::<NullableType<VariantType>, NullableType<VariantType>>(|val, _| {
1877+
val.and_then(|v| {
1878+
if matches!(RawJsonb::new(v).is_null(), Ok(true)) {
1879+
return None;
1880+
}
1881+
Some(v.to_vec())
1882+
})
1883+
}),
1884+
);
18721885
}
18731886

18741887
fn json_array_fn(args: &[Value<AnyType>], ctx: &mut EvalContext) -> Value<AnyType> {

‎src/query/functions/tests/it/scalars/testdata/function_list.txt

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3449,6 +3449,7 @@ Functions overloads:
34493449
1 strcmp(String NULL, String NULL) :: Int8 NULL
34503450
0 string_to_h3(String) :: UInt64
34513451
1 string_to_h3(String NULL) :: UInt64 NULL
3452+
0 strip_null_value(Variant NULL) :: Variant NULL
34523453
0 sub_bitmap(Bitmap, UInt64, UInt64) :: Bitmap
34533454
1 sub_bitmap(Bitmap NULL, UInt64 NULL, UInt64 NULL) :: Bitmap NULL
34543455
0 substr(String, Int64) :: String

‎src/query/functions/tests/it/scalars/testdata/variant.txt

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5815,3 +5815,75 @@ evaluation (internal):
58155815
+--------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
58165816

58175817

5818+
ast : strip_null_value(parse_json('1234'))
5819+
raw expr : strip_null_value(parse_json('1234'))
5820+
checked expr : strip_null_value<Variant NULL>(CAST(parse_json<String>("1234") AS Variant NULL))
5821+
optimized expr : 0x20000000200000035004d2
5822+
output type : Variant NULL
5823+
output domain : Undefined
5824+
output : '1234'
5825+
5826+
5827+
ast : strip_null_value(parse_json('null'))
5828+
raw expr : strip_null_value(parse_json('null'))
5829+
checked expr : strip_null_value<Variant NULL>(CAST(parse_json<String>("null") AS Variant NULL))
5830+
optimized expr : NULL
5831+
output type : Variant NULL
5832+
output domain : {NULL}
5833+
output : NULL
5834+
5835+
5836+
ast : strip_null_value(null)
5837+
raw expr : strip_null_value(NULL)
5838+
checked expr : strip_null_value<Variant NULL>(CAST(NULL AS Variant NULL))
5839+
optimized expr : NULL
5840+
output type : Variant NULL
5841+
output domain : {NULL}
5842+
output : NULL
5843+
5844+
5845+
ast : strip_null_value(parse_json(s))
5846+
raw expr : strip_null_value(parse_json(s::String))
5847+
checked expr : strip_null_value<Variant NULL>(CAST(parse_json<String>(s) AS Variant NULL))
5848+
evaluation:
5849+
+--------+-------------------------------------------------------+--------------------+
5850+
| | s | Output |
5851+
+--------+-------------------------------------------------------+--------------------+
5852+
| Type | String | Variant NULL |
5853+
| Domain | {"[\"a\",\"b\",\"c\"]"..="{ \"a\": 1, \"b\": null }"} | Unknown |
5854+
| Row 0 | '{ "a": 1, "b": null }' | '{"a":1,"b":null}' |
5855+
| Row 1 | 'null' | NULL |
5856+
| Row 2 | '["a","b","c"]' | '["a","b","c"]' |
5857+
+--------+-------------------------------------------------------+--------------------+
5858+
evaluation (internal):
5859+
+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
5860+
| Column | Data |
5861+
+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
5862+
| s | StringColumn[{ "a": 1, "b": null }, null, ["a","b","c"]] |
5863+
| Output | NullableColumn { column: BinaryColumn { data: 0x40000002100000011000000120000002000000006162500180000003100000011000000110000001616263, offsets: [0, 24, 24, 43] }, validity: [0b_____101] } |
5864+
+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
5865+
5866+
5867+
ast : strip_null_value(parse_json(s))
5868+
raw expr : strip_null_value(parse_json(s::String NULL))
5869+
checked expr : strip_null_value<Variant NULL>(parse_json<String NULL>(s))
5870+
evaluation:
5871+
+--------+----------------------------------------------------------------+--------------------+
5872+
| | s | Output |
5873+
+--------+----------------------------------------------------------------+--------------------+
5874+
| Type | String NULL | Variant NULL |
5875+
| Domain | {"[\"a\",\"b\",\"c\"]"..="{ \"a\": 1, \"b\": null }"} ∪ {NULL} | Unknown |
5876+
| Row 0 | '{ "a": 1, "b": null }' | '{"a":1,"b":null}' |
5877+
| Row 1 | 'null' | NULL |
5878+
| Row 2 | NULL | NULL |
5879+
| Row 3 | '["a","b","c"]' | '["a","b","c"]' |
5880+
+--------+----------------------------------------------------------------+--------------------+
5881+
evaluation (internal):
5882+
+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
5883+
| Column | Data |
5884+
+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
5885+
| s | NullableColumn { column: StringColumn[{ "a": 1, "b": null }, null, , ["a","b","c"]], validity: [0b____1011] } |
5886+
| Output | NullableColumn { column: BinaryColumn { data: 0x40000002100000011000000120000002000000006162500180000003100000011000000110000001616263, offsets: [0, 24, 24, 24, 43] }, validity: [0b____1001] } |
5887+
+--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
5888+
5889+

‎src/query/functions/tests/it/scalars/variant.rs

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -72,6 +72,7 @@ fn test_variant() {
7272
test_json_object_insert(file);
7373
test_json_object_delete(file);
7474
test_json_object_pick(file);
75+
test_strip_null_value(file);
7576
}
7677

7778
fn test_parse_json(file: &mut impl Write) {
@@ -2131,3 +2132,31 @@ fn test_json_object_pick(file: &mut impl Write) {
21312132
),
21322133
)]);
21332134
}
2135+
2136+
fn test_strip_null_value(file: &mut impl Write) {
2137+
run_ast(file, "strip_null_value(parse_json('1234'))", &[]);
2138+
run_ast(file, "strip_null_value(parse_json('null'))", &[]);
2139+
run_ast(file, "strip_null_value(null)", &[]);
2140+
2141+
run_ast(file, "strip_null_value(parse_json(s))", &[(
2142+
"s",
2143+
StringType::from_data(vec![
2144+
r#"{ "a": 1, "b": null }"#,
2145+
"null",
2146+
"[\"a\",\"b\",\"c\"]",
2147+
]),
2148+
)]);
2149+
2150+
run_ast(file, "strip_null_value(parse_json(s))", &[(
2151+
"s",
2152+
StringType::from_data_with_validity(
2153+
vec![
2154+
r#"{ "a": 1, "b": null }"#,
2155+
"null",
2156+
"",
2157+
"[\"a\",\"b\",\"c\"]",
2158+
],
2159+
vec![true, true, false, true],
2160+
),
2161+
)]);
2162+
}

‎tests/sqllogictests/suites/query/functions/02_0065_function_json.test

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1504,3 +1504,23 @@ SELECT id, json_object_pick(v1, 'a', 'k1') from t5
15041504

15051505
statement ok
15061506
DROP TABLE IF EXISTS t5
1507+
1508+
statement ok
1509+
CREATE OR REPLACE TABLE t6 (id INT, data VARIANT);
1510+
1511+
statement ok
1512+
INSERT INTO t6 VALUES
1513+
(1, '{"name": "Alice", "age": 30, "address": null}'),
1514+
(2, '{"name": "Bob", "age": null, "address": {"city": "New York", "zip": null}}'),
1515+
(3, '{"name": "Charlie", "age": 25, "address": {"city": "San Francisco", "zip": "94107"}}');
1516+
1517+
query II
1518+
SELECT id, data:address, data:address:zip, STRIP_NULL_VALUE(data:address), STRIP_NULL_VALUE(data:address:zip) FROM t6;
1519+
----
1520+
1 null NULL NULL NULL
1521+
2 {"city":"New York","zip":null} null {"city":"New York","zip":null} NULL
1522+
3 {"city":"San Francisco","zip":"94107"} "94107" {"city":"San Francisco","zip":"94107"} "94107"
1523+
1524+
1525+
statement ok
1526+
DROP TABLE IF EXISTS t6

0 commit comments

Comments
 (0)