From c3b0c0c4396db6e36fe374b1e7691c99dced62c5 Mon Sep 17 00:00:00 2001 From: baishen Date: Fri, 4 Jul 2025 15:11:55 +0800 Subject: [PATCH 1/3] chore(query): Introduce `enable_extended_json_syntax` setting to control extended JSON syntax --- Cargo.lock | 3 +- Cargo.toml | 1 + src/common/io/src/format_settings.rs | 2 + src/query/expression/src/function.rs | 2 + src/query/expression/src/types/variant.rs | 125 +- .../expression/src/utils/variant_transform.rs | 4 +- src/query/formats/src/common_settings.rs | 1 + .../formats/src/field_decoder/fast_values.rs | 9 +- src/query/formats/src/field_decoder/nested.rs | 9 +- .../src/field_decoder/separated_text.rs | 10 +- src/query/formats/src/file_format_type.rs | 5 + src/query/formats/src/output_format/json.rs | 1 + .../aggregates/aggregate_json_array_agg.rs | 2 +- .../aggregates/aggregate_json_object_agg.rs | 2 +- .../src/scalars/geographic/src/geometry.rs | 22 +- src/query/functions/src/scalars/variant.rs | 498 ++--- .../tests/it/aggregates/testdata/agg.txt | 38 +- .../it/aggregates/testdata/agg_group_by.txt | 38 +- src/query/functions/tests/it/scalars/cast.rs | 140 +- .../tests/it/scalars/testdata/cast.txt | 395 ++++ .../tests/it/scalars/testdata/geometry.txt | 2 +- .../tests/it/scalars/testdata/variant.txt | 1679 +++++++++++++++-- .../functions/tests/it/scalars/variant.rs | 518 +++-- src/query/service/src/sessions/query_ctx.rs | 4 + src/query/settings/src/settings_default.rs | 7 + .../settings/src/settings_getter_setter.rs | 4 + .../recordbatch_to_variant.rs | 19 +- .../src/parquet_variant_table/source.rs | 30 +- .../02_0000_function_aggregate_mix.test | 4 +- ..._function_semi_structureds_parse_json.test | 22 + .../02_0056_function_semi_structureds_as.test | 51 + .../functions/02_0065_function_json.test | 11 + 32 files changed, 2938 insertions(+), 720 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 6ef8ce8bfd002..583d8e851dc2e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9404,8 +9404,7 @@ dependencies = [ [[package]] name = "jsonb" version = "0.5.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "153dfebb6ad4484c84a5ae765a18976bf476b8a3f5165ce987aedd59d1f1e6c8" +source = "git+https://github.com/b41sh/jsonb?rev=6a34a381329a45f6cdf77136e5e6e468deed2762#6a34a381329a45f6cdf77136e5e6e468deed2762" dependencies = [ "byteorder", "ethnum", diff --git a/Cargo.toml b/Cargo.toml index 799d14717c64c..8f0599ba52a11 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -653,6 +653,7 @@ backtrace = { git = "https://github.com/rust-lang/backtrace-rs.git", rev = "7226 color-eyre = { git = "https://github.com/eyre-rs/eyre.git", rev = "e5d92c3" } deltalake = { git = "https://github.com/delta-io/delta-rs", rev = "9954bff" } display-more = { git = "https://github.com/databendlabs/display-more", tag = "v0.2.0" } +jsonb = { git = "https://github.com/b41sh/jsonb", rev = "6a34a381329a45f6cdf77136e5e6e468deed2762" } map-api = { git = "https://github.com/databendlabs/map-api", tag = "v0.2.3" } openai_api_rust = { git = "https://github.com/datafuse-extras/openai-api", rev = "819a0ed" } openraft = { git = "https://github.com/databendlabs/openraft", tag = "v0.10.0-alpha.9" } diff --git a/src/common/io/src/format_settings.rs b/src/common/io/src/format_settings.rs index 61c3fda891f44..0b03b2061f810 100644 --- a/src/common/io/src/format_settings.rs +++ b/src/common/io/src/format_settings.rs @@ -24,6 +24,7 @@ pub struct FormatSettings { pub geometry_format: GeometryDataType, pub enable_dst_hour_fix: bool, pub format_null_as_str: bool, + pub enable_extended_json_syntax: bool, } // only used for tests @@ -35,6 +36,7 @@ impl Default for FormatSettings { geometry_format: GeometryDataType::default(), enable_dst_hour_fix: false, format_null_as_str: false, + enable_extended_json_syntax: false, } } } diff --git a/src/query/expression/src/function.rs b/src/query/expression/src/function.rs index d3af570c2e0ed..e50f8bd5e163a 100755 --- a/src/query/expression/src/function.rs +++ b/src/query/expression/src/function.rs @@ -172,6 +172,7 @@ pub struct FunctionContext { pub geometry_output_format: GeometryDataType, pub parse_datetime_ignore_remainder: bool, pub enable_strict_datetime_parser: bool, + pub enable_extended_json_syntax: bool, pub random_function_seed: bool, pub week_start: u8, pub date_format_style: String, @@ -194,6 +195,7 @@ impl Default for FunctionContext { geometry_output_format: Default::default(), parse_datetime_ignore_remainder: false, enable_strict_datetime_parser: true, + enable_extended_json_syntax: false, random_function_seed: false, week_start: 0, date_format_style: "oracle".to_string(), diff --git a/src/query/expression/src/types/variant.rs b/src/query/expression/src/types/variant.rs index 835ae66dcece1..8d250275917c6 100644 --- a/src/query/expression/src/types/variant.rs +++ b/src/query/expression/src/types/variant.rs @@ -32,6 +32,7 @@ use super::number::NumberScalar; use super::timestamp::timestamp_to_string; use super::AccessType; use crate::property::Domain; +use crate::types::interval::interval_to_string; use crate::types::map::KvPair; use crate::types::AnyType; use crate::types::ArgType; @@ -224,6 +225,7 @@ impl VariantType { pub fn cast_scalar_to_variant( scalar: ScalarRef, tz: &TimeZone, + enable_extended: bool, buf: &mut Vec, table_data_type: Option<&TableDataType>, ) { @@ -243,41 +245,69 @@ pub fn cast_scalar_to_variant( NumberScalar::Float32(n) => n.0.into(), NumberScalar::Float64(n) => n.0.into(), }, - ScalarRef::Decimal(x) => match x { - DecimalScalar::Decimal64(value, size) => { - let dec = jsonb::Decimal64 { - scale: size.scale(), - value, - }; - jsonb::Value::Number(jsonb::Number::Decimal64(dec)) - } - DecimalScalar::Decimal128(value, size) => { - let dec = jsonb::Decimal128 { - scale: size.scale(), - value, - }; - jsonb::Value::Number(jsonb::Number::Decimal128(dec)) - } - DecimalScalar::Decimal256(value, size) => { - let dec = jsonb::Decimal256 { - scale: size.scale(), - value: value.0, - }; - jsonb::Value::Number(jsonb::Number::Decimal256(dec)) + ScalarRef::Decimal(x) => { + if enable_extended { + match x { + DecimalScalar::Decimal64(value, size) => { + let dec = jsonb::Decimal64 { + scale: size.scale(), + value, + }; + jsonb::Value::Number(jsonb::Number::Decimal64(dec)) + } + DecimalScalar::Decimal128(value, size) => { + let dec = jsonb::Decimal128 { + scale: size.scale(), + value, + }; + jsonb::Value::Number(jsonb::Number::Decimal128(dec)) + } + DecimalScalar::Decimal256(value, size) => { + let dec = jsonb::Decimal256 { + scale: size.scale(), + value: value.0, + }; + jsonb::Value::Number(jsonb::Number::Decimal256(dec)) + } + } + } else { + x.to_float64().into() } - }, + } ScalarRef::Boolean(b) => jsonb::Value::Bool(b), - ScalarRef::Binary(s) => jsonb::Value::Binary(s), + ScalarRef::Binary(s) => { + if enable_extended { + jsonb::Value::Binary(s) + } else { + jsonb::Value::String(hex::encode_upper(s).into()) + } + } ScalarRef::String(s) => jsonb::Value::String(s.into()), - ScalarRef::Timestamp(ts) => jsonb::Value::Timestamp(jsonb::Timestamp { value: ts }), - ScalarRef::Date(d) => jsonb::Value::Date(jsonb::Date { value: d }), + ScalarRef::Timestamp(ts) => { + if enable_extended { + jsonb::Value::Timestamp(jsonb::Timestamp { value: ts }) + } else { + timestamp_to_string(ts, tz).to_string().into() + } + } + ScalarRef::Date(d) => { + if enable_extended { + jsonb::Value::Date(jsonb::Date { value: d }) + } else { + date_to_string(d, tz).to_string().into() + } + } ScalarRef::Interval(i) => { - let interval = jsonb::Interval { - months: i.months(), - days: i.days(), - micros: i.microseconds(), - }; - jsonb::Value::Interval(interval) + if enable_extended { + let interval = jsonb::Interval { + months: i.months(), + days: i.days(), + micros: i.microseconds(), + }; + jsonb::Value::Interval(interval) + } else { + interval_to_string(&i).to_string().into() + } } ScalarRef::Array(col) => { let typ = if let Some(TableDataType::Array(typ)) = table_data_type { @@ -285,7 +315,7 @@ pub fn cast_scalar_to_variant( } else { None }; - let items = cast_scalars_to_variants(col.iter(), tz, typ.as_ref()); + let items = cast_scalars_to_variants(col.iter(), tz, enable_extended, typ.as_ref()); let owned_jsonb = OwnedJsonb::build_array(items.iter().map(RawJsonb::new)) .expect("failed to build jsonb array"); buf.extend_from_slice(owned_jsonb.as_ref()); @@ -311,7 +341,7 @@ pub fn cast_scalar_to_variant( _ => unreachable!(), }; let mut val = vec![]; - cast_scalar_to_variant(v, tz, &mut val, typ.as_ref()); + cast_scalar_to_variant(v, tz, enable_extended, &mut val, typ.as_ref()); kvs.insert(key, val); } let owned_jsonb = @@ -344,6 +374,7 @@ pub fn cast_scalar_to_variant( cast_scalar_to_variant( scalar, tz, + enable_extended, &mut builder.data, Some(&typ.remove_nullable()), ); @@ -359,7 +390,7 @@ pub fn cast_scalar_to_variant( .expect("failed to build jsonb object from tuple") } _ => { - let values = cast_scalars_to_variants(fields, tz, None); + let values = cast_scalars_to_variants(fields, tz, enable_extended, None); OwnedJsonb::build_object( values .iter() @@ -378,16 +409,24 @@ pub fn cast_scalar_to_variant( } ScalarRef::Geometry(bytes) => { let geom = Ewkb(bytes).to_json().expect("failed to decode wkb data"); - jsonb::parse_value(geom.as_bytes()) - .expect("failed to parse geojson to json value") + let res = if enable_extended { + jsonb::parse_value(geom.as_bytes()) + } else { + jsonb::parse_value_standard_mode(geom.as_bytes()) + }; + res.expect("failed to parse geojson to json value") .write_to_vec(buf); return; } ScalarRef::Geography(bytes) => { // todo: Implement direct conversion, omitting intermediate processes let geom = Ewkb(bytes.0).to_json().expect("failed to decode wkb data"); - jsonb::parse_value(geom.as_bytes()) - .expect("failed to parse geojson to json value") + let res = if enable_extended { + jsonb::parse_value(geom.as_bytes()) + } else { + jsonb::parse_value_standard_mode(geom.as_bytes()) + }; + res.expect("failed to parse geojson to json value") .write_to_vec(buf); return; } @@ -397,6 +436,7 @@ pub fn cast_scalar_to_variant( vals.iter() .map(|n| ScalarRef::Number(NumberScalar::NUM_TYPE(*n))), tz, + enable_extended, None, ); let owned_jsonb = OwnedJsonb::build_array(items.iter().map(RawJsonb::new)) @@ -412,12 +452,19 @@ pub fn cast_scalar_to_variant( pub fn cast_scalars_to_variants( scalars: impl IntoIterator, tz: &TimeZone, + enable_extended: bool, table_data_type: Option<&TableDataType>, ) -> BinaryColumn { let iter = scalars.into_iter(); let mut builder = BinaryColumnBuilder::with_capacity(iter.size_hint().0, 0); for scalar in iter { - cast_scalar_to_variant(scalar, tz, &mut builder.data, table_data_type); + cast_scalar_to_variant( + scalar, + tz, + enable_extended, + &mut builder.data, + table_data_type, + ); builder.commit_row(); } builder.build() diff --git a/src/query/expression/src/utils/variant_transform.rs b/src/query/expression/src/utils/variant_transform.rs index aeec1b2e15ac5..5b90c6f5af615 100644 --- a/src/query/expression/src/utils/variant_transform.rs +++ b/src/query/expression/src/utils/variant_transform.rs @@ -14,7 +14,7 @@ use databend_common_exception::ErrorCode; use databend_common_exception::Result; -use jsonb::parse_value; +use jsonb::parse_value_standard_mode; use jsonb::RawJsonb; use crate::types::AnyType; @@ -101,7 +101,7 @@ fn transform_scalar(scalar: ScalarRef<'_>, decode: bool) -> Result { let raw_jsonb = RawJsonb::new(data); Scalar::Variant(raw_jsonb.to_string().into_bytes()) } else { - let value = parse_value(data).map_err(|err| { + let value = parse_value_standard_mode(data).map_err(|err| { ErrorCode::UDFDataError(format!("parse json value error: {err}")) })?; Scalar::Variant(value.to_vec()) diff --git a/src/query/formats/src/common_settings.rs b/src/query/formats/src/common_settings.rs index 198be30185c56..08ba7373b4eef 100644 --- a/src/query/formats/src/common_settings.rs +++ b/src/query/formats/src/common_settings.rs @@ -28,6 +28,7 @@ pub struct InputCommonSettings { pub binary_format: BinaryFormat, pub is_rounding_mode: bool, pub enable_dst_hour_fix: bool, + pub enable_extended_json_syntax: bool, } #[derive(Clone)] diff --git a/src/query/formats/src/field_decoder/fast_values.rs b/src/query/formats/src/field_decoder/fast_values.rs index 6b92f62e92332..07d4ba0fd4f33 100644 --- a/src/query/formats/src/field_decoder/fast_values.rs +++ b/src/query/formats/src/field_decoder/fast_values.rs @@ -61,6 +61,7 @@ use databend_common_io::parse_bytes_to_ewkb; use databend_common_io::prelude::FormatSettings; use databend_common_io::Interval; use jsonb::parse_value; +use jsonb::parse_value_standard_mode; use lexical_core::FromLexical; use num_traits::NumCast; @@ -95,6 +96,7 @@ impl FastFieldDecoderValues { binary_format: Default::default(), is_rounding_mode, enable_dst_hour_fix: format.enable_dst_hour_fix, + enable_extended_json_syntax: format.enable_extended_json_syntax, }, } } @@ -468,7 +470,12 @@ impl FastFieldDecoderValues { ) -> Result<()> { let mut buf = Vec::new(); self.read_string_inner(reader, &mut buf, positions)?; - match parse_value(&buf) { + let res = if self.common_settings.enable_extended_json_syntax { + parse_value(&buf) + } else { + parse_value_standard_mode(&buf) + }; + match res { Ok(value) => { value.write_to_vec(&mut column.data); column.commit_row(); diff --git a/src/query/formats/src/field_decoder/nested.rs b/src/query/formats/src/field_decoder/nested.rs index 753fe52f694b0..e4df9e9139724 100644 --- a/src/query/formats/src/field_decoder/nested.rs +++ b/src/query/formats/src/field_decoder/nested.rs @@ -54,6 +54,7 @@ use databend_common_io::parse_bitmap; use databend_common_io::parse_bytes_to_ewkb; use databend_common_io::Interval; use jsonb::parse_value; +use jsonb::parse_value_standard_mode; use lexical_core::FromLexical; use crate::binary::decode_binary; @@ -87,6 +88,7 @@ impl NestedValues { binary_format: Default::default(), is_rounding_mode: options_ext.is_rounding_mode, enable_dst_hour_fix: options_ext.enable_dst_hour_fix, + enable_extended_json_syntax: options_ext.enable_extended_json_syntax, }, } } @@ -306,7 +308,12 @@ impl NestedValues { ) -> Result<()> { let mut buf = Vec::new(); self.read_string_inner(reader, &mut buf)?; - match parse_value(&buf) { + let res = if self.common_settings.enable_extended_json_syntax { + parse_value(&buf) + } else { + parse_value_standard_mode(&buf) + }; + match res { Ok(value) => { value.write_to_vec(&mut column.data); column.commit_row(); diff --git a/src/query/formats/src/field_decoder/separated_text.rs b/src/query/formats/src/field_decoder/separated_text.rs index 7f060b694529f..2049a0c03fc46 100644 --- a/src/query/formats/src/field_decoder/separated_text.rs +++ b/src/query/formats/src/field_decoder/separated_text.rs @@ -52,6 +52,7 @@ use databend_common_io::Interval; use databend_common_meta_app::principal::CsvFileFormatParams; use databend_common_meta_app::principal::TsvFileFormatParams; use jsonb::parse_value; +use jsonb::parse_value_standard_mode; use lexical_core::FromLexical; use num_traits::NumCast; @@ -89,6 +90,7 @@ impl SeparatedTextDecoder { binary_format: params.binary_format, is_rounding_mode: options_ext.is_rounding_mode, enable_dst_hour_fix: options_ext.enable_dst_hour_fix, + enable_extended_json_syntax: options_ext.enable_extended_json_syntax, }, nested_decoder: NestedValues::create(options_ext), } @@ -106,6 +108,7 @@ impl SeparatedTextDecoder { binary_format: Default::default(), is_rounding_mode: options_ext.is_rounding_mode, enable_dst_hour_fix: options_ext.enable_dst_hour_fix, + enable_extended_json_syntax: options_ext.enable_extended_json_syntax, }, nested_decoder: NestedValues::create(options_ext), } @@ -287,7 +290,12 @@ impl SeparatedTextDecoder { } fn read_variant(&self, column: &mut BinaryColumnBuilder, data: &[u8]) -> Result<()> { - match parse_value(data) { + let res = if self.common_settings.enable_extended_json_syntax { + parse_value(data) + } else { + parse_value_standard_mode(data) + }; + match res { Ok(value) => { value.write_to_vec(&mut column.data); column.commit_row(); diff --git a/src/query/formats/src/file_format_type.rs b/src/query/formats/src/file_format_type.rs index ba1388e501497..aa9471cd9437f 100644 --- a/src/query/formats/src/file_format_type.rs +++ b/src/query/formats/src/file_format_type.rs @@ -52,6 +52,7 @@ pub struct FileFormatOptionsExt { pub is_rounding_mode: bool, pub geometry_format: GeometryDataType, pub enable_dst_hour_fix: bool, + pub enable_extended_json_syntax: bool, } impl FileFormatOptionsExt { @@ -62,6 +63,7 @@ impl FileFormatOptionsExt { let timezone = parse_timezone(settings)?; let jiff_timezone = parse_jiff_timezone(settings)?; let enable_dst_hour_fix = settings.get_enable_dst_hour_fix()?; + let enable_extended_json_syntax = settings.get_enable_extended_json_syntax()?; let geometry_format = settings.get_geometry_output_format()?; let numeric_cast_option = settings .get_numeric_cast_option() @@ -81,6 +83,7 @@ impl FileFormatOptionsExt { is_rounding_mode, geometry_format, enable_dst_hour_fix, + enable_extended_json_syntax, }; Ok(options) } @@ -93,6 +96,7 @@ impl FileFormatOptionsExt { let jiff_timezone = parse_jiff_timezone(settings)?; let geometry_format = settings.get_geometry_output_format()?; let enable_dst_hour_fix = settings.get_enable_dst_hour_fix()?; + let enable_extended_json_syntax = settings.get_enable_extended_json_syntax()?; let mut options = FileFormatOptionsExt { ident_case_sensitive: settings.get_unquoted_ident_case_sensitive()?, headers: 0, @@ -106,6 +110,7 @@ impl FileFormatOptionsExt { is_rounding_mode: true, geometry_format, enable_dst_hour_fix, + enable_extended_json_syntax, }; let suf = &clickhouse_type.suffixes; options.headers = suf.headers; diff --git a/src/query/formats/src/output_format/json.rs b/src/query/formats/src/output_format/json.rs index 1f82421093ad3..dd1f3cabb965c 100644 --- a/src/query/formats/src/output_format/json.rs +++ b/src/query/formats/src/output_format/json.rs @@ -51,6 +51,7 @@ impl JSONOutputFormat { geometry_format: options.geometry_format, enable_dst_hour_fix: options.enable_dst_hour_fix, format_null_as_str: true, + enable_extended_json_syntax: options.enable_extended_json_syntax, }, } } diff --git a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs index f1ba8d35807fa..6b2730b0bee97 100644 --- a/src/query/functions/src/aggregates/aggregate_json_array_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_array_agg.rs @@ -120,7 +120,7 @@ where continue; } let mut val = vec![]; - cast_scalar_to_variant(v.as_ref(), &tz, &mut val, None); + cast_scalar_to_variant(v.as_ref(), &tz, false, &mut val, None); items.push(val); } let owned_jsonb = OwnedJsonb::build_array(items.iter().map(|v| RawJsonb::new(v))) diff --git a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs index 9efe5fc0f6b90..7cd680b4da226 100644 --- a/src/query/functions/src/aggregates/aggregate_json_object_agg.rs +++ b/src/query/functions/src/aggregates/aggregate_json_object_agg.rs @@ -184,7 +184,7 @@ where continue; } let mut val = vec![]; - cast_scalar_to_variant(v.as_ref(), &tz, &mut val, None); + cast_scalar_to_variant(v.as_ref(), &tz, false, &mut val, None); values.push((key, val)); } let owned_jsonb = diff --git a/src/query/functions/src/scalars/geographic/src/geometry.rs b/src/query/functions/src/scalars/geographic/src/geometry.rs index 2fd36b2315f61..6f18c5f4cd9d4 100644 --- a/src/query/functions/src/scalars/geographic/src/geometry.rs +++ b/src/query/functions/src/scalars/geographic/src/geometry.rs @@ -76,6 +76,7 @@ use geozero::GeozeroGeometry; use geozero::ToGeo; use geozero::ToWkb; use jsonb::parse_value; +use jsonb::parse_value_standard_mode; use jsonb::RawJsonb; use num_traits::AsPrimitive; use proj4rs::transform::transform; @@ -134,14 +135,21 @@ pub fn register(registry: &mut FunctionRegistry) { } match ewkb_to_geo(&mut Ewkb(ewkb)).and_then(|(geo, _)| geo_to_json(geo)) { - Ok(json) => match parse_value(json.as_bytes()) { - Ok(json_val) => { - json_val.write_to_vec(&mut builder.data); - } - Err(e) => { - ctx.set_error(builder.len(), e.to_string()); + Ok(json) => { + let res = if ctx.func_ctx.enable_extended_json_syntax { + parse_value(json.as_bytes()) + } else { + parse_value_standard_mode(json.as_bytes()) + }; + match res { + Ok(json_val) => { + json_val.write_to_vec(&mut builder.data); + } + Err(e) => { + ctx.set_error(builder.len(), e.to_string()); + } } - }, + } Err(e) => { ctx.set_error(builder.len(), e.to_string()); } diff --git a/src/query/functions/src/scalars/variant.rs b/src/query/functions/src/scalars/variant.rs index 90ded61aafc8d..bdb766cd621cd 100644 --- a/src/query/functions/src/scalars/variant.rs +++ b/src/query/functions/src/scalars/variant.rs @@ -77,6 +77,7 @@ use jiff::Unit; use jsonb::jsonpath::parse_json_path; use jsonb::keypath::parse_key_paths; use jsonb::parse_value; +use jsonb::parse_value_standard_mode; use jsonb::OwnedJsonb; use jsonb::RawJsonb; use jsonb::Value as JsonbValue; @@ -113,7 +114,12 @@ pub fn register(registry: &mut FunctionRegistry) { } // Variant value may be an invalid JSON, convert them to string and then parse. let val = RawJsonb::new(s).to_string(); - match parse_value(val.as_bytes()) { + let res = if ctx.func_ctx.enable_extended_json_syntax { + parse_value(val.as_bytes()) + } else { + parse_value_standard_mode(val.as_bytes()) + }; + match res { Ok(value) => { value.write_to_vec(&mut output.data); } @@ -139,7 +145,12 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - match parse_value(s.as_bytes()) { + let res = if ctx.func_ctx.enable_extended_json_syntax { + parse_value(s.as_bytes()) + } else { + parse_value_standard_mode(s.as_bytes()) + }; + match res { Ok(value) => { value.write_to_vec(&mut output.data); } @@ -167,7 +178,12 @@ pub fn register(registry: &mut FunctionRegistry) { } // Variant value may be an invalid JSON, convert them to string and then parse. let val = RawJsonb::new(s).to_string(); - match parse_value(val.as_bytes()) { + let res = if ctx.func_ctx.enable_extended_json_syntax { + parse_value(val.as_bytes()) + } else { + parse_value_standard_mode(val.as_bytes()) + }; + match res { Ok(value) => { output.validity.push(true); value.write_to_vec(&mut output.builder.data); @@ -188,7 +204,12 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - match parse_value(s.as_bytes()) { + let res = if ctx.func_ctx.enable_extended_json_syntax { + parse_value(s.as_bytes()) + } else { + parse_value_standard_mode(s.as_bytes()) + }; + match res { Ok(value) => { output.validity.push(true); value.write_to_vec(&mut output.builder.data); @@ -211,7 +232,12 @@ pub fn register(registry: &mut FunctionRegistry) { } // Variant value may be an invalid JSON, convert them to string and then check. let val = RawJsonb::new(s).to_string(); - match parse_value(val.as_bytes()) { + let res = if ctx.func_ctx.enable_extended_json_syntax { + parse_value(val.as_bytes()) + } else { + parse_value_standard_mode(val.as_bytes()) + }; + match res { Ok(_) => output.push_null(), Err(e) => output.push(&e.to_string()), } @@ -228,7 +254,12 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - match parse_value(s.as_bytes()) { + let res = if ctx.func_ctx.enable_extended_json_syntax { + parse_value(s.as_bytes()) + } else { + parse_value_standard_mode(s.as_bytes()) + }; + match res { Ok(_) => output.push_null(), Err(e) => output.push(&e.to_string()), } @@ -237,30 +268,28 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_1_arg_core::, NullableType, _, _>( "length", - |_, _| FunctionDomain::Full, - vectorize_1_arg::, NullableType>(|val, _| { + |_, _| FunctionDomain::MayThrow, + vectorize_1_arg::, NullableType>(|val, ctx| { val.and_then(|v| match RawJsonb::new(v).array_length() { Ok(len) => len.map(|len| len as u32), - Err(_) => parse_value(v) - .ok() - .and_then(|v| v.array_length().map(|len| len as u32)), + Err(err) => { + ctx.set_error(0, err.to_string()); + None + } }) }), ); registry.register_1_arg_core::, NullableType, _, _>( "object_keys", - |_, _| FunctionDomain::Full, - vectorize_1_arg::, NullableType>(|val, _| { + |_, _| FunctionDomain::MayThrow, + vectorize_1_arg::, NullableType>(|val, ctx| { val.and_then(|v| match RawJsonb::new(v).object_keys() { Ok(obj_keys) => obj_keys.map(|v| v.to_vec()), - Err(_) => parse_value(v).ok().and_then(|v| { - v.object_keys().map(|obj_keys| { - let mut buf = Vec::new(); - obj_keys.write_to_vec(&mut buf); - buf - }) - }), + Err(err) => { + ctx.set_error(0, err.to_string()); + None + } }) }), ); @@ -315,7 +344,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_2_arg::( "get", - |_, _, _| FunctionDomain::Full, + |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( |val, name, output, ctx| { if let Some(validity) = &ctx.validity { @@ -331,7 +360,8 @@ pub fn register(registry: &mut FunctionRegistry) { Ok(None) => { output.push_null(); } - Err(_) => { + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -341,7 +371,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_2_arg::( "get", - |_, _, _| FunctionDomain::Full, + |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( |val, idx, output, ctx| { if let Some(validity) = &ctx.validity { @@ -360,7 +390,8 @@ pub fn register(registry: &mut FunctionRegistry) { Ok(None) => { output.push_null(); } - Err(_) => { + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -371,7 +402,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_2_arg::( "get_ignore_case", - |_, _, _| FunctionDomain::Full, + |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( |val, name, output, ctx| { if let Some(validity) = &ctx.validity { @@ -383,7 +414,8 @@ pub fn register(registry: &mut FunctionRegistry) { match RawJsonb::new(val).get_by_name(name, true) { Ok(Some(v)) => output.push(v.as_ref()), Ok(None) => output.push_null(), - Err(_) => { + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -393,7 +425,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_2_arg::( "get_string", - |_, _, _| FunctionDomain::Full, + |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( |val, name, output, ctx| { if let Some(validity) = &ctx.validity { @@ -414,7 +446,11 @@ pub fn register(registry: &mut FunctionRegistry) { output.push(&json_str); } } - _ => { + Ok(None) => { + output.push_null(); + } + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -424,7 +460,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_2_arg::( "get_string", - |_, _, _| FunctionDomain::Full, + |_, _, _| FunctionDomain::MayThrow, vectorize_with_builder_2_arg::>( |val, idx, output, ctx| { if let Some(validity) = &ctx.validity { @@ -448,7 +484,11 @@ pub fn register(registry: &mut FunctionRegistry) { output.push(&json_str); } } - _ => { + Ok(None) => { + output.push_null(); + } + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -674,7 +714,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_1_arg::( "as_boolean", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -685,17 +725,17 @@ pub fn register(registry: &mut FunctionRegistry) { match RawJsonb::new(v).as_bool() { Ok(Some(res)) => output.push(res), Ok(None) => output.push_null(), - Err(_) => match parse_value(v).ok().and_then(|v| v.as_bool()) { - Some(res) => output.push(res), - None => output.push_null(), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push_null(); + } } }), ); registry.register_combine_nullable_1_arg::( "as_integer", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -706,17 +746,17 @@ pub fn register(registry: &mut FunctionRegistry) { match RawJsonb::new(v).as_i64() { Ok(Some(res)) => output.push(res), Ok(None) => output.push_null(), - Err(_) => match parse_value(v).ok().and_then(|v| v.as_i64()) { - Some(res) => output.push(res), - None => output.push_null(), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push_null(); + } } }), ); registry.register_combine_nullable_1_arg::( "as_float", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -727,17 +767,17 @@ pub fn register(registry: &mut FunctionRegistry) { match RawJsonb::new(v).as_f64() { Ok(Some(res)) => output.push(res.into()), Ok(None) => output.push_null(), - Err(_) => match parse_value(v).ok().and_then(|v| v.as_f64()) { - Some(res) => output.push(res.into()), - None => output.push_null(), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push_null(); + } } }), ); registry.register_combine_nullable_1_arg::( "as_string", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -748,13 +788,8 @@ pub fn register(registry: &mut FunctionRegistry) { match RawJsonb::new(v).as_str() { Ok(Some(res)) => output.push(&res), Ok(None) => output.push_null(), - Err(_) => { - if let Ok(val) = parse_value(v) { - if let Some(res) = val.as_str() { - output.push(res); - return; - } - } + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -763,7 +798,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "is_binary", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -773,14 +808,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_binary() { Ok(res) => output.push(res), - Err(_) => output.push(false), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_combine_nullable_1_arg::( "as_binary", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -790,14 +828,18 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).as_binary() { Ok(Some(res)) => output.push(&res), - _ => output.push_null(), + Ok(None) => output.push_null(), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push_null(); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_date", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -807,14 +849,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_date() { Ok(res) => output.push(res), - Err(_) => output.push(false), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_combine_nullable_1_arg::( "as_date", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -824,14 +869,18 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).as_date() { Ok(Some(res)) => output.push(res.value), - _ => output.push_null(), + Ok(None) => output.push_null(), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push_null(); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_timestamp", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -841,14 +890,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_timestamp() { Ok(res) => output.push(res), - Err(_) => output.push(false), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_combine_nullable_1_arg::( "as_timestamp", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>( |v, output, ctx| { if let Some(validity) = &ctx.validity { @@ -859,7 +911,11 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).as_timestamp() { Ok(Some(res)) => output.push(res.value), - _ => output.push_null(), + Ok(None) => output.push_null(), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push_null(); + } } }, ), @@ -867,7 +923,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "is_interval", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -877,14 +933,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_interval() { Ok(res) => output.push(res), - Err(_) => output.push(false), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_combine_nullable_1_arg::( "as_interval", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>( |v, output, ctx| { if let Some(validity) = &ctx.validity { @@ -897,7 +956,11 @@ pub fn register(registry: &mut FunctionRegistry) { Ok(Some(res)) => { output.push(months_days_micros::new(res.months, res.days, res.micros)) } - _ => output.push_null(), + Ok(None) => output.push_null(), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push_null(); + } } }, ), @@ -905,7 +968,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_1_arg::( "as_array", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -916,15 +979,8 @@ pub fn register(registry: &mut FunctionRegistry) { match RawJsonb::new(v).is_array() { Ok(true) => output.push(v.as_bytes()), Ok(false) => output.push_null(), - Err(_) => { - if let Ok(val) = parse_value(v) { - if val.is_array() { - let mut buf = Vec::new(); - val.write_to_vec(&mut buf); - output.push(&buf); - return; - } - } + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -933,7 +989,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_combine_nullable_1_arg::( "as_object", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::>(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -944,15 +1000,8 @@ pub fn register(registry: &mut FunctionRegistry) { match RawJsonb::new(v).is_object() { Ok(true) => output.push(v.as_bytes()), Ok(false) => output.push_null(), - Err(_) => { - if let Ok(val) = parse_value(v) { - if val.is_object() { - let mut buf = Vec::new(); - val.write_to_vec(&mut buf); - output.push(&buf); - return; - } - } + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push_null(); } } @@ -961,7 +1010,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "is_null_value", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -971,17 +1020,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_null() { Ok(res) => output.push(res), - Err(_) => match parse_value(v).ok().map(|v| v.is_null()) { - Some(res) => output.push(res), - None => output.push(false), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_boolean", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -991,17 +1040,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_boolean() { Ok(res) => output.push(res), - Err(_) => match parse_value(v).ok().map(|v| v.is_boolean()) { - Some(res) => output.push(res), - None => output.push(false), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_integer", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -1011,17 +1060,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_i64() { Ok(res) => output.push(res), - Err(_) => match parse_value(v).ok().map(|v| v.is_i64()) { - Some(res) => output.push(res), - None => output.push(false), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_float", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -1031,17 +1080,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_f64() { Ok(res) => output.push(res), - Err(_) => match parse_value(v).ok().map(|v| v.is_f64()) { - Some(res) => output.push(res), - None => output.push(false), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_decimal", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -1054,7 +1103,9 @@ pub fn register(registry: &mut FunctionRegistry) { jsonb::Number::Float64(_) => output.push(false), _ => output.push(true), }, - _ => { + Ok(None) => output.push(false), + Err(err) => { + ctx.set_error(output.len(), err.to_string()); output.push(false); } } @@ -1063,7 +1114,7 @@ pub fn register(registry: &mut FunctionRegistry) { registry.register_passthrough_nullable_1_arg::( "is_string", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -1073,17 +1124,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_string() { Ok(res) => output.push(res), - Err(_) => match parse_value(v).ok().map(|v| v.is_string()) { - Some(res) => output.push(res), - None => output.push(false), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_array", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -1093,17 +1144,17 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_array() { Ok(res) => output.push(res), - Err(_) => match parse_value(v).ok().map(|v| v.is_array()) { - Some(res) => output.push(res), - None => output.push(false), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); registry.register_passthrough_nullable_1_arg::( "is_object", - |_, _| FunctionDomain::Full, + |_, _| FunctionDomain::MayThrow, vectorize_with_builder_1_arg::(|v, output, ctx| { if let Some(validity) = &ctx.validity { if !validity.get_bit(output.len()) { @@ -1113,10 +1164,10 @@ pub fn register(registry: &mut FunctionRegistry) { } match RawJsonb::new(v).is_object() { Ok(res) => output.push(res), - Err(_) => match parse_value(v).ok().map(|v| v.is_object()) { - Some(res) => output.push(res), - None => output.push(false), - }, + Err(err) => { + ctx.set_error(output.len(), err.to_string()); + output.push(false); + } } }), ); @@ -1155,6 +1206,7 @@ pub fn register(registry: &mut FunctionRegistry) { cast_scalar_to_variant( scalar.as_ref(), &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, &mut buf, None, ); @@ -1169,7 +1221,12 @@ pub fn register(registry: &mut FunctionRegistry) { } _ => None, }; - let new_col = cast_scalars_to_variants(col.iter(), &ctx.func_ctx.tz, None); + let new_col = cast_scalars_to_variants( + col.iter(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + None, + ); if let Some(validity) = validity { Value::Column(NullableColumn::new_column( Column::Variant(new_col), @@ -1202,7 +1259,13 @@ pub fn register(registry: &mut FunctionRegistry) { Scalar::Null => Value::Scalar(None), _ => { let mut buf = Vec::new(); - cast_scalar_to_variant(scalar.as_ref(), &ctx.func_ctx.tz, &mut buf, None); + cast_scalar_to_variant( + scalar.as_ref(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + &mut buf, + None, + ); Value::Scalar(Some(buf)) } }, @@ -1212,7 +1275,12 @@ pub fn register(registry: &mut FunctionRegistry) { Column::Nullable(box ref nullable_column) => nullable_column.validity.clone(), _ => Bitmap::new_constant(true, col.len()), }; - let new_col = cast_scalars_to_variants(col.iter(), &ctx.func_ctx.tz, None); + let new_col = cast_scalars_to_variants( + col.iter(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + None, + ); Value::Column(NullableColumn::new_unchecked(new_col, validity)) } }, @@ -1232,7 +1300,7 @@ pub fn register(registry: &mut FunctionRegistry) { if RawJsonb::new(val).is_null().unwrap_or_default() { output.push_null(); } else { - match cast_to_bool(val) { + match RawJsonb::new(val).to_bool() { Ok(value) => output.push(value), Err(err) => { ctx.set_error(output.len(), err.to_string()); @@ -1254,7 +1322,7 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - match cast_to_bool(v) { + match RawJsonb::new(v).to_bool() { Ok(res) => output.push(res), Err(_) => output.push_null(), } @@ -1413,11 +1481,20 @@ pub fn register(registry: &mut FunctionRegistry) { } type Native = ::Native; let value: Option = if dest_type.is_float() { - cast_to_f64(val).ok().and_then(num_traits::cast::cast) + RawJsonb::new(val) + .to_f64() + .ok() + .and_then(num_traits::cast::cast) } else if dest_type.is_signed() { - cast_to_i64(val).ok().and_then(num_traits::cast::cast) + RawJsonb::new(val) + .to_i64() + .ok() + .and_then(num_traits::cast::cast) } else { - cast_to_u64(val).ok().and_then(num_traits::cast::cast) + RawJsonb::new(val) + .to_u64() + .ok() + .and_then(num_traits::cast::cast) }; match value { Some(value) => output.push(value.into()), @@ -1448,7 +1525,7 @@ pub fn register(registry: &mut FunctionRegistry) { } } if dest_type.is_float() { - if let Ok(value) = cast_to_f64(v) { + if let Ok(value) = RawJsonb::new(v).to_f64() { if let Some(new_value) = num_traits::cast::cast(value) { output.push(new_value); } else { @@ -1458,7 +1535,7 @@ pub fn register(registry: &mut FunctionRegistry) { output.push_null(); } } else if dest_type.is_signed() { - if let Ok(value) = cast_to_i64(v) { + if let Ok(value) = RawJsonb::new(v).to_i64() { if let Some(new_value) = num_traits::cast::cast(value) { output.push(new_value); } else { @@ -1468,7 +1545,7 @@ pub fn register(registry: &mut FunctionRegistry) { output.push_null(); } } else { - if let Ok(value) = cast_to_u64(v) { + if let Ok(value) = RawJsonb::new(v).to_u64() { if let Some(new_value) = num_traits::cast::cast(value) { output.push(new_value); } else { @@ -1826,7 +1903,13 @@ pub fn register(registry: &mut FunctionRegistry) { } let array_val = RawJsonb::new(val); let mut item_buf = vec![]; - cast_scalar_to_variant(item.clone(), &ctx.func_ctx.tz, &mut item_buf, None); + cast_scalar_to_variant( + item.clone(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + &mut item_buf, + None, + ); let item_val = OwnedJsonb::new(item_buf); match array_val.array_values() { Ok(vals_opt) => { @@ -1984,7 +2067,13 @@ pub fn register(registry: &mut FunctionRegistry) { } let array_val = RawJsonb::new(val); let mut item_buf = vec![]; - cast_scalar_to_variant(item.clone(), &ctx.func_ctx.tz, &mut item_buf, None); + cast_scalar_to_variant( + item.clone(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + &mut item_buf, + None, + ); let item_val = OwnedJsonb::new(item_buf); match array_val.array_values() { Ok(Some(vals)) => { @@ -2018,7 +2107,13 @@ pub fn register(registry: &mut FunctionRegistry) { } let array_val = RawJsonb::new(val); let mut item_buf = vec![]; - cast_scalar_to_variant(item.clone(), &ctx.func_ctx.tz, &mut item_buf, None); + cast_scalar_to_variant( + item.clone(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + &mut item_buf, + None, + ); let item_val = OwnedJsonb::new(item_buf); match array_val.array_values() { Ok(vals_opt) => { @@ -2116,7 +2211,13 @@ pub fn register(registry: &mut FunctionRegistry) { vectorize_with_builder_2_arg::, NullableType, VariantType>( |item, arr, output, ctx| { let mut item_buf = vec![]; - cast_scalar_to_variant(item.clone(), &ctx.func_ctx.tz, &mut item_buf, None); + cast_scalar_to_variant( + item.clone(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + &mut item_buf, + None, + ); let item_val = OwnedJsonb::new(item_buf); let new_vals = if let Some(arr) = arr { let array_val = RawJsonb::new(arr); @@ -2157,7 +2258,13 @@ pub fn register(registry: &mut FunctionRegistry) { vectorize_with_builder_2_arg::, GenericType<0>, VariantType>( |arr, item, output, ctx| { let mut item_buf = vec![]; - cast_scalar_to_variant(item.clone(), &ctx.func_ctx.tz, &mut item_buf, None); + cast_scalar_to_variant( + item.clone(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + &mut item_buf, + None, + ); let item_val = OwnedJsonb::new(item_buf); let new_vals = if let Some(arr) = arr { let array_val = RawJsonb::new(arr); @@ -2386,12 +2493,12 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - match type_of(v) { + match RawJsonb::new(v).type_of() { Ok(result) => output.put_str(result), Err(err) => { ctx.set_error(output.len(), err.to_string()); } - }; + } output.commit_row(); }), ); @@ -2406,12 +2513,12 @@ pub fn register(registry: &mut FunctionRegistry) { return; } } - match type_of(v) { + match RawJsonb::new(v).type_of() { Ok(result) => output.put_str(result), Err(err) => { ctx.set_error(output.len(), err.to_string()); } - }; + } output.commit_row(); }), ); @@ -2736,7 +2843,13 @@ fn array_construct_fn(args: &[Value], ctx: &mut EvalContext) -> Value { // if the new value is not a json value, cast it to json. let mut new_val_buf = vec![]; - cast_scalar_to_variant(new_val.clone(), &ctx.func_ctx.tz, &mut new_val_buf, None); + cast_scalar_to_variant( + new_val.clone(), + &ctx.func_ctx.tz, + ctx.func_ctx.enable_extended_json_syntax, + &mut new_val_buf, + None, + ); let new_val = RawJsonb::new(new_val_buf.as_bytes()); value.object_insert(&new_key_str, &new_val, update_flag) } @@ -3330,66 +3455,6 @@ fn object_pick_or_delete_fn( } } -fn cast_to_bool(v: &[u8]) -> Result { - match RawJsonb::new(v).to_bool() { - Ok(val) => Ok(val), - Err(err) => { - if err.to_string() == "InvalidJsonb" { - let s = unsafe { std::str::from_utf8_unchecked(v) }; - let owned_jsonb = s.parse::()?; - let raw_jsonb = owned_jsonb.as_raw(); - return raw_jsonb.to_bool(); - } - Err(err) - } - } -} - -fn cast_to_i64(v: &[u8]) -> Result { - match RawJsonb::new(v).to_i64() { - Ok(val) => Ok(val), - Err(err) => { - if err.to_string() == "InvalidJsonb" { - let s = unsafe { std::str::from_utf8_unchecked(v) }; - let owned_jsonb = s.parse::()?; - let raw_jsonb = owned_jsonb.as_raw(); - return raw_jsonb.to_i64(); - } - Err(err) - } - } -} - -fn cast_to_u64(v: &[u8]) -> Result { - match RawJsonb::new(v).to_u64() { - Ok(val) => Ok(val), - Err(err) => { - if err.to_string() == "InvalidJsonb" { - let s = unsafe { std::str::from_utf8_unchecked(v) }; - let owned_jsonb = s.parse::()?; - let raw_jsonb = owned_jsonb.as_raw(); - return raw_jsonb.to_u64(); - } - Err(err) - } - } -} - -fn cast_to_f64(v: &[u8]) -> Result { - match RawJsonb::new(v).to_f64() { - Ok(val) => Ok(val), - Err(err) => { - if err.to_string() == "InvalidJsonb" { - let s = unsafe { std::str::from_utf8_unchecked(v) }; - let owned_jsonb = s.parse::()?; - let raw_jsonb = owned_jsonb.as_raw(); - return raw_jsonb.to_f64(); - } - Err(err) - } - } -} - fn cast_to_date(val: &[u8], tz: &TimeZone) -> Result, jsonb::Error> { let value = jsonb::from_slice(val)?; match value { @@ -3460,18 +3525,3 @@ fn cast_to_binary(val: &[u8]) -> Result>, jsonb::Error> { _ => Err(jsonb::Error::InvalidJsonType), } } - -pub(crate) fn type_of(v: &[u8]) -> Result<&'static str, jsonb::Error> { - match RawJsonb::new(v).type_of() { - Ok(val) => Ok(val), - Err(err) => { - if err.to_string() == "InvalidJsonb" { - let s = unsafe { std::str::from_utf8_unchecked(v) }; - let owned_jsonb = s.parse::()?; - let raw_jsonb = owned_jsonb.as_raw(); - return raw_jsonb.type_of(); - } - Err(err) - } - } -} diff --git a/src/query/functions/tests/it/aggregates/testdata/agg.txt b/src/query/functions/tests/it/aggregates/testdata/agg.txt index 28293801cf2f6..4aee528a08d3f 100644 --- a/src/query/functions/tests/it/aggregates/testdata/agg.txt +++ b/src/query/functions/tests/it/aggregates/testdata/agg.txt @@ -1467,12 +1467,12 @@ evaluation (internal): ast: json_array_agg(dt) evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| dt | Timestamp([1, 0, 2, 3]) | -| Output | Variant([0x8000000460000009600000096000000960000009200000000000000001200000000000000000200000000000000002200000000000000003]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| dt | Timestamp([1, 0, 2, 3]) | +| Output | Variant([0x800000041000001a1000001a1000001a1000001a313937302d30312d30312030303a30303a30302e303030303031313937302d30312d30312030303a30303a30302e303030303030313937302d30312d30312030303a30303a30302e303030303032313937302d30312d30312030303a30303a30302e303030303033]) | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast: json_array_agg(event1) @@ -1487,12 +1487,12 @@ evaluation (internal): ast: json_array_agg(dec) evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | -| Output | Variant([0x800000032000000a2000000a2000000a70000000000000006e027000000000000000dc0270000000000000014a02]) | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | +| Output | Variant([0x80000003200000092000000920000009603ff199999999999a60400199999999999a60400a666666666666]) | ++--------+-----------------------------------------------------------------------------------------------------+ error: Json object have duplicate key 'k' @@ -1523,13 +1523,13 @@ error: json_object_agg does not support key type 'Number(Int64)' ast: json_object_agg(s, dec) evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn[abc, def, opq, xyz] | -| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | -| Output | Variant([0x400000031000000310000003100000032000000a2000000a2000000a61626364656678797a70000000000000006e027000000000000000dc0270000000000000014a02]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[abc, def, opq, xyz] | +| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | +| Output | Variant([0x4000000310000003100000031000000320000009200000092000000961626364656678797a603ff199999999999a60400199999999999a60400a666666666666]) | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ ast: mode(1) diff --git a/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt b/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt index 02a4edd1dc367..6eb540bfa978e 100644 --- a/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt +++ b/src/query/functions/tests/it/aggregates/testdata/agg_group_by.txt @@ -1405,12 +1405,12 @@ evaluation (internal): ast: json_array_agg(dt) evaluation (internal): -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ -| dt | Timestamp([1, 0, 2, 3]) | -| Output | Variant([0x8000000460000009600000096000000960000009200000000000000001200000000000000000200000000000000002200000000000000003]) | -+--------+-------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| dt | Timestamp([1, 0, 2, 3]) | +| Output | Variant([0x800000041000001a1000001a1000001a1000001a313937302d30312d30312030303a30303a30302e303030303031313937302d30312d30312030303a30303a30302e303030303030313937302d30312d30312030303a30303a30302e303030303032313937302d30312d30312030303a30303a30302e303030303033]) | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast: json_array_agg(event1) @@ -1425,12 +1425,12 @@ evaluation (internal): ast: json_array_agg(dec) evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------+ -| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | -| Output | Variant([0x800000032000000a2000000a2000000a70000000000000006e027000000000000000dc0270000000000000014a02]) | -+--------+-----------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------+ +| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | +| Output | Variant([0x80000003200000092000000920000009603ff199999999999a60400199999999999a60400a666666666666]) | ++--------+-----------------------------------------------------------------------------------------------------+ error: Json object have duplicate key 'k' @@ -1461,13 +1461,13 @@ error: json_object_agg does not support key type 'Number(Int64)' ast: json_object_agg(s, dec) evaluation (internal): -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn[abc, def, opq, xyz] | -| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | -| Output | Variant([0x400000031000000310000003100000032000000a2000000a2000000a61626364656678797a70000000000000006e027000000000000000dc0270000000000000014a02]) | -+--------+-----------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[abc, def, opq, xyz] | +| dec | NullableColumn { column: Decimal64([1.10, 2.20, 0.00, 3.30]), validity: [0b____1011] } | +| Output | Variant([0x4000000310000003100000031000000320000009200000092000000961626364656678797a603ff199999999999a60400199999999999a60400a666666666666]) | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------+ ast: mode(1) diff --git a/src/query/functions/tests/it/scalars/cast.rs b/src/query/functions/tests/it/scalars/cast.rs index eedd9e86a3f6f..bc1d69a971898 100644 --- a/src/query/functions/tests/it/scalars/cast.rs +++ b/src/query/functions/tests/it/scalars/cast.rs @@ -33,7 +33,8 @@ fn test_cast() { for is_try in [false, true] { test_cast_primitive(file, is_try); - test_cast_to_variant(file, is_try); + test_cast_to_variant(file, is_try, false); + test_cast_to_variant(file, is_try, true); test_cast_number_to_timestamp(file, is_try); test_cast_number_to_date(file, is_try); test_cast_between_number_and_string(file, is_try); @@ -138,57 +139,130 @@ fn test_cast_primitive(file: &mut impl Write, is_try: bool) { )]); } -fn test_cast_to_variant(file: &mut impl Write, is_try: bool) { +fn test_cast_to_variant(file: &mut impl Write, is_try: bool, enable_extended_json_syntax: bool) { let prefix = if is_try { "TRY_" } else { "" }; + let func_ctx = FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }; + let test_ctx = TestContext { + func_ctx: func_ctx.clone(), + ..TestContext::default() + }; - run_ast(file, format!("{prefix}CAST(NULL AS VARIANT)"), &[]); - run_ast(file, format!("{prefix}CAST(0 AS VARIANT)"), &[]); - run_ast(file, format!("{prefix}CAST(-1 AS VARIANT)"), &[]); - run_ast(file, format!("{prefix}CAST(1.1 AS VARIANT)"), &[]); - run_ast( + run_ast_with_context( + file, + format!("{prefix}CAST(NULL AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}CAST(0 AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}CAST(-1 AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}CAST(1.1 AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( file, format!("{prefix}CAST('🍦 が美味しい' AS VARIANT)"), - &[], + test_ctx.clone(), ); - run_ast(file, format!("{prefix}CAST([0, 1, 2] AS VARIANT)"), &[]); - run_ast( + run_ast_with_context( + file, + format!("{prefix}CAST([0, 1, 2] AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( file, format!("{prefix}CAST([0::VARIANT, '\"a\"'::VARIANT] AS VARIANT)"), - &[], + test_ctx.clone(), ); - run_ast( + run_ast_with_context( file, format!("{prefix}CAST(to_timestamp(1000000) AS VARIANT)"), - &[], + test_ctx.clone(), ); - run_ast(file, format!("{prefix}CAST(false AS VARIANT)"), &[]); - run_ast(file, format!("{prefix}CAST(true AS VARIANT)"), &[]); - run_ast( + run_ast_with_context( + file, + format!("{prefix}CAST(false AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}CAST(true AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( file, format!("{prefix}CAST({prefix}CAST('\"🍦 が美味しい\"' AS VARIANT) AS VARIANT)"), - &[], + test_ctx.clone(), ); - run_ast(file, format!("{prefix}CAST((1,) AS VARIANT)"), &[]); - run_ast(file, format!("{prefix}CAST((1, 2) AS VARIANT)"), &[]); - run_ast(file, format!("{prefix}CAST((false, true) AS VARIANT)"), &[]); - run_ast(file, format!("{prefix}CAST(('a',) AS VARIANT)"), &[]); - run_ast( + run_ast_with_context( + file, + format!("{prefix}CAST((1,) AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}CAST((1, 2) AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}CAST((false, true) AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}CAST(('a',) AS VARIANT)"), + test_ctx.clone(), + ); + run_ast_with_context( file, format!("{prefix}CAST((1, 2, (false, true, ('a',))) AS VARIANT)"), - &[], + test_ctx.clone(), ); - run_ast(file, format!("{prefix}CAST(a AS VARIANT)"), &[( - "a", - StringType::from_data_with_validity(vec!["true", "{\"k\":\"v\"}", "[1,2,3]"], vec![ - true, false, true, - ]), - )]); + let test_ctx_with_column = TestContext { + columns: &[( + "a", + StringType::from_data_with_validity(vec!["true", "{\"k\":\"v\"}", "[1,2,3]"], vec![ + true, false, true, + ]), + )], + func_ctx: FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }, + ..TestContext::default() + }; + run_ast_with_context( + file, + format!("{prefix}CAST(a AS VARIANT)"), + test_ctx_with_column, + ); - run_ast(file, format!("{prefix}CAST(a AS VARIANT)"), &[( - "a", - gen_bitmap_data(), - )]); + let test_ctx_with_column = TestContext { + columns: &[("a", gen_bitmap_data())], + func_ctx: FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }, + ..TestContext::default() + }; + run_ast_with_context( + file, + format!("{prefix}CAST(a AS VARIANT)"), + test_ctx_with_column, + ); } fn test_cast_number_to_timestamp(file: &mut impl Write, is_try: bool) { diff --git a/src/query/functions/tests/it/scalars/testdata/cast.txt b/src/query/functions/tests/it/scalars/testdata/cast.txt index 7f2861d9290b8..2f9c17d661b0d 100644 --- a/src/query/functions/tests/it/scalars/testdata/cast.txt +++ b/src/query/functions/tests/it/scalars/testdata/cast.txt @@ -228,10 +228,186 @@ output domain : Undefined output : '-1' +ast : CAST(1.1 AS VARIANT) +raw expr : CAST(1.1 AS Variant) +checked expr : CAST(1.1_d64(2,1) AS Variant) +optimized expr : 0x2000000020000009603ff199999999999a +output type : Variant +output domain : Undefined +output : '1.1' + + +error: + --> SQL:1:1 + | +1 | CAST('🍦 が美味しい' AS VARIANT) + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ expected value, pos 1 while evaluating function `parse_json('🍦 が美味しい')` in expr `CAST('🍦 が美味しい' AS Variant)` + + + +ast : CAST([0, 1, 2] AS VARIANT) +raw expr : CAST(array(0, 1, 2) AS Variant) +checked expr : CAST(array(0_u8, 1_u8, 2_u8) AS Variant) +optimized expr : 0x800000032000000120000002200000020050015002 +output type : Variant +output domain : Undefined +output : '[0,1,2]' + + +ast : CAST([0::VARIANT, '"a"'::VARIANT] AS VARIANT) +raw expr : CAST(array(CAST(0 AS Variant), CAST('"a"' AS Variant)) AS Variant) +checked expr : CAST(array(CAST(0_u8 AS Variant), CAST("\"a\"" AS Variant)) AS Variant) +optimized expr : 0x8000000220000001100000010061 +output type : Variant +output domain : Undefined +output : '[0,"a"]' + + +ast : CAST(to_timestamp(1000000) AS VARIANT) +raw expr : CAST(to_timestamp(1000000) AS Variant) +checked expr : CAST(CAST(CAST(1000000_u32 AS Int64) AS Timestamp) AS Variant) +optimized expr : 0x200000001000001a313937302d30312d31322031333a34363a34302e303030303030 +output type : Variant +output domain : Undefined +output : '"1970-01-12 13:46:40.000000"' + + +ast : CAST(false AS VARIANT) +raw expr : CAST(false AS Variant) +checked expr : CAST(false AS Variant) +optimized expr : 0x2000000030000000 +output type : Variant +output domain : Undefined +output : 'false' + + +ast : CAST(true AS VARIANT) +raw expr : CAST(true AS Variant) +checked expr : CAST(true AS Variant) +optimized expr : 0x2000000040000000 +output type : Variant +output domain : Undefined +output : 'true' + + +ast : CAST(CAST('"🍦 が美味しい"' AS VARIANT) AS VARIANT) +raw expr : CAST(CAST('"🍦 が美味しい"' AS Variant) AS Variant) +checked expr : CAST("\"🍦 が美味しい\"" AS Variant) +optimized expr : 0x2000000010000014f09f8da620e3818ce7be8ee591b3e38197e38184 +output type : Variant +output domain : Undefined +output : '"🍦 が美味しい"' + + +ast : CAST((1,) AS VARIANT) +raw expr : CAST(tuple(1) AS Variant) +checked expr : CAST(tuple(1_u8) AS Variant) +optimized expr : 0x400000011000000120000002315001 +output type : Variant +output domain : Undefined +output : '{"1":1}' + + +ast : CAST((1, 2) AS VARIANT) +raw expr : CAST(tuple(1, 2) AS Variant) +checked expr : CAST(tuple(1_u8, 2_u8) AS Variant) +optimized expr : 0x4000000210000001100000012000000220000002313250015002 +output type : Variant +output domain : Undefined +output : '{"1":1,"2":2}' + + +ast : CAST((false, true) AS VARIANT) +raw expr : CAST(tuple(false, true) AS Variant) +checked expr : CAST(tuple(false, true) AS Variant) +optimized expr : 0x40000002100000011000000130000000400000003132 +output type : Variant +output domain : Undefined +output : '{"1":false,"2":true}' + + +ast : CAST(('a',) AS VARIANT) +raw expr : CAST(tuple('a') AS Variant) +checked expr : CAST(tuple("a") AS Variant) +optimized expr : 0x4000000110000001100000013161 +output type : Variant +output domain : Undefined +output : '{"1":"a"}' + + +ast : CAST((1, 2, (false, true, ('a',))) AS VARIANT) +raw expr : CAST(tuple(1, 2, tuple(false, true, tuple('a'))) AS Variant) +checked expr : CAST(tuple(1_u8, 2_u8, tuple(false, true, tuple("a"))) AS Variant) +optimized expr : 0x4000000310000001100000011000000120000002200000025000002d313233500150024000000310000001100000011000000130000000400000005000000e3132334000000110000001100000013161 +output type : Variant +output domain : Undefined +output : '{"1":1,"2":2,"3":{"1":false,"2":true,"3":{"1":"a"}}}' + + +error: + --> SQL:1:1 + | +1 | CAST(a AS VARIANT) + | ^^^^^^^^^^^^^^^^^^ unable to cast `NULL` to type `Variant`, during run expr: `CAST(a AS Variant)` + + + +ast : CAST(a AS VARIANT) +raw expr : CAST(a::Bitmap AS Variant) +checked expr : CAST(a AS Variant) +evaluation: ++--------+-------------+---------------+ +| | a | Output | ++--------+-------------+---------------+ +| Type | Bitmap | Variant | +| Domain | Undefined | Undefined | +| Row 0 | '0,1,2,3,4' | '[0,1,2,3,4]' | +| Row 1 | '1,2,3,4,5' | '[1,2,3,4,5]' | +| Row 2 | '2,3,4,5,6' | '[2,3,4,5,6]' | +| Row 3 | '3,4,5,6,7' | '[3,4,5,6,7]' | ++--------+-------------+---------------+ +evaluation (internal): ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | Bitmap([0x0100000000000000000000003a30000001000000000004001000000000000100020003000400, 0x0100000000000000000000003a30000001000000000004001000000001000200030004000500, 0x0100000000000000000000003a30000001000000000004001000000002000300040005000600, 0x0100000000000000000000003a30000001000000000004001000000003000400050006000700]) | +| Output | Variant([0x800000052000000120000002200000022000000220000002005001500250035004, 0x80000005200000022000000220000002200000022000000250015002500350045005, 0x80000005200000022000000220000002200000022000000250025003500450055006, 0x80000005200000022000000220000002200000022000000250035004500550065007]) | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +error: + --> SQL:1:1 + | +1 | CAST(NULL AS VARIANT) + | ^^^^^^^^^^^^^^^^^^^^^ unable to cast type `NULL` to type `Variant`, during run expr: `CAST(NULL AS Variant)` + + + +ast : CAST(0 AS VARIANT) +raw expr : CAST(0 AS Variant) +checked expr : CAST(0_u8 AS Variant) +optimized expr : 0x200000002000000100 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : '0' + + +ast : CAST(-1 AS VARIANT) +raw expr : CAST(minus(1) AS Variant) +checked expr : CAST(minus(1_u8) AS Variant) +optimized expr : 0x200000002000000240ff +func ctx : (modified) +output type : Variant +output domain : Undefined +output : '-1' + + ast : CAST(1.1 AS VARIANT) raw expr : CAST(1.1 AS Variant) checked expr : CAST(1.1_d64(2,1) AS Variant) optimized expr : 0x200000002000000a70000000000000000b01 +func ctx : (modified) output type : Variant output domain : Undefined output : '1.1' @@ -249,6 +425,7 @@ ast : CAST([0, 1, 2] AS VARIANT) raw expr : CAST(array(0, 1, 2) AS Variant) checked expr : CAST(array(0_u8, 1_u8, 2_u8) AS Variant) optimized expr : 0x800000032000000120000002200000020050015002 +func ctx : (modified) output type : Variant output domain : Undefined output : '[0,1,2]' @@ -258,6 +435,7 @@ ast : CAST([0::VARIANT, '"a"'::VARIANT] AS VARIANT) raw expr : CAST(array(CAST(0 AS Variant), CAST('"a"' AS Variant)) AS Variant) checked expr : CAST(array(CAST(0_u8 AS Variant), CAST("\"a\"" AS Variant)) AS Variant) optimized expr : 0x8000000220000001100000010061 +func ctx : (modified) output type : Variant output domain : Undefined output : '[0,"a"]' @@ -267,6 +445,7 @@ ast : CAST(to_timestamp(1000000) AS VARIANT) raw expr : CAST(to_timestamp(1000000) AS Variant) checked expr : CAST(CAST(CAST(1000000_u32 AS Int64) AS Timestamp) AS Variant) optimized expr : 0x200000006000000920000000e8d4a51000 +func ctx : (modified) output type : Variant output domain : Undefined output : '"1970-01-12 13:46:40.000000"' @@ -276,6 +455,7 @@ ast : CAST(false AS VARIANT) raw expr : CAST(false AS Variant) checked expr : CAST(false AS Variant) optimized expr : 0x2000000030000000 +func ctx : (modified) output type : Variant output domain : Undefined output : 'false' @@ -285,6 +465,7 @@ ast : CAST(true AS VARIANT) raw expr : CAST(true AS Variant) checked expr : CAST(true AS Variant) optimized expr : 0x2000000040000000 +func ctx : (modified) output type : Variant output domain : Undefined output : 'true' @@ -294,6 +475,7 @@ ast : CAST(CAST('"🍦 が美味しい"' AS VARIANT) AS VARIANT) raw expr : CAST(CAST('"🍦 が美味しい"' AS Variant) AS Variant) checked expr : CAST("\"🍦 が美味しい\"" AS Variant) optimized expr : 0x2000000010000014f09f8da620e3818ce7be8ee591b3e38197e38184 +func ctx : (modified) output type : Variant output domain : Undefined output : '"🍦 が美味しい"' @@ -303,6 +485,7 @@ ast : CAST((1,) AS VARIANT) raw expr : CAST(tuple(1) AS Variant) checked expr : CAST(tuple(1_u8) AS Variant) optimized expr : 0x400000011000000120000002315001 +func ctx : (modified) output type : Variant output domain : Undefined output : '{"1":1}' @@ -312,6 +495,7 @@ ast : CAST((1, 2) AS VARIANT) raw expr : CAST(tuple(1, 2) AS Variant) checked expr : CAST(tuple(1_u8, 2_u8) AS Variant) optimized expr : 0x4000000210000001100000012000000220000002313250015002 +func ctx : (modified) output type : Variant output domain : Undefined output : '{"1":1,"2":2}' @@ -321,6 +505,7 @@ ast : CAST((false, true) AS VARIANT) raw expr : CAST(tuple(false, true) AS Variant) checked expr : CAST(tuple(false, true) AS Variant) optimized expr : 0x40000002100000011000000130000000400000003132 +func ctx : (modified) output type : Variant output domain : Undefined output : '{"1":false,"2":true}' @@ -330,6 +515,7 @@ ast : CAST(('a',) AS VARIANT) raw expr : CAST(tuple('a') AS Variant) checked expr : CAST(tuple("a") AS Variant) optimized expr : 0x4000000110000001100000013161 +func ctx : (modified) output type : Variant output domain : Undefined output : '{"1":"a"}' @@ -339,6 +525,7 @@ ast : CAST((1, 2, (false, true, ('a',))) AS VARIANT) raw expr : CAST(tuple(1, 2, tuple(false, true, tuple('a'))) AS Variant) checked expr : CAST(tuple(1_u8, 2_u8, tuple(false, true, tuple("a"))) AS Variant) optimized expr : 0x4000000310000001100000011000000120000002200000025000002d313233500150024000000310000001100000011000000130000000400000005000000e3132334000000110000001100000013161 +func ctx : (modified) output type : Variant output domain : Undefined output : '{"1":1,"2":2,"3":{"1":false,"2":true,"3":{"1":"a"}}}' @@ -355,6 +542,7 @@ error: ast : CAST(a AS VARIANT) raw expr : CAST(a::Bitmap AS Variant) checked expr : CAST(a AS Variant) +func ctx : (modified) evaluation: +--------+-------------+---------------+ | | a | Output | @@ -2406,10 +2594,203 @@ output domain : Undefined output : '-1' +ast : TRY_CAST(1.1 AS VARIANT) +raw expr : TRY_CAST(1.1 AS Variant) +checked expr : TRY_CAST(1.1_d64(2,1) AS Variant NULL) +optimized expr : 0x2000000020000009603ff199999999999a +output type : Variant NULL +output domain : Undefined +output : '1.1' + + +ast : TRY_CAST('🍦 が美味しい' AS VARIANT) +raw expr : TRY_CAST('🍦 が美味しい' AS Variant) +checked expr : TRY_CAST("🍦 が美味しい" AS Variant NULL) +optimized expr : NULL +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : TRY_CAST([0, 1, 2] AS VARIANT) +raw expr : TRY_CAST(array(0, 1, 2) AS Variant) +checked expr : TRY_CAST(array(0_u8, 1_u8, 2_u8) AS Variant NULL) +optimized expr : 0x800000032000000120000002200000020050015002 +output type : Variant NULL +output domain : Undefined +output : '[0,1,2]' + + +ast : TRY_CAST([0::VARIANT, '"a"'::VARIANT] AS VARIANT) +raw expr : TRY_CAST(array(CAST(0 AS Variant), CAST('"a"' AS Variant)) AS Variant) +checked expr : TRY_CAST(array(CAST(0_u8 AS Variant), CAST("\"a\"" AS Variant)) AS Variant NULL) +optimized expr : 0x8000000220000001100000010061 +output type : Variant NULL +output domain : Undefined +output : '[0,"a"]' + + +ast : TRY_CAST(to_timestamp(1000000) AS VARIANT) +raw expr : TRY_CAST(to_timestamp(1000000) AS Variant) +checked expr : TRY_CAST(CAST(CAST(1000000_u32 AS Int64) AS Timestamp) AS Variant NULL) +optimized expr : 0x200000001000001a313937302d30312d31322031333a34363a34302e303030303030 +output type : Variant NULL +output domain : Undefined +output : '"1970-01-12 13:46:40.000000"' + + +ast : TRY_CAST(false AS VARIANT) +raw expr : TRY_CAST(false AS Variant) +checked expr : TRY_CAST(false AS Variant NULL) +optimized expr : 0x2000000030000000 +output type : Variant NULL +output domain : Undefined +output : 'false' + + +ast : TRY_CAST(true AS VARIANT) +raw expr : TRY_CAST(true AS Variant) +checked expr : TRY_CAST(true AS Variant NULL) +optimized expr : 0x2000000040000000 +output type : Variant NULL +output domain : Undefined +output : 'true' + + +ast : TRY_CAST(TRY_CAST('"🍦 が美味しい"' AS VARIANT) AS VARIANT) +raw expr : TRY_CAST(TRY_CAST('"🍦 が美味しい"' AS Variant) AS Variant) +checked expr : TRY_CAST("\"🍦 が美味しい\"" AS Variant NULL) +optimized expr : 0x2000000010000014f09f8da620e3818ce7be8ee591b3e38197e38184 +output type : Variant NULL +output domain : Undefined +output : '"🍦 が美味しい"' + + +ast : TRY_CAST((1,) AS VARIANT) +raw expr : TRY_CAST(tuple(1) AS Variant) +checked expr : TRY_CAST(tuple(1_u8) AS Variant NULL) +optimized expr : 0x400000011000000120000002315001 +output type : Variant NULL +output domain : Undefined +output : '{"1":1}' + + +ast : TRY_CAST((1, 2) AS VARIANT) +raw expr : TRY_CAST(tuple(1, 2) AS Variant) +checked expr : TRY_CAST(tuple(1_u8, 2_u8) AS Variant NULL) +optimized expr : 0x4000000210000001100000012000000220000002313250015002 +output type : Variant NULL +output domain : Undefined +output : '{"1":1,"2":2}' + + +ast : TRY_CAST((false, true) AS VARIANT) +raw expr : TRY_CAST(tuple(false, true) AS Variant) +checked expr : TRY_CAST(tuple(false, true) AS Variant NULL) +optimized expr : 0x40000002100000011000000130000000400000003132 +output type : Variant NULL +output domain : Undefined +output : '{"1":false,"2":true}' + + +ast : TRY_CAST(('a',) AS VARIANT) +raw expr : TRY_CAST(tuple('a') AS Variant) +checked expr : TRY_CAST(tuple("a") AS Variant NULL) +optimized expr : 0x4000000110000001100000013161 +output type : Variant NULL +output domain : Undefined +output : '{"1":"a"}' + + +ast : TRY_CAST((1, 2, (false, true, ('a',))) AS VARIANT) +raw expr : TRY_CAST(tuple(1, 2, tuple(false, true, tuple('a'))) AS Variant) +checked expr : TRY_CAST(tuple(1_u8, 2_u8, tuple(false, true, tuple("a"))) AS Variant NULL) +optimized expr : 0x4000000310000001100000011000000120000002200000025000002d313233500150024000000310000001100000011000000130000000400000005000000e3132334000000110000001100000013161 +output type : Variant NULL +output domain : Undefined +output : '{"1":1,"2":2,"3":{"1":false,"2":true,"3":{"1":"a"}}}' + + +ast : TRY_CAST(a AS VARIANT) +raw expr : TRY_CAST(a::String NULL AS Variant) +checked expr : TRY_CAST(a AS Variant NULL) +evaluation: ++--------+-------------------------------+--------------------+ +| | a | Output | ++--------+-------------------------------+--------------------+ +| Type | String NULL | Variant NULL | +| Domain | {"[1,2,3]"..="true"} ∪ {NULL} | Undefined ∪ {NULL} | +| Row 0 | 'true' | 'true' | +| Row 1 | NULL | NULL | +| Row 2 | '[1,2,3]' | '[1,2,3]' | ++--------+-------------------------------+--------------------+ +evaluation (internal): ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ +| a | NullableColumn { column: StringColumn[true, {"k":"v"}, [1,2,3]], validity: [0b_____101] } | +| Output | NullableColumn { column: Variant([0x2000000040000000, 0x, 0x80000003200000022000000220000002500150025003]), validity: [0b_____101] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : TRY_CAST(a AS VARIANT) +raw expr : TRY_CAST(a::Bitmap AS Variant) +checked expr : TRY_CAST(a AS Variant NULL) +evaluation: ++--------+-------------+---------------+ +| | a | Output | ++--------+-------------+---------------+ +| Type | Bitmap | Variant NULL | +| Domain | Undefined | Undefined | +| Row 0 | '0,1,2,3,4' | '[0,1,2,3,4]' | +| Row 1 | '1,2,3,4,5' | '[1,2,3,4,5]' | +| Row 2 | '2,3,4,5,6' | '[2,3,4,5,6]' | +| Row 3 | '3,4,5,6,7' | '[3,4,5,6,7]' | ++--------+-------------+---------------+ +evaluation (internal): ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| a | Bitmap([0x0100000000000000000000003a30000001000000000004001000000000000100020003000400, 0x0100000000000000000000003a30000001000000000004001000000001000200030004000500, 0x0100000000000000000000003a30000001000000000004001000000002000300040005000600, 0x0100000000000000000000003a30000001000000000004001000000003000400050006000700]) | +| Output | NullableColumn { column: Variant([0x800000052000000120000002200000022000000220000002005001500250035004, 0x80000005200000022000000220000002200000022000000250015002500350045005, 0x80000005200000022000000220000002200000022000000250025003500450055006, 0x80000005200000022000000220000002200000022000000250035004500550065007]), validity: [0b____1111] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : TRY_CAST(NULL AS VARIANT) +raw expr : TRY_CAST(NULL AS Variant) +checked expr : TRY_CAST(NULL AS Variant NULL) +optimized expr : NULL +func ctx : (modified) +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : TRY_CAST(0 AS VARIANT) +raw expr : TRY_CAST(0 AS Variant) +checked expr : TRY_CAST(0_u8 AS Variant NULL) +optimized expr : 0x200000002000000100 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '0' + + +ast : TRY_CAST(-1 AS VARIANT) +raw expr : TRY_CAST(minus(1) AS Variant) +checked expr : TRY_CAST(minus(1_u8) AS Variant NULL) +optimized expr : 0x200000002000000240ff +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '-1' + + ast : TRY_CAST(1.1 AS VARIANT) raw expr : TRY_CAST(1.1 AS Variant) checked expr : TRY_CAST(1.1_d64(2,1) AS Variant NULL) optimized expr : 0x200000002000000a70000000000000000b01 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '1.1' @@ -2419,6 +2800,7 @@ ast : TRY_CAST('🍦 が美味しい' AS VARIANT) raw expr : TRY_CAST('🍦 が美味しい' AS Variant) checked expr : TRY_CAST("🍦 が美味しい" AS Variant NULL) optimized expr : NULL +func ctx : (modified) output type : Variant NULL output domain : {NULL} output : NULL @@ -2428,6 +2810,7 @@ ast : TRY_CAST([0, 1, 2] AS VARIANT) raw expr : TRY_CAST(array(0, 1, 2) AS Variant) checked expr : TRY_CAST(array(0_u8, 1_u8, 2_u8) AS Variant NULL) optimized expr : 0x800000032000000120000002200000020050015002 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '[0,1,2]' @@ -2437,6 +2820,7 @@ ast : TRY_CAST([0::VARIANT, '"a"'::VARIANT] AS VARIANT) raw expr : TRY_CAST(array(CAST(0 AS Variant), CAST('"a"' AS Variant)) AS Variant) checked expr : TRY_CAST(array(CAST(0_u8 AS Variant), CAST("\"a\"" AS Variant)) AS Variant NULL) optimized expr : 0x8000000220000001100000010061 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '[0,"a"]' @@ -2446,6 +2830,7 @@ ast : TRY_CAST(to_timestamp(1000000) AS VARIANT) raw expr : TRY_CAST(to_timestamp(1000000) AS Variant) checked expr : TRY_CAST(CAST(CAST(1000000_u32 AS Int64) AS Timestamp) AS Variant NULL) optimized expr : 0x200000006000000920000000e8d4a51000 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '"1970-01-12 13:46:40.000000"' @@ -2455,6 +2840,7 @@ ast : TRY_CAST(false AS VARIANT) raw expr : TRY_CAST(false AS Variant) checked expr : TRY_CAST(false AS Variant NULL) optimized expr : 0x2000000030000000 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : 'false' @@ -2464,6 +2850,7 @@ ast : TRY_CAST(true AS VARIANT) raw expr : TRY_CAST(true AS Variant) checked expr : TRY_CAST(true AS Variant NULL) optimized expr : 0x2000000040000000 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : 'true' @@ -2473,6 +2860,7 @@ ast : TRY_CAST(TRY_CAST('"🍦 が美味しい"' AS VARIANT) AS VARIA raw expr : TRY_CAST(TRY_CAST('"🍦 が美味しい"' AS Variant) AS Variant) checked expr : TRY_CAST("\"🍦 が美味しい\"" AS Variant NULL) optimized expr : 0x2000000010000014f09f8da620e3818ce7be8ee591b3e38197e38184 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '"🍦 が美味しい"' @@ -2482,6 +2870,7 @@ ast : TRY_CAST((1,) AS VARIANT) raw expr : TRY_CAST(tuple(1) AS Variant) checked expr : TRY_CAST(tuple(1_u8) AS Variant NULL) optimized expr : 0x400000011000000120000002315001 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '{"1":1}' @@ -2491,6 +2880,7 @@ ast : TRY_CAST((1, 2) AS VARIANT) raw expr : TRY_CAST(tuple(1, 2) AS Variant) checked expr : TRY_CAST(tuple(1_u8, 2_u8) AS Variant NULL) optimized expr : 0x4000000210000001100000012000000220000002313250015002 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '{"1":1,"2":2}' @@ -2500,6 +2890,7 @@ ast : TRY_CAST((false, true) AS VARIANT) raw expr : TRY_CAST(tuple(false, true) AS Variant) checked expr : TRY_CAST(tuple(false, true) AS Variant NULL) optimized expr : 0x40000002100000011000000130000000400000003132 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '{"1":false,"2":true}' @@ -2509,6 +2900,7 @@ ast : TRY_CAST(('a',) AS VARIANT) raw expr : TRY_CAST(tuple('a') AS Variant) checked expr : TRY_CAST(tuple("a") AS Variant NULL) optimized expr : 0x4000000110000001100000013161 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '{"1":"a"}' @@ -2518,6 +2910,7 @@ ast : TRY_CAST((1, 2, (false, true, ('a',))) AS VARIANT) raw expr : TRY_CAST(tuple(1, 2, tuple(false, true, tuple('a'))) AS Variant) checked expr : TRY_CAST(tuple(1_u8, 2_u8, tuple(false, true, tuple("a"))) AS Variant NULL) optimized expr : 0x4000000310000001100000011000000120000002200000025000002d313233500150024000000310000001100000011000000130000000400000005000000e3132334000000110000001100000013161 +func ctx : (modified) output type : Variant NULL output domain : Undefined output : '{"1":1,"2":2,"3":{"1":false,"2":true,"3":{"1":"a"}}}' @@ -2526,6 +2919,7 @@ output : '{"1":1,"2":2,"3":{"1":false,"2":true,"3":{"1":"a"}}}' ast : TRY_CAST(a AS VARIANT) raw expr : TRY_CAST(a::String NULL AS Variant) checked expr : TRY_CAST(a AS Variant NULL) +func ctx : (modified) evaluation: +--------+-------------------------------+--------------------+ | | a | Output | @@ -2548,6 +2942,7 @@ evaluation (internal): ast : TRY_CAST(a AS VARIANT) raw expr : TRY_CAST(a::Bitmap AS Variant) checked expr : TRY_CAST(a AS Variant NULL) +func ctx : (modified) evaluation: +--------+-------------+---------------+ | | a | Output | diff --git a/src/query/functions/tests/it/scalars/testdata/geometry.txt b/src/query/functions/tests/it/scalars/testdata/geometry.txt index 82c91d5514026..5a3388663d6b5 100644 --- a/src/query/functions/tests/it/scalars/testdata/geometry.txt +++ b/src/query/functions/tests/it/scalars/testdata/geometry.txt @@ -217,7 +217,7 @@ output : '9q9j8ue2v71y' ast : st_asgeojson(st_geometryfromwkb(unhex('0101000020797f000066666666a9cb17411f85ebc19e325641'))) raw expr : st_asgeojson(st_geometryfromwkb(unhex('0101000020797f000066666666a9cb17411f85ebc19e325641'))) checked expr : st_asgeojson(st_geometryfromwkb(from_hex("0101000020797f000066666666a9cb17411f85ebc19e325641"))) -optimized expr : 0x400000021000000b100000045000002010000005636f6f7264696e6174657374797065800000022000000a2000000a70000000000252e38b02700000000022af180f02506f696e74 +optimized expr : 0x400000021000000b100000045000001e10000005636f6f7264696e6174657374797065800000022000000920000009604117cba966666666604156329ec1eb851f506f696e74 output type : Variant output domain : Undefined output : '{"coordinates":[389866.35,5819003.03],"type":"Point"}' diff --git a/src/query/functions/tests/it/scalars/testdata/variant.txt b/src/query/functions/tests/it/scalars/testdata/variant.txt index b247e8e7cc4d7..508bd017538b3 100644 --- a/src/query/functions/tests/it/scalars/testdata/variant.txt +++ b/src/query/functions/tests/it/scalars/testdata/variant.txt @@ -107,6 +107,204 @@ evaluation: | Row 9 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | +--------+-----------------------------------------------------------+-------------------------+ evaluation (internal): ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | +| Output | Variant([0x2000000000000000, 0x2000000040000000, 0x2000000020000009507fffffffffffffff, 0x2000000020000003408000, 0x20000000200000096040934a456d5cfaad, 0x2000000020000009604067e66666666666, 0x20000000100000075c226162635c22, 0x20000000100000086461746162656e64, 0x4000000210000001100000011000000110000001616b6276, 0x800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263]) | ++--------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : parse_json(s) +raw expr : parse_json(s::String NULL) +checked expr : CAST(s AS Variant NULL) +evaluation: ++--------+----------------------------+--------------+ +| | s | Output | ++--------+----------------------------+--------------+ +| Type | String NULL | Variant NULL | +| Domain | {"1234"..="true"} ∪ {NULL} | Unknown | +| Row 0 | 'true' | 'true' | +| Row 1 | 'false' | 'false' | +| Row 2 | NULL | NULL | +| Row 3 | '1234' | '1234' | ++--------+----------------------------+--------------+ +evaluation (internal): ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, false, , 1234], validity: [0b____1011] } | +| Output | NullableColumn { column: Variant([0x2000000040000000, 0x2000000030000000, 0x, 0x20000000200000035004d2]), validity: [0b____1011] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ + + +error: + --> SQL:1:1 + | +1 | parse_json('+10') + | ^^^^^^^^^^^^^^^^^ invalid number, pos 1 while evaluating function `parse_json('+10')` in expr `CAST('+10' AS Variant)` + + + +error: + --> SQL:1:1 + | +1 | parse_json('001') + | ^^^^^^^^^^^^^^^^^ invalid number, pos 2 while evaluating function `parse_json('001')` in expr `CAST('001' AS Variant)` + + + +error: + --> SQL:1:1 + | +1 | parse_json('.12') + | ^^^^^^^^^^^^^^^^^ invalid number, pos 1 while evaluating function `parse_json('.12')` in expr `CAST('.12' AS Variant)` + + + +error: + --> SQL:1:1 + | +1 | parse_json('12.') + | ^^^^^^^^^^^^^^^^^ invalid number, pos 4 while evaluating function `parse_json('12.')` in expr `CAST('12.' AS Variant)` + + + +ast : parse_json('99999999999999999999999999999999999999') +raw expr : parse_json('99999999999999999999999999999999999999') +checked expr : CAST("99999999999999999999999999999999999999" AS Variant) +optimized expr : 0x20000000200000096047d2ced32a16a1b1 +output type : Variant +output domain : Undefined +output : '1e38' + + +error: + --> SQL:1:1 + | +1 | parse_json('[1,2,,4]') + | ^^^^^^^^^^^^^^^^^^^^^^ expected value, pos 5 while evaluating function `parse_json('[1,2,,4]')` in expr `CAST('[1,2,,4]' AS Variant)` + + + +ast : parse_json(NULL) +raw expr : parse_json(NULL) +checked expr : CAST(CAST(NULL AS Variant NULL) AS Variant NULL) +optimized expr : NULL +func ctx : (modified) +output type : Variant NULL +output domain : {NULL} +output : NULL + + +error: + --> SQL:1:1 + | +1 | parse_json('nuLL') + | ^^^^^^^^^^^^^^^^^^ expected ident, pos 3 while evaluating function `parse_json('nuLL')` in expr `CAST('nuLL' AS Variant)` + + + +ast : parse_json('null') +raw expr : parse_json('null') +checked expr : CAST("null" AS Variant) +optimized expr : 0x2000000000000000 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : 'null' + + +error: + --> SQL:1:1 + | +1 | parse_json(' ') + | ^^^^^^^^^^^^^^^^ EOF while parsing a value, pos 2 while evaluating function `parse_json(' ')` in expr `CAST(' ' AS Variant)` + + + +ast : parse_json('true') +raw expr : parse_json('true') +checked expr : CAST("true" AS Variant) +optimized expr : 0x2000000040000000 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : 'true' + + +ast : parse_json('false') +raw expr : parse_json('false') +checked expr : CAST("false" AS Variant) +optimized expr : 0x2000000030000000 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : 'false' + + +ast : parse_json('"测试"') +raw expr : parse_json('"测试"') +checked expr : CAST("\"测试\"" AS Variant) +optimized expr : 0x2000000010000006e6b58be8af95 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : '"测试"' + + +ast : parse_json('1234') +raw expr : parse_json('1234') +checked expr : CAST("1234" AS Variant) +optimized expr : 0x20000000200000035004d2 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : '1234' + + +ast : parse_json('[1,2,3,4]') +raw expr : parse_json('[1,2,3,4]') +checked expr : CAST("[1,2,3,4]" AS Variant) +optimized expr : 0x80000004200000022000000220000002200000025001500250035004 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : '[1,2,3,4]' + + +ast : parse_json('{"a":"b","c":"d"}') +raw expr : parse_json('{"a":"b","c":"d"}') +checked expr : CAST("{\"a\":\"b\",\"c\":\"d\"}" AS Variant) +optimized expr : 0x400000021000000110000001100000011000000161636264 +func ctx : (modified) +output type : Variant +output domain : Undefined +output : '{"a":"b","c":"d"}' + + +ast : parse_json(s) +raw expr : parse_json(s::String) +checked expr : CAST(s AS Variant) +func ctx : (modified) +evaluation: ++--------+-----------------------------------------------------------+-------------------------+ +| | s | Output | ++--------+-----------------------------------------------------------+-------------------------+ +| Type | String | Variant | +| Domain | {"\"\\\\\\\"abc\\\\\\\"\""..="{\"k\":\"v\",\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | 'null' | +| Row 1 | 'true' | 'true' | +| Row 2 | '9223372036854775807' | '9223372036854775807' | +| Row 3 | '-32768' | '-32768' | +| Row 4 | '1234.5678' | '1234.5678' | +| Row 5 | '1.912e2' | '191.2' | +| Row 6 | '"\\\\\\"abc\\\\\\""' | '"\\\\\\"abc\\\\\\""' | +| Row 7 | '"databend"' | '"databend"' | +| Row 8 | '{"k":"v","a":"b"}' | '{"a":"b","k":"v"}' | +| Row 9 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | ++--------+-----------------------------------------------------------+-------------------------+ +evaluation (internal): +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ | Column | Data | +--------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ @@ -118,6 +316,7 @@ evaluation (internal): ast : parse_json(s) raw expr : parse_json(s::String NULL) checked expr : CAST(s AS Variant NULL) +func ctx : (modified) evaluation: +--------+----------------------------+--------------+ | | s | Output | @@ -142,6 +341,7 @@ ast : parse_json('+10') raw expr : parse_json('+10') checked expr : CAST("+10" AS Variant) optimized expr : 0x2000000020000002500a +func ctx : (modified) output type : Variant output domain : Undefined output : '10' @@ -151,6 +351,7 @@ ast : parse_json('001') raw expr : parse_json('001') checked expr : CAST("001" AS Variant) optimized expr : 0x20000000200000025001 +func ctx : (modified) output type : Variant output domain : Undefined output : '1' @@ -160,6 +361,7 @@ ast : parse_json('.12') raw expr : parse_json('.12') checked expr : CAST(".12" AS Variant) optimized expr : 0x200000002000000a70000000000000000c02 +func ctx : (modified) output type : Variant output domain : Undefined output : '0.12' @@ -169,6 +371,7 @@ ast : parse_json('12.') raw expr : parse_json('12.') checked expr : CAST("12." AS Variant) optimized expr : 0x2000000020000002500c +func ctx : (modified) output type : Variant output domain : Undefined output : '12' @@ -178,6 +381,7 @@ ast : parse_json('99999999999999999999999999999999999999') raw expr : parse_json('99999999999999999999999999999999999999') checked expr : CAST("99999999999999999999999999999999999999" AS Variant) optimized expr : 0x2000000020000012704b3b4ca85a86c47a098a223fffffffff00 +func ctx : (modified) output type : Variant output domain : Undefined output : '99999999999999999999999999999999999999' @@ -187,6 +391,7 @@ ast : parse_json('[1,2,,4]') raw expr : parse_json('[1,2,,4]') checked expr : CAST("[1,2,,4]" AS Variant) optimized expr : 0x8000000420000002200000020000000020000002500150025004 +func ctx : (modified) output type : Variant output domain : Undefined output : '[1,2,null,4]' @@ -219,6 +424,15 @@ output domain : Undefined output : 'null' +ast : try_parse_json(' ') +raw expr : try_parse_json(' ') +checked expr : try_parse_json(" ") +optimized expr : NULL +output type : Variant NULL +output domain : {NULL} +output : NULL + + ast : try_parse_json('true') raw expr : try_parse_json('true') checked expr : try_parse_json("true") @@ -294,104 +508,372 @@ evaluation: | Row 9 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | +--------+-----------------------------------------------------------+-------------------------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | -| Output | NullableColumn { column: Variant([0x2000000000000000, 0x2000000040000000, 0x2000000020000009507fffffffffffffff, 0x2000000020000003408000, 0x200000002000000a700000000000bc614e04, 0x2000000020000009604067e66666666666, 0x20000000100000075c226162635c22, 0x20000000100000086461746162656e64, 0x4000000210000001100000011000000110000001616b6276, 0x800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263]), validity: [0b11111111, 0b______11] } | -+--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | +| Output | NullableColumn { column: Variant([0x2000000000000000, 0x2000000040000000, 0x2000000020000009507fffffffffffffff, 0x2000000020000003408000, 0x20000000200000096040934a456d5cfaad, 0x2000000020000009604067e66666666666, 0x20000000100000075c226162635c22, 0x20000000100000086461746162656e64, 0x4000000210000001100000011000000110000001616b6276, 0x800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263]), validity: [0b11111111, 0b______11] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ ast : try_parse_json(s) raw expr : try_parse_json(s::String NULL) checked expr : try_parse_json(s) evaluation: -+--------+---------------------------+--------------------+ -| | s | Output | -+--------+---------------------------+--------------------+ -| Type | String NULL | Variant NULL | -| Domain | {"1234"..="ttt"} ∪ {NULL} | Undefined ∪ {NULL} | -| Row 0 | 'true' | 'true' | -| Row 1 | 'ttt' | NULL | -| Row 2 | NULL | NULL | -| Row 3 | '1234' | '1234' | -+--------+---------------------------+--------------------+ ++--------+----------------------------+--------------------+ +| | s | Output | ++--------+----------------------------+--------------------+ +| Type | String NULL | Variant NULL | +| Domain | {"1234"..="true"} ∪ {NULL} | Undefined ∪ {NULL} | +| Row 0 | 'true' | 'true' | +| Row 1 | 'false' | 'false' | +| Row 2 | NULL | NULL | +| Row 3 | '1234' | '1234' | ++--------+----------------------------+--------------------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------+ -| s | NullableColumn { column: StringColumn[true, ttt, , 1234], validity: [0b____1011] } | -| Output | NullableColumn { column: Variant([0x2000000040000000, 0x, 0x, 0x20000000200000035004d2]), validity: [0b____1001] } | -+--------+--------------------------------------------------------------------------------------------------------------------+ ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, false, , 1234], validity: [0b____1011] } | +| Output | NullableColumn { column: Variant([0x2000000040000000, 0x2000000030000000, 0x, 0x20000000200000035004d2]), validity: [0b____1011] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ -ast : check_json(NULL) -raw expr : check_json(NULL) -checked expr : check_json(CAST(NULL AS Variant NULL)) +ast : try_parse_json('+10') +raw expr : try_parse_json('+10') +checked expr : try_parse_json("+10") optimized expr : NULL -output type : String NULL +output type : Variant NULL output domain : {NULL} output : NULL -ast : check_json('true') -raw expr : check_json('true') -checked expr : check_json("true") +ast : try_parse_json('001') +raw expr : try_parse_json('001') +checked expr : try_parse_json("001") optimized expr : NULL -output type : String NULL +output type : Variant NULL output domain : {NULL} output : NULL -ast : check_json('nuLL') -raw expr : check_json('nuLL') -checked expr : check_json("nuLL") -optimized expr : "expected ident, pos 3" -output type : String NULL -output domain : {"expected ident, pos 3"..="expected ident, pos 3"} -output : 'expected ident, pos 3' +ast : try_parse_json('.12') +raw expr : try_parse_json('.12') +checked expr : try_parse_json(".12") +optimized expr : NULL +output type : Variant NULL +output domain : {NULL} +output : NULL -ast : check_json(s) -raw expr : check_json(s::String) -checked expr : check_json(s) -evaluation: -+--------+------------------+-------------------------+ -| | s | Output | -+--------+------------------+-------------------------+ -| Type | String | String NULL | -| Domain | {"abc"..="true"} | {""..} ∪ {NULL} | -| Row 0 | 'null' | NULL | -| Row 1 | 'abc' | 'expected value, pos 1' | -| Row 2 | 'true' | NULL | -+--------+------------------+-------------------------+ -evaluation (internal): -+--------+--------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------+ -| s | StringColumn[null, abc, true] | -| Output | NullableColumn { column: StringColumn[, expected value, pos 1, ], validity: [0b_____010] } | -+--------+--------------------------------------------------------------------------------------------+ +ast : try_parse_json('12.') +raw expr : try_parse_json('12.') +checked expr : try_parse_json("12.") +optimized expr : NULL +output type : Variant NULL +output domain : {NULL} +output : NULL -ast : check_json(s) -raw expr : check_json(s::String NULL) -checked expr : check_json(s) -evaluation: -+--------+---------------------------+-------------------------+ -| | s | Output | -+--------+---------------------------+-------------------------+ -| Type | String NULL | String NULL | -| Domain | {"1234"..="ttt"} ∪ {NULL} | {""..} ∪ {NULL} | -| Row 0 | 'true' | NULL | -| Row 1 | 'ttt' | 'expected ident, pos 2' | -| Row 2 | NULL | NULL | -| Row 3 | '1234' | NULL | -+--------+---------------------------+-------------------------+ -evaluation (internal): -+--------+----------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------+ +ast : try_parse_json('99999999999999999999999999999999999999') +raw expr : try_parse_json('99999999999999999999999999999999999999') +checked expr : try_parse_json("99999999999999999999999999999999999999") +optimized expr : 0x20000000200000096047d2ced32a16a1b1 +output type : Variant NULL +output domain : Undefined +output : '1e38' + + +ast : try_parse_json('[1,2,,4]') +raw expr : try_parse_json('[1,2,,4]') +checked expr : try_parse_json("[1,2,,4]") +optimized expr : NULL +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : try_parse_json(NULL) +raw expr : try_parse_json(NULL) +checked expr : try_parse_json(CAST(NULL AS Variant NULL)) +optimized expr : NULL +func ctx : (modified) +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : try_parse_json('nuLL') +raw expr : try_parse_json('nuLL') +checked expr : try_parse_json("nuLL") +optimized expr : NULL +func ctx : (modified) +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : try_parse_json('null') +raw expr : try_parse_json('null') +checked expr : try_parse_json("null") +optimized expr : 0x2000000000000000 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : 'null' + + +ast : try_parse_json(' ') +raw expr : try_parse_json(' ') +checked expr : try_parse_json(" ") +optimized expr : NULL +func ctx : (modified) +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : try_parse_json('true') +raw expr : try_parse_json('true') +checked expr : try_parse_json("true") +optimized expr : 0x2000000040000000 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : 'true' + + +ast : try_parse_json('false') +raw expr : try_parse_json('false') +checked expr : try_parse_json("false") +optimized expr : 0x2000000030000000 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : 'false' + + +ast : try_parse_json('"测试"') +raw expr : try_parse_json('"测试"') +checked expr : try_parse_json("\"测试\"") +optimized expr : 0x2000000010000006e6b58be8af95 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '"测试"' + + +ast : try_parse_json('1234') +raw expr : try_parse_json('1234') +checked expr : try_parse_json("1234") +optimized expr : 0x20000000200000035004d2 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '1234' + + +ast : try_parse_json('[1,2,3,4]') +raw expr : try_parse_json('[1,2,3,4]') +checked expr : try_parse_json("[1,2,3,4]") +optimized expr : 0x80000004200000022000000220000002200000025001500250035004 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '[1,2,3,4]' + + +ast : try_parse_json('{"a":"b","c":"d"}') +raw expr : try_parse_json('{"a":"b","c":"d"}') +checked expr : try_parse_json("{\"a\":\"b\",\"c\":\"d\"}") +optimized expr : 0x400000021000000110000001100000011000000161636264 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '{"a":"b","c":"d"}' + + +ast : try_parse_json(s) +raw expr : try_parse_json(s::String) +checked expr : try_parse_json(s) +func ctx : (modified) +evaluation: ++--------+-----------------------------------------------------------+-------------------------+ +| | s | Output | ++--------+-----------------------------------------------------------+-------------------------+ +| Type | String | Variant NULL | +| Domain | {"\"\\\\\\\"abc\\\\\\\"\""..="{\"k\":\"v\",\"a\":\"b\"}"} | Undefined ∪ {NULL} | +| Row 0 | 'null' | 'null' | +| Row 1 | 'true' | 'true' | +| Row 2 | '9223372036854775807' | '9223372036854775807' | +| Row 3 | '-32768' | '-32768' | +| Row 4 | '1234.5678' | '1234.5678' | +| Row 5 | '1.912e2' | '191.2' | +| Row 6 | '"\\\\\\"abc\\\\\\""' | '"\\\\\\"abc\\\\\\""' | +| Row 7 | '"databend"' | '"databend"' | +| Row 8 | '{"k":"v","a":"b"}' | '{"a":"b","k":"v"}' | +| Row 9 | '[1,2,3,["a","b","c"]]' | '[1,2,3,["a","b","c"]]' | ++--------+-----------------------------------------------------------+-------------------------+ +evaluation (internal): ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 9223372036854775807, -32768, 1234.5678, 1.912e2, "\\\"abc\\\"", "databend", {"k":"v","a":"b"}, [1,2,3,["a","b","c"]]] | +| Output | NullableColumn { column: Variant([0x2000000000000000, 0x2000000040000000, 0x2000000020000009507fffffffffffffff, 0x2000000020000003408000, 0x200000002000000a700000000000bc614e04, 0x2000000020000009604067e66666666666, 0x20000000100000075c226162635c22, 0x20000000100000086461746162656e64, 0x4000000210000001100000011000000110000001616b6276, 0x800000042000000220000002200000025000001350015002500380000003100000011000000110000001616263]), validity: [0b11111111, 0b______11] } | ++--------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : try_parse_json(s) +raw expr : try_parse_json(s::String NULL) +checked expr : try_parse_json(s) +func ctx : (modified) +evaluation: ++--------+----------------------------+--------------------+ +| | s | Output | ++--------+----------------------------+--------------------+ +| Type | String NULL | Variant NULL | +| Domain | {"1234"..="true"} ∪ {NULL} | Undefined ∪ {NULL} | +| Row 0 | 'true' | 'true' | +| Row 1 | 'false' | 'false' | +| Row 2 | NULL | NULL | +| Row 3 | '1234' | '1234' | ++--------+----------------------------+--------------------+ +evaluation (internal): ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ +| s | NullableColumn { column: StringColumn[true, false, , 1234], validity: [0b____1011] } | +| Output | NullableColumn { column: Variant([0x2000000040000000, 0x2000000030000000, 0x, 0x20000000200000035004d2]), validity: [0b____1011] } | ++--------+------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : try_parse_json('+10') +raw expr : try_parse_json('+10') +checked expr : try_parse_json("+10") +optimized expr : 0x2000000020000002500a +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '10' + + +ast : try_parse_json('001') +raw expr : try_parse_json('001') +checked expr : try_parse_json("001") +optimized expr : 0x20000000200000025001 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '1' + + +ast : try_parse_json('.12') +raw expr : try_parse_json('.12') +checked expr : try_parse_json(".12") +optimized expr : 0x200000002000000a70000000000000000c02 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '0.12' + + +ast : try_parse_json('12.') +raw expr : try_parse_json('12.') +checked expr : try_parse_json("12.") +optimized expr : 0x2000000020000002500c +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '12' + + +ast : try_parse_json('99999999999999999999999999999999999999') +raw expr : try_parse_json('99999999999999999999999999999999999999') +checked expr : try_parse_json("99999999999999999999999999999999999999") +optimized expr : 0x2000000020000012704b3b4ca85a86c47a098a223fffffffff00 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '99999999999999999999999999999999999999' + + +ast : try_parse_json('[1,2,,4]') +raw expr : try_parse_json('[1,2,,4]') +checked expr : try_parse_json("[1,2,,4]") +optimized expr : 0x8000000420000002200000020000000020000002500150025004 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '[1,2,null,4]' + + +ast : check_json(NULL) +raw expr : check_json(NULL) +checked expr : check_json(CAST(NULL AS Variant NULL)) +optimized expr : NULL +output type : String NULL +output domain : {NULL} +output : NULL + + +ast : check_json('true') +raw expr : check_json('true') +checked expr : check_json("true") +optimized expr : NULL +output type : String NULL +output domain : {NULL} +output : NULL + + +ast : check_json('nuLL') +raw expr : check_json('nuLL') +checked expr : check_json("nuLL") +optimized expr : "expected ident, pos 3" +output type : String NULL +output domain : {"expected ident, pos 3"..="expected ident, pos 3"} +output : 'expected ident, pos 3' + + +ast : check_json(s) +raw expr : check_json(s::String) +checked expr : check_json(s) +evaluation: ++--------+------------------+-------------------------+ +| | s | Output | ++--------+------------------+-------------------------+ +| Type | String | String NULL | +| Domain | {"abc"..="true"} | {""..} ∪ {NULL} | +| Row 0 | 'null' | NULL | +| Row 1 | 'abc' | 'expected value, pos 1' | +| Row 2 | 'true' | NULL | ++--------+------------------+-------------------------+ +evaluation (internal): ++--------+--------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------+ +| s | StringColumn[null, abc, true] | +| Output | NullableColumn { column: StringColumn[, expected value, pos 1, ], validity: [0b_____010] } | ++--------+--------------------------------------------------------------------------------------------+ + + +ast : check_json(s) +raw expr : check_json(s::String NULL) +checked expr : check_json(s) +evaluation: ++--------+---------------------------+-------------------------+ +| | s | Output | ++--------+---------------------------+-------------------------+ +| Type | String NULL | String NULL | +| Domain | {"1234"..="ttt"} ∪ {NULL} | {""..} ∪ {NULL} | +| Row 0 | 'true' | NULL | +| Row 1 | 'ttt' | 'expected ident, pos 2' | +| Row 2 | NULL | NULL | +| Row 3 | '1234' | NULL | ++--------+---------------------------+-------------------------+ +evaluation (internal): ++--------+----------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------+ | s | NullableColumn { column: StringColumn[true, ttt, , 1234], validity: [0b____1011] } | | Output | NullableColumn { column: StringColumn[, expected ident, pos 2, , ], validity: [0b____0010] } | +--------+----------------------------------------------------------------------------------------------+ @@ -1139,37 +1621,37 @@ output : '{"a":"b"}' ast : as_binary(to_binary('abcd')::variant) raw expr : as_binary(CAST(to_binary('abcd') AS Variant)) checked expr : as_binary(CAST(CAST("abcd" AS Binary) AS Variant)) -optimized expr : 61626364 +optimized expr : NULL output type : Binary NULL -output domain : Undefined -output : 61626364 +output domain : {NULL} +output : NULL ast : as_date(to_date('2025-10-11')::variant) raw expr : as_date(CAST(to_date('2025-10-11') AS Variant)) checked expr : as_date(CAST(CAST("2025-10-11" AS Date) AS Variant)) -optimized expr : 20372 +optimized expr : NULL output type : Date NULL -output domain : {20372..=20372} -output : '2025-10-11' +output domain : {NULL} +output : NULL ast : as_timestamp(to_timestamp('2025-05-01 10:00:00')::variant) raw expr : as_timestamp(CAST(to_timestamp('2025-05-01 10:00:00') AS Variant)) checked expr : as_timestamp(CAST(CAST("2025-05-01 10:00:00" AS Timestamp) AS Variant)) -optimized expr : 1746093600000000 +optimized expr : NULL output type : Timestamp NULL -output domain : {1746093600000000..=1746093600000000} -output : '2025-05-01 10:00:00.000000' +output domain : {NULL} +output : NULL ast : as_interval(to_interval('1 year 2 month')::variant) raw expr : as_interval(CAST(to_interval('1 year 2 month') AS Variant)) checked expr : as_interval(CAST(CAST("1 year 2 month" AS Interval) AS Variant)) -optimized expr : 1 year 2 months +optimized expr : NULL output type : Interval NULL -output domain : SimpleDomain { min: months_days_micros(1109194275199700726309615304704), max: months_days_micros(1109194275199700726309615304704) } -output : '1 year 2 months' +output domain : {NULL} +output : NULL ast : as_boolean(parse_json(s)) @@ -1228,110 +1710,815 @@ ast : as_float(parse_json(s)) raw expr : as_float(parse_json(s::String)) checked expr : as_float(CAST(s AS Variant)) evaluation: -+--------+------------------------------+--------------+ -| | s | Output | -+--------+------------------------------+--------------+ -| Type | String | Float64 NULL | -| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | -| Row 0 | 'null' | NULL | -| Row 1 | 'true' | NULL | -| Row 2 | '123' | 123 | -| Row 3 | '12.34' | 12.34 | -| Row 4 | '"ab"' | NULL | -| Row 5 | '[1,2,3]' | NULL | -| Row 6 | '{"a":"b"}' | NULL | -+--------+------------------------------+--------------+ ++--------+------------------------------+--------------+ +| | s | Output | ++--------+------------------------------+--------------+ +| Type | String | Float64 NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | 123 | +| Row 3 | '12.34' | 12.34 | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+--------------+ +evaluation (internal): ++--------+-----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Float64([0, 0, 123, 12.34, 0, 0, 0]), validity: [0b_0001100] } | ++--------+-----------------------------------------------------------------------------------------+ + + +ast : as_string(parse_json(s)) +raw expr : as_string(parse_json(s::String)) +checked expr : as_string(CAST(s AS Variant)) +evaluation: ++--------+------------------------------+-------------+ +| | s | Output | ++--------+------------------------------+-------------+ +| Type | String | String NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | NULL | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | 'ab' | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+-------------+ +evaluation (internal): ++--------+---------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: StringColumn[, , , , ab, , ], validity: [0b_0010000] } | ++--------+---------------------------------------------------------------------------------+ + + +ast : as_array(parse_json(s)) +raw expr : as_array(parse_json(s::String)) +checked expr : as_array(CAST(s AS Variant)) +evaluation: ++--------+------------------------------+--------------+ +| | s | Output | ++--------+------------------------------+--------------+ +| Type | String | Variant NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | NULL | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | '[1,2,3]' | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+--------------+ +evaluation (internal): ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Variant([0x, 0x, 0x, 0x, 0x, 0x80000003200000022000000220000002500150025003, 0x]), validity: [0b_0100000] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : as_object(parse_json(s)) +raw expr : as_object(parse_json(s::String)) +checked expr : as_object(CAST(s AS Variant)) +evaluation: ++--------+------------------------------+--------------+ +| | s | Output | ++--------+------------------------------+--------------+ +| Type | String | Variant NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | NULL | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | '{"a":"b"}' | ++--------+------------------------------+--------------+ +evaluation (internal): ++--------+----------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Variant([0x, 0x, 0x, 0x, 0x, 0x, 0x4000000110000001100000016162]), validity: [0b_1000000] } | ++--------+----------------------------------------------------------------------------------------------------------------------+ + + +ast : as_boolean(parse_json('true')) +raw expr : as_boolean(parse_json('true')) +checked expr : as_boolean(CAST("true" AS Variant)) +optimized expr : true +func ctx : (modified) +output type : Boolean NULL +output domain : {TRUE} +output : true + + +ast : as_boolean(parse_json('123')) +raw expr : as_boolean(parse_json('123')) +checked expr : as_boolean(CAST("123" AS Variant)) +optimized expr : NULL +func ctx : (modified) +output type : Boolean NULL +output domain : {NULL} +output : NULL + + +ast : as_integer(parse_json('true')) +raw expr : as_integer(parse_json('true')) +checked expr : as_integer(CAST("true" AS Variant)) +optimized expr : NULL +func ctx : (modified) +output type : Int64 NULL +output domain : {NULL} +output : NULL + + +ast : as_integer(parse_json('123')) +raw expr : as_integer(parse_json('123')) +checked expr : as_integer(CAST("123" AS Variant)) +optimized expr : 123_i64 +func ctx : (modified) +output type : Int64 NULL +output domain : {123..=123} +output : 123 + + +ast : as_float(parse_json('"ab"')) +raw expr : as_float(parse_json('"ab"')) +checked expr : as_float(CAST("\"ab\"" AS Variant)) +optimized expr : NULL +func ctx : (modified) +output type : Float64 NULL +output domain : {NULL} +output : NULL + + +ast : as_float(parse_json('12.34')) +raw expr : as_float(parse_json('12.34')) +checked expr : as_float(CAST("12.34" AS Variant)) +optimized expr : 12.34_f64 +func ctx : (modified) +output type : Float64 NULL +output domain : {12.34..=12.34} +output : 12.34 + + +ast : as_decimal(parse_json('12.34')) +raw expr : as_decimal(parse_json('12.34')) +checked expr : as_decimal(CAST("12.34" AS Variant)) +optimized expr : 12_d128(38,0) +func ctx : (modified) +output type : Decimal(38, 0) NULL +output domain : {12..=12} +output : 12 + + +ast : as_decimal(10, 2)(parse_json('12.34')) +raw expr : as_decimal(10, 2)(parse_json('12.34')) +checked expr : as_decimal(10, 2)(CAST("12.34" AS Variant)) +optimized expr : 12.34_d64(10,2) +func ctx : (modified) +output type : Decimal(10, 2) NULL +output domain : {12.34..=12.34} +output : 12.34 + + +ast : as_string(parse_json('"ab"')) +raw expr : as_string(parse_json('"ab"')) +checked expr : as_string(CAST("\"ab\"" AS Variant)) +optimized expr : "ab" +func ctx : (modified) +output type : String NULL +output domain : {"ab"..="ab"} +output : 'ab' + + +ast : as_string(parse_json('12.34')) +raw expr : as_string(parse_json('12.34')) +checked expr : as_string(CAST("12.34" AS Variant)) +optimized expr : NULL +func ctx : (modified) +output type : String NULL +output domain : {NULL} +output : NULL + + +ast : as_array(parse_json('[1,2,3]')) +raw expr : as_array(parse_json('[1,2,3]')) +checked expr : as_array(CAST("[1,2,3]" AS Variant)) +optimized expr : 0x80000003200000022000000220000002500150025003 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '[1,2,3]' + + +ast : as_array(parse_json('{"a":"b"}')) +raw expr : as_array(parse_json('{"a":"b"}')) +checked expr : as_array(CAST("{\"a\":\"b\"}" AS Variant)) +optimized expr : NULL +func ctx : (modified) +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : as_object(parse_json('[1,2,3]')) +raw expr : as_object(parse_json('[1,2,3]')) +checked expr : as_object(CAST("[1,2,3]" AS Variant)) +optimized expr : NULL +func ctx : (modified) +output type : Variant NULL +output domain : {NULL} +output : NULL + + +ast : as_object(parse_json('{"a":"b"}')) +raw expr : as_object(parse_json('{"a":"b"}')) +checked expr : as_object(CAST("{\"a\":\"b\"}" AS Variant)) +optimized expr : 0x4000000110000001100000016162 +func ctx : (modified) +output type : Variant NULL +output domain : Undefined +output : '{"a":"b"}' + + +ast : as_binary(to_binary('abcd')::variant) +raw expr : as_binary(CAST(to_binary('abcd') AS Variant)) +checked expr : as_binary(CAST(CAST("abcd" AS Binary) AS Variant)) +optimized expr : 61626364 +func ctx : (modified) +output type : Binary NULL +output domain : Undefined +output : 61626364 + + +ast : as_date(to_date('2025-10-11')::variant) +raw expr : as_date(CAST(to_date('2025-10-11') AS Variant)) +checked expr : as_date(CAST(CAST("2025-10-11" AS Date) AS Variant)) +optimized expr : 20372 +func ctx : (modified) +output type : Date NULL +output domain : {20372..=20372} +output : '2025-10-11' + + +ast : as_timestamp(to_timestamp('2025-05-01 10:00:00')::variant) +raw expr : as_timestamp(CAST(to_timestamp('2025-05-01 10:00:00') AS Variant)) +checked expr : as_timestamp(CAST(CAST("2025-05-01 10:00:00" AS Timestamp) AS Variant)) +optimized expr : 1746093600000000 +func ctx : (modified) +output type : Timestamp NULL +output domain : {1746093600000000..=1746093600000000} +output : '2025-05-01 10:00:00.000000' + + +ast : as_interval(to_interval('1 year 2 month')::variant) +raw expr : as_interval(CAST(to_interval('1 year 2 month') AS Variant)) +checked expr : as_interval(CAST(CAST("1 year 2 month" AS Interval) AS Variant)) +optimized expr : 1 year 2 months +func ctx : (modified) +output type : Interval NULL +output domain : SimpleDomain { min: months_days_micros(1109194275199700726309615304704), max: months_days_micros(1109194275199700726309615304704) } +output : '1 year 2 months' + + +ast : as_boolean(parse_json(s)) +raw expr : as_boolean(parse_json(s::String)) +checked expr : as_boolean(CAST(s AS Variant)) +func ctx : (modified) +evaluation: ++--------+------------------------------+--------------+ +| | s | Output | ++--------+------------------------------+--------------+ +| Type | String | Boolean NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | true | +| Row 2 | '123' | NULL | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+--------------+ +evaluation (internal): ++--------+--------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Boolean([0b_0000010]), validity: [0b_0000010] } | ++--------+--------------------------------------------------------------------------+ + + +ast : as_integer(parse_json(s)) +raw expr : as_integer(parse_json(s::String)) +checked expr : as_integer(CAST(s AS Variant)) +func ctx : (modified) +evaluation: ++--------+------------------------------+------------+ +| | s | Output | ++--------+------------------------------+------------+ +| Type | String | Int64 NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | 123 | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+------------+ +evaluation (internal): ++--------+-----------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Int64([0, 0, 123, 0, 0, 0, 0]), validity: [0b_0000100] } | ++--------+-----------------------------------------------------------------------------------+ + + +ast : as_float(parse_json(s)) +raw expr : as_float(parse_json(s::String)) +checked expr : as_float(CAST(s AS Variant)) +func ctx : (modified) +evaluation: ++--------+------------------------------+--------------+ +| | s | Output | ++--------+------------------------------+--------------+ +| Type | String | Float64 NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | 123 | +| Row 3 | '12.34' | 12.34 | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+--------------+ +evaluation (internal): ++--------+-----------------------------------------------------------------------------------------+ +| Column | Data | ++--------+-----------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Float64([0, 0, 123, 12.34, 0, 0, 0]), validity: [0b_0001100] } | ++--------+-----------------------------------------------------------------------------------------+ + + +ast : as_string(parse_json(s)) +raw expr : as_string(parse_json(s::String)) +checked expr : as_string(CAST(s AS Variant)) +func ctx : (modified) +evaluation: ++--------+------------------------------+-------------+ +| | s | Output | ++--------+------------------------------+-------------+ +| Type | String | String NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | NULL | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | 'ab' | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+-------------+ +evaluation (internal): ++--------+---------------------------------------------------------------------------------+ +| Column | Data | ++--------+---------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: StringColumn[, , , , ab, , ], validity: [0b_0010000] } | ++--------+---------------------------------------------------------------------------------+ + + +ast : as_array(parse_json(s)) +raw expr : as_array(parse_json(s::String)) +checked expr : as_array(CAST(s AS Variant)) +func ctx : (modified) +evaluation: ++--------+------------------------------+--------------+ +| | s | Output | ++--------+------------------------------+--------------+ +| Type | String | Variant NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | NULL | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | '[1,2,3]' | +| Row 6 | '{"a":"b"}' | NULL | ++--------+------------------------------+--------------+ +evaluation (internal): ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Variant([0x, 0x, 0x, 0x, 0x, 0x80000003200000022000000220000002500150025003, 0x]), validity: [0b_0100000] } | ++--------+--------------------------------------------------------------------------------------------------------------------------------------+ + + +ast : as_object(parse_json(s)) +raw expr : as_object(parse_json(s::String)) +checked expr : as_object(CAST(s AS Variant)) +func ctx : (modified) +evaluation: ++--------+------------------------------+--------------+ +| | s | Output | ++--------+------------------------------+--------------+ +| Type | String | Variant NULL | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | NULL | +| Row 1 | 'true' | NULL | +| Row 2 | '123' | NULL | +| Row 3 | '12.34' | NULL | +| Row 4 | '"ab"' | NULL | +| Row 5 | '[1,2,3]' | NULL | +| Row 6 | '{"a":"b"}' | '{"a":"b"}' | ++--------+------------------------------+--------------+ +evaluation (internal): ++--------+----------------------------------------------------------------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | NullableColumn { column: Variant([0x, 0x, 0x, 0x, 0x, 0x, 0x4000000110000001100000016162]), validity: [0b_1000000] } | ++--------+----------------------------------------------------------------------------------------------------------------------+ + + +ast : is_null_value(parse_json('null')) +raw expr : is_null_value(parse_json('null')) +checked expr : is_null_value(CAST("null" AS Variant)) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + +ast : is_null_value(parse_json('[1,2]')) +raw expr : is_null_value(parse_json('[1,2]')) +checked expr : is_null_value(CAST("[1,2]" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_boolean(parse_json('true')) +raw expr : is_boolean(parse_json('true')) +checked expr : is_boolean(CAST("true" AS Variant)) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + +ast : is_boolean(parse_json('123')) +raw expr : is_boolean(parse_json('123')) +checked expr : is_boolean(CAST("123" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_integer(parse_json('true')) +raw expr : is_integer(parse_json('true')) +checked expr : is_integer(CAST("true" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_integer(parse_json('123')) +raw expr : is_integer(parse_json('123')) +checked expr : is_integer(CAST("123" AS Variant)) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + +ast : is_float(parse_json('"ab"')) +raw expr : is_float(parse_json('"ab"')) +checked expr : is_float(CAST("\"ab\"" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_float(parse_json('12.34')) +raw expr : is_float(parse_json('12.34')) +checked expr : is_float(CAST("12.34" AS Variant)) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + +ast : is_decimal(parse_json('99999999999999999999999999999999999999')) +raw expr : is_decimal(parse_json('99999999999999999999999999999999999999')) +checked expr : is_decimal(CAST("99999999999999999999999999999999999999" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_decimal(parse_json('99999999999999999999999999999999999999999999999999999999999999999999999999991')) +raw expr : is_decimal(parse_json('99999999999999999999999999999999999999999999999999999999999999999999999999991')) +checked expr : is_decimal(CAST("99999999999999999999999999999999999999999999999999999999999999999999999999991" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_string(parse_json('"ab"')) +raw expr : is_string(parse_json('"ab"')) +checked expr : is_string(CAST("\"ab\"" AS Variant)) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + +ast : is_string(parse_json('12.34')) +raw expr : is_string(parse_json('12.34')) +checked expr : is_string(CAST("12.34" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_array(parse_json('[1,2,3]')) +raw expr : is_array(parse_json('[1,2,3]')) +checked expr : is_array(CAST("[1,2,3]" AS Variant)) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + +ast : is_array(parse_json('{"a":"b"}')) +raw expr : is_array(parse_json('{"a":"b"}')) +checked expr : is_array(CAST("{\"a\":\"b\"}" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_object(parse_json('[1,2,3]')) +raw expr : is_object(parse_json('[1,2,3]')) +checked expr : is_object(CAST("[1,2,3]" AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_object(parse_json('{"a":"b"}')) +raw expr : is_object(parse_json('{"a":"b"}')) +checked expr : is_object(CAST("{\"a\":\"b\"}" AS Variant)) +optimized expr : true +output type : Boolean +output domain : {TRUE} +output : true + + +ast : is_binary(to_binary('abcd')::variant) +raw expr : is_binary(CAST(to_binary('abcd') AS Variant)) +checked expr : is_binary(CAST(CAST("abcd" AS Binary) AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_date(to_date('2025-10-11')::variant) +raw expr : is_date(CAST(to_date('2025-10-11') AS Variant)) +checked expr : is_date(CAST(CAST("2025-10-11" AS Date) AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_timestamp(to_timestamp('2025-05-01 10:00:00')::variant) +raw expr : is_timestamp(CAST(to_timestamp('2025-05-01 10:00:00') AS Variant)) +checked expr : is_timestamp(CAST(CAST("2025-05-01 10:00:00" AS Timestamp) AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_interval(to_interval('1 year 2 month')::variant) +raw expr : is_interval(CAST(to_interval('1 year 2 month') AS Variant)) +checked expr : is_interval(CAST(CAST("1 year 2 month" AS Interval) AS Variant)) +optimized expr : false +output type : Boolean +output domain : {FALSE} +output : false + + +ast : is_null_value(parse_json(s)) +raw expr : is_null_value(parse_json(s::String)) +checked expr : is_null_value(CAST(s AS Variant)) +evaluation: ++--------+------------------------------+---------+ +| | s | Output | ++--------+------------------------------+---------+ +| Type | String | Boolean | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | true | +| Row 1 | 'true' | false | +| Row 2 | '123' | false | +| Row 3 | '12.34' | false | +| Row 4 | '"ab"' | false | +| Row 5 | '[1,2,3]' | false | +| Row 6 | '{"a":"b"}' | false | ++--------+------------------------------+---------+ +evaluation (internal): ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0000001]) | ++--------+----------------------------------------------------------------+ + + +ast : is_boolean(parse_json(s)) +raw expr : is_boolean(parse_json(s::String)) +checked expr : is_boolean(CAST(s AS Variant)) +evaluation: ++--------+------------------------------+---------+ +| | s | Output | ++--------+------------------------------+---------+ +| Type | String | Boolean | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | false | +| Row 1 | 'true' | true | +| Row 2 | '123' | false | +| Row 3 | '12.34' | false | +| Row 4 | '"ab"' | false | +| Row 5 | '[1,2,3]' | false | +| Row 6 | '{"a":"b"}' | false | ++--------+------------------------------+---------+ +evaluation (internal): ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0000010]) | ++--------+----------------------------------------------------------------+ + + +ast : is_integer(parse_json(s)) +raw expr : is_integer(parse_json(s::String)) +checked expr : is_integer(CAST(s AS Variant)) +evaluation: ++--------+------------------------------+---------+ +| | s | Output | ++--------+------------------------------+---------+ +| Type | String | Boolean | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | false | +| Row 1 | 'true' | false | +| Row 2 | '123' | true | +| Row 3 | '12.34' | false | +| Row 4 | '"ab"' | false | +| Row 5 | '[1,2,3]' | false | +| Row 6 | '{"a":"b"}' | false | ++--------+------------------------------+---------+ +evaluation (internal): ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0000100]) | ++--------+----------------------------------------------------------------+ + + +ast : is_float(parse_json(s)) +raw expr : is_float(parse_json(s::String)) +checked expr : is_float(CAST(s AS Variant)) +evaluation: ++--------+------------------------------+---------+ +| | s | Output | ++--------+------------------------------+---------+ +| Type | String | Boolean | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | false | +| Row 1 | 'true' | false | +| Row 2 | '123' | true | +| Row 3 | '12.34' | true | +| Row 4 | '"ab"' | false | +| Row 5 | '[1,2,3]' | false | +| Row 6 | '{"a":"b"}' | false | ++--------+------------------------------+---------+ evaluation (internal): -+--------+-----------------------------------------------------------------------------------------+ -| Column | Data | -+--------+-----------------------------------------------------------------------------------------+ -| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | -| Output | NullableColumn { column: Float64([0, 0, 123, 12.34, 0, 0, 0]), validity: [0b_0001100] } | -+--------+-----------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0001100]) | ++--------+----------------------------------------------------------------+ -ast : as_string(parse_json(s)) -raw expr : as_string(parse_json(s::String)) -checked expr : as_string(CAST(s AS Variant)) +ast : is_string(parse_json(s)) +raw expr : is_string(parse_json(s::String)) +checked expr : is_string(CAST(s AS Variant)) evaluation: -+--------+------------------------------+-------------+ -| | s | Output | -+--------+------------------------------+-------------+ -| Type | String | String NULL | -| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | -| Row 0 | 'null' | NULL | -| Row 1 | 'true' | NULL | -| Row 2 | '123' | NULL | -| Row 3 | '12.34' | NULL | -| Row 4 | '"ab"' | 'ab' | -| Row 5 | '[1,2,3]' | NULL | -| Row 6 | '{"a":"b"}' | NULL | -+--------+------------------------------+-------------+ ++--------+------------------------------+---------+ +| | s | Output | ++--------+------------------------------+---------+ +| Type | String | Boolean | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | false | +| Row 1 | 'true' | false | +| Row 2 | '123' | false | +| Row 3 | '12.34' | false | +| Row 4 | '"ab"' | true | +| Row 5 | '[1,2,3]' | false | +| Row 6 | '{"a":"b"}' | false | ++--------+------------------------------+---------+ evaluation (internal): -+--------+---------------------------------------------------------------------------------+ -| Column | Data | -+--------+---------------------------------------------------------------------------------+ -| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | -| Output | NullableColumn { column: StringColumn[, , , , ab, , ], validity: [0b_0010000] } | -+--------+---------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0010000]) | ++--------+----------------------------------------------------------------+ -ast : as_array(parse_json(s)) -raw expr : as_array(parse_json(s::String)) -checked expr : as_array(CAST(s AS Variant)) +ast : is_array(parse_json(s)) +raw expr : is_array(parse_json(s::String)) +checked expr : is_array(CAST(s AS Variant)) evaluation: -+--------+------------------------------+--------------+ -| | s | Output | -+--------+------------------------------+--------------+ -| Type | String | Variant NULL | -| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | -| Row 0 | 'null' | NULL | -| Row 1 | 'true' | NULL | -| Row 2 | '123' | NULL | -| Row 3 | '12.34' | NULL | -| Row 4 | '"ab"' | NULL | -| Row 5 | '[1,2,3]' | '[1,2,3]' | -| Row 6 | '{"a":"b"}' | NULL | -+--------+------------------------------+--------------+ ++--------+------------------------------+---------+ +| | s | Output | ++--------+------------------------------+---------+ +| Type | String | Boolean | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | false | +| Row 1 | 'true' | false | +| Row 2 | '123' | false | +| Row 3 | '12.34' | false | +| Row 4 | '"ab"' | false | +| Row 5 | '[1,2,3]' | true | +| Row 6 | '{"a":"b"}' | false | ++--------+------------------------------+---------+ evaluation (internal): -+--------+--------------------------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+--------------------------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | -| Output | NullableColumn { column: Variant([0x, 0x, 0x, 0x, 0x, 0x80000003200000022000000220000002500150025003, 0x]), validity: [0b_0100000] } | -+--------+--------------------------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_0100000]) | ++--------+----------------------------------------------------------------+ -ast : as_object(parse_json(s)) -raw expr : as_object(parse_json(s::String)) -checked expr : as_object(CAST(s AS Variant)) +ast : is_object(parse_json(s)) +raw expr : is_object(parse_json(s::String)) +checked expr : is_object(CAST(s AS Variant)) evaluation: -+--------+------------------------------+--------------+ -| | s | Output | -+--------+------------------------------+--------------+ -| Type | String | Variant NULL | -| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | -| Row 0 | 'null' | NULL | -| Row 1 | 'true' | NULL | -| Row 2 | '123' | NULL | -| Row 3 | '12.34' | NULL | -| Row 4 | '"ab"' | NULL | -| Row 5 | '[1,2,3]' | NULL | -| Row 6 | '{"a":"b"}' | '{"a":"b"}' | -+--------+------------------------------+--------------+ ++--------+------------------------------+---------+ +| | s | Output | ++--------+------------------------------+---------+ +| Type | String | Boolean | +| Domain | {"\"ab\""..="{\"a\":\"b\"}"} | Unknown | +| Row 0 | 'null' | false | +| Row 1 | 'true' | false | +| Row 2 | '123' | false | +| Row 3 | '12.34' | false | +| Row 4 | '"ab"' | false | +| Row 5 | '[1,2,3]' | false | +| Row 6 | '{"a":"b"}' | true | ++--------+------------------------------+---------+ evaluation (internal): -+--------+----------------------------------------------------------------------------------------------------------------------+ -| Column | Data | -+--------+----------------------------------------------------------------------------------------------------------------------+ -| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | -| Output | NullableColumn { column: Variant([0x, 0x, 0x, 0x, 0x, 0x, 0x4000000110000001100000016162]), validity: [0b_1000000] } | -+--------+----------------------------------------------------------------------------------------------------------------------+ ++--------+----------------------------------------------------------------+ +| Column | Data | ++--------+----------------------------------------------------------------+ +| s | StringColumn[null, true, 123, 12.34, "ab", [1,2,3], {"a":"b"}] | +| Output | Boolean([0b_1000000]) | ++--------+----------------------------------------------------------------+ ast : is_null_value(parse_json('null')) raw expr : is_null_value(parse_json('null')) checked expr : is_null_value(CAST("null" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1341,6 +2528,7 @@ ast : is_null_value(parse_json('[1,2]')) raw expr : is_null_value(parse_json('[1,2]')) checked expr : is_null_value(CAST("[1,2]" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1350,6 +2538,7 @@ ast : is_boolean(parse_json('true')) raw expr : is_boolean(parse_json('true')) checked expr : is_boolean(CAST("true" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1359,6 +2548,7 @@ ast : is_boolean(parse_json('123')) raw expr : is_boolean(parse_json('123')) checked expr : is_boolean(CAST("123" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1368,6 +2558,7 @@ ast : is_integer(parse_json('true')) raw expr : is_integer(parse_json('true')) checked expr : is_integer(CAST("true" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1377,6 +2568,7 @@ ast : is_integer(parse_json('123')) raw expr : is_integer(parse_json('123')) checked expr : is_integer(CAST("123" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1386,6 +2578,7 @@ ast : is_float(parse_json('"ab"')) raw expr : is_float(parse_json('"ab"')) checked expr : is_float(CAST("\"ab\"" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1395,6 +2588,7 @@ ast : is_float(parse_json('12.34')) raw expr : is_float(parse_json('12.34')) checked expr : is_float(CAST("12.34" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1404,6 +2598,7 @@ ast : is_decimal(parse_json('99999999999999999999999999999999999999') raw expr : is_decimal(parse_json('99999999999999999999999999999999999999')) checked expr : is_decimal(CAST("99999999999999999999999999999999999999" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1413,6 +2608,7 @@ ast : is_decimal(parse_json('9999999999999999999999999999999999999999 raw expr : is_decimal(parse_json('99999999999999999999999999999999999999999999999999999999999999999999999999991')) checked expr : is_decimal(CAST("99999999999999999999999999999999999999999999999999999999999999999999999999991" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1422,6 +2618,7 @@ ast : is_string(parse_json('"ab"')) raw expr : is_string(parse_json('"ab"')) checked expr : is_string(CAST("\"ab\"" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1431,6 +2628,7 @@ ast : is_string(parse_json('12.34')) raw expr : is_string(parse_json('12.34')) checked expr : is_string(CAST("12.34" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1440,6 +2638,7 @@ ast : is_array(parse_json('[1,2,3]')) raw expr : is_array(parse_json('[1,2,3]')) checked expr : is_array(CAST("[1,2,3]" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1449,6 +2648,7 @@ ast : is_array(parse_json('{"a":"b"}')) raw expr : is_array(parse_json('{"a":"b"}')) checked expr : is_array(CAST("{\"a\":\"b\"}" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1458,6 +2658,7 @@ ast : is_object(parse_json('[1,2,3]')) raw expr : is_object(parse_json('[1,2,3]')) checked expr : is_object(CAST("[1,2,3]" AS Variant)) optimized expr : false +func ctx : (modified) output type : Boolean output domain : {FALSE} output : false @@ -1467,6 +2668,7 @@ ast : is_object(parse_json('{"a":"b"}')) raw expr : is_object(parse_json('{"a":"b"}')) checked expr : is_object(CAST("{\"a\":\"b\"}" AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1476,6 +2678,7 @@ ast : is_binary(to_binary('abcd')::variant) raw expr : is_binary(CAST(to_binary('abcd') AS Variant)) checked expr : is_binary(CAST(CAST("abcd" AS Binary) AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1485,6 +2688,7 @@ ast : is_date(to_date('2025-10-11')::variant) raw expr : is_date(CAST(to_date('2025-10-11') AS Variant)) checked expr : is_date(CAST(CAST("2025-10-11" AS Date) AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1494,6 +2698,7 @@ ast : is_timestamp(to_timestamp('2025-05-01 10:00:00')::variant) raw expr : is_timestamp(CAST(to_timestamp('2025-05-01 10:00:00') AS Variant)) checked expr : is_timestamp(CAST(CAST("2025-05-01 10:00:00" AS Timestamp) AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1503,6 +2708,7 @@ ast : is_interval(to_interval('1 year 2 month')::variant) raw expr : is_interval(CAST(to_interval('1 year 2 month') AS Variant)) checked expr : is_interval(CAST(CAST("1 year 2 month" AS Interval) AS Variant)) optimized expr : true +func ctx : (modified) output type : Boolean output domain : {TRUE} output : true @@ -1511,6 +2717,7 @@ output : true ast : is_null_value(parse_json(s)) raw expr : is_null_value(parse_json(s::String)) checked expr : is_null_value(CAST(s AS Variant)) +func ctx : (modified) evaluation: +--------+------------------------------+---------+ | | s | Output | @@ -1537,6 +2744,7 @@ evaluation (internal): ast : is_boolean(parse_json(s)) raw expr : is_boolean(parse_json(s::String)) checked expr : is_boolean(CAST(s AS Variant)) +func ctx : (modified) evaluation: +--------+------------------------------+---------+ | | s | Output | @@ -1563,6 +2771,7 @@ evaluation (internal): ast : is_integer(parse_json(s)) raw expr : is_integer(parse_json(s::String)) checked expr : is_integer(CAST(s AS Variant)) +func ctx : (modified) evaluation: +--------+------------------------------+---------+ | | s | Output | @@ -1589,6 +2798,7 @@ evaluation (internal): ast : is_float(parse_json(s)) raw expr : is_float(parse_json(s::String)) checked expr : is_float(CAST(s AS Variant)) +func ctx : (modified) evaluation: +--------+------------------------------+---------+ | | s | Output | @@ -1615,6 +2825,7 @@ evaluation (internal): ast : is_string(parse_json(s)) raw expr : is_string(parse_json(s::String)) checked expr : is_string(CAST(s AS Variant)) +func ctx : (modified) evaluation: +--------+------------------------------+---------+ | | s | Output | @@ -1641,6 +2852,7 @@ evaluation (internal): ast : is_array(parse_json(s)) raw expr : is_array(parse_json(s::String)) checked expr : is_array(CAST(s AS Variant)) +func ctx : (modified) evaluation: +--------+------------------------------+---------+ | | s | Output | @@ -1667,6 +2879,7 @@ evaluation (internal): ast : is_object(parse_json(s)) raw expr : is_object(parse_json(s::String)) checked expr : is_object(CAST(s AS Variant)) +func ctx : (modified) evaluation: +--------+------------------------------+---------+ | | s | Output | @@ -1902,7 +3115,7 @@ error: --> SQL:1:1 | 1 | to_decimal(1, 1)(to_variant(1.23)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow while evaluating function `to_decimal(1, 1)('1.23')` in expr `to_decimal(1, 1)(to_variant(1.23))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow, value `1.23` while evaluating function `to_decimal(1, 1)('1.23')` in expr `to_decimal(1, 1)(to_variant(1.23))` @@ -1928,59 +3141,59 @@ error: --> SQL:1:1 | 1 | to_decimal(3, 3)(to_variant(1.23)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow while evaluating function `to_decimal(3, 3)('1.23')` in expr `to_decimal(3, 3)(to_variant(1.23))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow, value `1.23` while evaluating function `to_decimal(3, 3)('1.23')` in expr `to_decimal(3, 3)(to_variant(1.23))` ast : to_decimal(38, 2)(to_variant(100000000000000000000000000000000000.01)) raw expr : to_decimal(38, 2)(to_variant(100000000000000000000000000000000000.01)) checked expr : to_decimal(38, 2)(to_variant(100000000000000000000000000000000000.01_d128(38,2))) -optimized expr : 100000000000000000000000000000000000.01_d128(38,2) +optimized expr : 99999999999999995387626582021211422.72_d128(38,2) output type : Decimal(38, 2) NULL -output domain : {100000000000000000000000000000000000.01..=100000000000000000000000000000000000.01} -output : 100000000000000000000000000000000000.01 +output domain : {99999999999999995387626582021211422.72..=99999999999999995387626582021211422.72} +output : 99999999999999995387626582021211422.72 ast : to_decimal(39, 2)(to_variant(1000000000000000000000000000000000000.01)) raw expr : to_decimal(39, 2)(to_variant(1000000000000000000000000000000000000.01)) checked expr : to_decimal(39, 2)(to_variant(1000000000000000000000000000000000000.01_d256(39,2))) -optimized expr : 1000000000000000000000000000000000000.01_d256(39,2) +optimized expr : 999999999999999977488098234560340295.68_d256(39,2) output type : Decimal(39, 2) NULL -output domain : {1000000000000000000000000000000000000.01..=1000000000000000000000000000000000000.01} -output : 1000000000000000000000000000000000000.01 +output domain : {999999999999999977488098234560340295.68..=999999999999999977488098234560340295.68} +output : 999999999999999977488098234560340295.68 ast : to_decimal(38, 1)(to_variant(1000000000000000000000000000000000000.01)) raw expr : to_decimal(38, 1)(to_variant(1000000000000000000000000000000000000.01)) checked expr : to_decimal(38, 1)(to_variant(1000000000000000000000000000000000000.01_d256(39,2))) -optimized expr : 1000000000000000000000000000000000000.0_d128(38,1) +optimized expr : 1000000000000000071935427891953244569.6_d128(38,1) output type : Decimal(38, 1) NULL -output domain : {1000000000000000000000000000000000000.0..=1000000000000000000000000000000000000.0} -output : 1000000000000000000000000000000000000.0 +output domain : {1000000000000000071935427891953244569.6..=1000000000000000071935427891953244569.6} +output : 1000000000000000071935427891953244569.6 error: --> SQL:1:1 | 1 | to_decimal(38, 3)(to_variant(1000000000000000000000000000000000000.01)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow while evaluating function `to_decimal(38, 3)('1000000000000000000000000000000000000.01')` in expr `to_decimal(38, 3)(to_variant(1000000000000000000000000000000000000.01))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow, value `1000000000000000000000000000000000000` while evaluating function `to_decimal(38, 3)('1e36')` in expr `to_decimal(38, 3)(to_variant(1000000000000000000000000000000000000.01))` ast : to_decimal(39, 1)(to_variant(1000000000000000000000000000000000000.01)) raw expr : to_decimal(39, 1)(to_variant(1000000000000000000000000000000000000.01)) checked expr : to_decimal(39, 1)(to_variant(1000000000000000000000000000000000000.01_d256(39,2))) -optimized expr : 1000000000000000000000000000000000000.0_d256(39,1) +optimized expr : 1000000000000000071935427891953244569.6_d256(39,1) output type : Decimal(39, 1) NULL -output domain : {1000000000000000000000000000000000000.0..=1000000000000000000000000000000000000.0} -output : 1000000000000000000000000000000000000.0 +output domain : {1000000000000000071935427891953244569.6..=1000000000000000071935427891953244569.6} +output : 1000000000000000071935427891953244569.6 error: --> SQL:1:1 | 1 | to_decimal(39, 3)(to_variant(1000000000000000000000000000000000000.01)) - | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow while evaluating function `to_decimal(39, 3)('1000000000000000000000000000000000000.01')` in expr `to_decimal(39, 3)(to_variant(1000000000000000000000000000000000000.01))` + | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ Decimal overflow, value `1000000000000000000000000000000000000` while evaluating function `to_decimal(39, 3)('1e36')` in expr `to_decimal(39, 3)(to_variant(1000000000000000000000000000000000000.01))` @@ -3202,10 +4415,97 @@ output domain : {"INTEGER"..="INTEGER"} output : 'INTEGER' +ast : json_typeof(parse_json('-1.12')) +raw expr : json_typeof(parse_json('-1.12')) +checked expr : json_typeof(CAST("-1.12" AS Variant)) +optimized expr : "DOUBLE" +output type : String +output domain : {"DOUBLE"..="DOUBLE"} +output : 'DOUBLE' + + +ast : json_typeof(parse_json('1.12e10')) +raw expr : json_typeof(parse_json('1.12e10')) +checked expr : json_typeof(CAST("1.12e10" AS Variant)) +optimized expr : "DOUBLE" +output type : String +output domain : {"DOUBLE"..="DOUBLE"} +output : 'DOUBLE' + + +ast : json_typeof(parse_json('[1,2,3]')) +raw expr : json_typeof(parse_json('[1,2,3]')) +checked expr : json_typeof(CAST("[1,2,3]" AS Variant)) +optimized expr : "ARRAY" +output type : String +output domain : {"ARRAY"..="ARRAY"} +output : 'ARRAY' + + +ast : json_typeof(parse_json('{"a":1,"b":2}')) +raw expr : json_typeof(parse_json('{"a":1,"b":2}')) +checked expr : json_typeof(CAST("{\"a\":1,\"b\":2}" AS Variant)) +optimized expr : "OBJECT" +output type : String +output domain : {"OBJECT"..="OBJECT"} +output : 'OBJECT' + + +ast : json_typeof(NULL) +raw expr : json_typeof(NULL) +checked expr : json_typeof(CAST(NULL AS Variant NULL)) +optimized expr : NULL +func ctx : (modified) +output type : String NULL +output domain : {NULL} +output : NULL + + +ast : json_typeof(parse_json('null')) +raw expr : json_typeof(parse_json('null')) +checked expr : json_typeof(CAST("null" AS Variant)) +optimized expr : "NULL_VALUE" +func ctx : (modified) +output type : String +output domain : {"NULL_VALUE"..="NULL_VALUE"} +output : 'NULL_VALUE' + + +ast : json_typeof(parse_json('true')) +raw expr : json_typeof(parse_json('true')) +checked expr : json_typeof(CAST("true" AS Variant)) +optimized expr : "BOOLEAN" +func ctx : (modified) +output type : String +output domain : {"BOOLEAN"..="BOOLEAN"} +output : 'BOOLEAN' + + +ast : json_typeof(parse_json('"test"')) +raw expr : json_typeof(parse_json('"test"')) +checked expr : json_typeof(CAST("\"test\"" AS Variant)) +optimized expr : "STRING" +func ctx : (modified) +output type : String +output domain : {"STRING"..="STRING"} +output : 'STRING' + + +ast : json_typeof(parse_json('123')) +raw expr : json_typeof(parse_json('123')) +checked expr : json_typeof(CAST("123" AS Variant)) +optimized expr : "INTEGER" +func ctx : (modified) +output type : String +output domain : {"INTEGER"..="INTEGER"} +output : 'INTEGER' + + ast : json_typeof(parse_json('-1.12')) raw expr : json_typeof(parse_json('-1.12')) checked expr : json_typeof(CAST("-1.12" AS Variant)) optimized expr : "DECIMAL" +func ctx : (modified) output type : String output domain : {"DECIMAL"..="DECIMAL"} output : 'DECIMAL' @@ -3215,6 +4515,7 @@ ast : json_typeof(parse_json('1.12e10')) raw expr : json_typeof(parse_json('1.12e10')) checked expr : json_typeof(CAST("1.12e10" AS Variant)) optimized expr : "DOUBLE" +func ctx : (modified) output type : String output domain : {"DOUBLE"..="DOUBLE"} output : 'DOUBLE' @@ -3224,6 +4525,7 @@ ast : json_typeof(parse_json('[1,2,3]')) raw expr : json_typeof(parse_json('[1,2,3]')) checked expr : json_typeof(CAST("[1,2,3]" AS Variant)) optimized expr : "ARRAY" +func ctx : (modified) output type : String output domain : {"ARRAY"..="ARRAY"} output : 'ARRAY' @@ -3233,6 +4535,7 @@ ast : json_typeof(parse_json('{"a":1,"b":2}')) raw expr : json_typeof(parse_json('{"a":1,"b":2}')) checked expr : json_typeof(CAST("{\"a\":1,\"b\":2}" AS Variant)) optimized expr : "OBJECT" +func ctx : (modified) output type : String output domain : {"OBJECT"..="OBJECT"} output : 'OBJECT' diff --git a/src/query/functions/tests/it/scalars/variant.rs b/src/query/functions/tests/it/scalars/variant.rs index e886a9e428354..8bb8aad269935 100644 --- a/src/query/functions/tests/it/scalars/variant.rs +++ b/src/query/functions/tests/it/scalars/variant.rs @@ -16,17 +16,22 @@ use std::io::Write; use databend_common_expression::types::*; use databend_common_expression::FromData; +use databend_common_expression::FunctionContext; use goldenfile::Mint; use super::run_ast; +use super::run_ast_with_context; +use super::TestContext; #[test] fn test_variant() { let mut mint = Mint::new("tests/it/scalars/testdata"); let file = &mut mint.new_goldenfile("variant.txt").unwrap(); - test_parse_json(file); - test_try_parse_json(file); + test_parse_json(file, false, false); + test_parse_json(file, false, true); + test_parse_json(file, true, false); + test_parse_json(file, true, true); test_check_json(file); test_length(file); test_object_keys(file); @@ -34,8 +39,10 @@ fn test_variant() { test_get_ignore_case(file); test_get_path(file); test_json_extract_path_text(file); - test_as_type(file); - test_is_type(file); + test_as_type(file, false); + test_as_type(file, true); + test_is_type(file, false); + test_is_type(file, true); test_to_type(file); test_try_to_type(file); test_object_construct(file); @@ -45,7 +52,8 @@ fn test_variant() { test_json_to_string(file); test_json_pretty(file); test_json_strip_nulls(file); - test_json_typeof(file); + test_json_typeof(file, false); + test_json_typeof(file, true); test_array_construct(file); test_json_path_exists(file); test_get_arrow_op(file); @@ -87,87 +95,113 @@ fn test_variant() { test_array_slice(file); } -fn test_parse_json(file: &mut impl Write) { - run_ast(file, "parse_json(NULL)", &[]); - run_ast(file, "parse_json('nuLL')", &[]); - run_ast(file, "parse_json('null')", &[]); - run_ast(file, "parse_json(' ')", &[]); - run_ast(file, "parse_json('true')", &[]); - run_ast(file, "parse_json('false')", &[]); - run_ast(file, "parse_json('\"测试\"')", &[]); - run_ast(file, "parse_json('1234')", &[]); - run_ast(file, "parse_json('[1,2,3,4]')", &[]); - run_ast(file, "parse_json('{\"a\":\"b\",\"c\":\"d\"}')", &[]); - - run_ast(file, "parse_json(s)", &[( - "s", - StringType::from_data(vec![ - r#"null"#, - r#"true"#, - r#"9223372036854775807"#, - r#"-32768"#, - r#"1234.5678"#, - r#"1.912e2"#, - r#""\\\"abc\\\"""#, - r#""databend""#, - r#"{"k":"v","a":"b"}"#, - r#"[1,2,3,["a","b","c"]]"#, - ]), - )]); +fn test_parse_json(file: &mut impl Write, is_try: bool, enable_extended_json_syntax: bool) { + let prefix = if is_try { "try_" } else { "" }; + let func_ctx = FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }; + let test_ctx = TestContext { + func_ctx: func_ctx.clone(), + ..TestContext::default() + }; + run_ast_with_context(file, format!("{prefix}parse_json(NULL)"), test_ctx.clone()); + run_ast_with_context( + file, + format!("{prefix}parse_json('nuLL')"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}parse_json('null')"), + test_ctx.clone(), + ); + run_ast_with_context(file, format!("{prefix}parse_json(' ')"), test_ctx.clone()); + run_ast_with_context( + file, + format!("{prefix}parse_json('true')"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}parse_json('false')"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}parse_json('\"测试\"')"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}parse_json('1234')"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}parse_json('[1,2,3,4]')"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}parse_json('{{\"a\":\"b\",\"c\":\"d\"}}')"), + test_ctx.clone(), + ); - run_ast(file, "parse_json(s)", &[( - "s", - StringType::from_data_with_validity(vec!["true", "false", "", "1234"], vec![ - true, true, false, true, - ]), - )]); + let test_ctx_with_column = TestContext { + columns: &[( + "s", + StringType::from_data(vec![ + r#"null"#, + r#"true"#, + r#"9223372036854775807"#, + r#"-32768"#, + r#"1234.5678"#, + r#"1.912e2"#, + r#""\\\"abc\\\"""#, + r#""databend""#, + r#"{"k":"v","a":"b"}"#, + r#"[1,2,3,["a","b","c"]]"#, + ]), + )], + func_ctx: FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }, + ..TestContext::default() + }; + run_ast_with_context(file, format!("{prefix}parse_json(s)"), test_ctx_with_column); + + let test_ctx_with_column = TestContext { + columns: &[( + "s", + StringType::from_data_with_validity(vec!["true", "false", "", "1234"], vec![ + true, true, false, true, + ]), + )], + func_ctx: FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }, + ..TestContext::default() + }; + run_ast_with_context(file, format!("{prefix}parse_json(s)"), test_ctx_with_column); // json extension syntax - run_ast(file, "parse_json('+10')", &[]); - run_ast(file, "parse_json('001')", &[]); - run_ast(file, "parse_json('.12')", &[]); - run_ast(file, "parse_json('12.')", &[]); - run_ast( + run_ast_with_context(file, format!("{prefix}parse_json('+10')"), test_ctx.clone()); + run_ast_with_context(file, format!("{prefix}parse_json('001')"), test_ctx.clone()); + run_ast_with_context(file, format!("{prefix}parse_json('.12')"), test_ctx.clone()); + run_ast_with_context(file, format!("{prefix}parse_json('12.')"), test_ctx.clone()); + run_ast_with_context( file, - "parse_json('99999999999999999999999999999999999999')", - &[], + format!("{prefix}parse_json('99999999999999999999999999999999999999')"), + test_ctx.clone(), + ); + run_ast_with_context( + file, + format!("{prefix}parse_json('[1,2,,4]')"), + test_ctx.clone(), ); - run_ast(file, "parse_json('[1,2,,4]')", &[]); -} - -fn test_try_parse_json(file: &mut impl Write) { - run_ast(file, "try_parse_json(NULL)", &[]); - run_ast(file, "try_parse_json('nuLL')", &[]); - run_ast(file, "try_parse_json('null')", &[]); - run_ast(file, "try_parse_json('true')", &[]); - run_ast(file, "try_parse_json('false')", &[]); - run_ast(file, "try_parse_json('\"测试\"')", &[]); - run_ast(file, "try_parse_json('1234')", &[]); - run_ast(file, "try_parse_json('[1,2,3,4]')", &[]); - run_ast(file, "try_parse_json('{\"a\":\"b\",\"c\":\"d\"}')", &[]); - - run_ast(file, "try_parse_json(s)", &[( - "s", - StringType::from_data(vec![ - r#"null"#, - r#"true"#, - r#"9223372036854775807"#, - r#"-32768"#, - r#"1234.5678"#, - r#"1.912e2"#, - r#""\\\"abc\\\"""#, - r#""databend""#, - r#"{"k":"v","a":"b"}"#, - r#"[1,2,3,["a","b","c"]]"#, - ]), - )]); - - run_ast(file, "try_parse_json(s)", &[( - "s", - StringType::from_data_with_validity(vec!["true", "ttt", "", "1234"], vec![ - true, true, false, true, - ]), - )]); } fn test_check_json(file: &mut impl Write) { @@ -521,107 +555,217 @@ fn test_json_extract_path_text(file: &mut impl Write) { ]); } -fn test_as_type(file: &mut impl Write) { - run_ast(file, "as_boolean(parse_json('true'))", &[]); - run_ast(file, "as_boolean(parse_json('123'))", &[]); - run_ast(file, "as_integer(parse_json('true'))", &[]); - run_ast(file, "as_integer(parse_json('123'))", &[]); - run_ast(file, "as_float(parse_json('\"ab\"'))", &[]); - run_ast(file, "as_float(parse_json('12.34'))", &[]); - run_ast(file, "as_decimal(parse_json('12.34'))", &[]); - run_ast(file, "as_decimal(10, 2)(parse_json('12.34'))", &[]); - run_ast(file, "as_string(parse_json('\"ab\"'))", &[]); - run_ast(file, "as_string(parse_json('12.34'))", &[]); - run_ast(file, "as_array(parse_json('[1,2,3]'))", &[]); - run_ast(file, "as_array(parse_json('{\"a\":\"b\"}'))", &[]); - run_ast(file, "as_object(parse_json('[1,2,3]'))", &[]); - run_ast(file, "as_object(parse_json('{\"a\":\"b\"}'))", &[]); - run_ast(file, "as_binary(to_binary('abcd')::variant)", &[]); - run_ast(file, "as_date(to_date('2025-10-11')::variant)", &[]); - run_ast( +fn test_as_type(file: &mut impl Write, enable_extended_json_syntax: bool) { + let func_ctx = FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }; + let test_ctx = TestContext { + func_ctx: func_ctx.clone(), + ..TestContext::default() + }; + run_ast_with_context(file, "as_boolean(parse_json('true'))", test_ctx.clone()); + run_ast_with_context(file, "as_boolean(parse_json('123'))", test_ctx.clone()); + run_ast_with_context(file, "as_integer(parse_json('true'))", test_ctx.clone()); + run_ast_with_context(file, "as_integer(parse_json('123'))", test_ctx.clone()); + run_ast_with_context(file, "as_float(parse_json('\"ab\"'))", test_ctx.clone()); + run_ast_with_context(file, "as_float(parse_json('12.34'))", test_ctx.clone()); + run_ast_with_context(file, "as_decimal(parse_json('12.34'))", test_ctx.clone()); + run_ast_with_context( + file, + "as_decimal(10, 2)(parse_json('12.34'))", + test_ctx.clone(), + ); + run_ast_with_context(file, "as_string(parse_json('\"ab\"'))", test_ctx.clone()); + run_ast_with_context(file, "as_string(parse_json('12.34'))", test_ctx.clone()); + run_ast_with_context(file, "as_array(parse_json('[1,2,3]'))", test_ctx.clone()); + run_ast_with_context( + file, + "as_array(parse_json('{\"a\":\"b\"}'))", + test_ctx.clone(), + ); + run_ast_with_context(file, "as_object(parse_json('[1,2,3]'))", test_ctx.clone()); + run_ast_with_context( + file, + "as_object(parse_json('{\"a\":\"b\"}'))", + test_ctx.clone(), + ); + run_ast_with_context( + file, + "as_binary(to_binary('abcd')::variant)", + test_ctx.clone(), + ); + run_ast_with_context( + file, + "as_date(to_date('2025-10-11')::variant)", + test_ctx.clone(), + ); + run_ast_with_context( file, "as_timestamp(to_timestamp('2025-05-01 10:00:00')::variant)", - &[], + test_ctx.clone(), ); - run_ast( + run_ast_with_context( file, "as_interval(to_interval('1 year 2 month')::variant)", - &[], + test_ctx.clone(), ); - let columns = &[( - "s", - StringType::from_data(vec![ - "null", - "true", - "123", - "12.34", - "\"ab\"", - "[1,2,3]", - "{\"a\":\"b\"}", - ]), - )]; - run_ast(file, "as_boolean(parse_json(s))", columns); - run_ast(file, "as_integer(parse_json(s))", columns); - run_ast(file, "as_float(parse_json(s))", columns); - run_ast(file, "as_string(parse_json(s))", columns); - run_ast(file, "as_array(parse_json(s))", columns); - run_ast(file, "as_object(parse_json(s))", columns); + let test_ctx_with_column = TestContext { + columns: &[( + "s", + StringType::from_data(vec![ + "null", + "true", + "123", + "12.34", + "\"ab\"", + "[1,2,3]", + "{\"a\":\"b\"}", + ]), + )], + func_ctx: FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }, + ..TestContext::default() + }; + run_ast_with_context( + file, + "as_boolean(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "as_integer(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "as_float(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "as_string(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "as_array(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context(file, "as_object(parse_json(s))", test_ctx_with_column); } -fn test_is_type(file: &mut impl Write) { - run_ast(file, "is_null_value(parse_json('null'))", &[]); - run_ast(file, "is_null_value(parse_json('[1,2]'))", &[]); - run_ast(file, "is_boolean(parse_json('true'))", &[]); - run_ast(file, "is_boolean(parse_json('123'))", &[]); - run_ast(file, "is_integer(parse_json('true'))", &[]); - run_ast(file, "is_integer(parse_json('123'))", &[]); - run_ast(file, "is_float(parse_json('\"ab\"'))", &[]); - run_ast(file, "is_float(parse_json('12.34'))", &[]); - run_ast( +fn test_is_type(file: &mut impl Write, enable_extended_json_syntax: bool) { + let func_ctx = FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }; + let test_ctx = TestContext { + func_ctx: func_ctx.clone(), + ..TestContext::default() + }; + run_ast_with_context(file, "is_null_value(parse_json('null'))", test_ctx.clone()); + run_ast_with_context(file, "is_null_value(parse_json('[1,2]'))", test_ctx.clone()); + run_ast_with_context(file, "is_boolean(parse_json('true'))", test_ctx.clone()); + run_ast_with_context(file, "is_boolean(parse_json('123'))", test_ctx.clone()); + run_ast_with_context(file, "is_integer(parse_json('true'))", test_ctx.clone()); + run_ast_with_context(file, "is_integer(parse_json('123'))", test_ctx.clone()); + run_ast_with_context(file, "is_float(parse_json('\"ab\"'))", test_ctx.clone()); + run_ast_with_context(file, "is_float(parse_json('12.34'))", test_ctx.clone()); + run_ast_with_context( file, "is_decimal(parse_json('99999999999999999999999999999999999999'))", - &[], + test_ctx.clone(), ); - run_ast(file, "is_decimal(parse_json('99999999999999999999999999999999999999999999999999999999999999999999999999991'))", &[]); - run_ast(file, "is_string(parse_json('\"ab\"'))", &[]); - run_ast(file, "is_string(parse_json('12.34'))", &[]); - run_ast(file, "is_array(parse_json('[1,2,3]'))", &[]); - run_ast(file, "is_array(parse_json('{\"a\":\"b\"}'))", &[]); - run_ast(file, "is_object(parse_json('[1,2,3]'))", &[]); - run_ast(file, "is_object(parse_json('{\"a\":\"b\"}'))", &[]); - run_ast(file, "is_binary(to_binary('abcd')::variant)", &[]); - run_ast(file, "is_date(to_date('2025-10-11')::variant)", &[]); - run_ast( + run_ast_with_context(file, "is_decimal(parse_json('99999999999999999999999999999999999999999999999999999999999999999999999999991'))", test_ctx.clone()); + run_ast_with_context(file, "is_string(parse_json('\"ab\"'))", test_ctx.clone()); + run_ast_with_context(file, "is_string(parse_json('12.34'))", test_ctx.clone()); + run_ast_with_context(file, "is_array(parse_json('[1,2,3]'))", test_ctx.clone()); + run_ast_with_context( + file, + "is_array(parse_json('{\"a\":\"b\"}'))", + test_ctx.clone(), + ); + run_ast_with_context(file, "is_object(parse_json('[1,2,3]'))", test_ctx.clone()); + run_ast_with_context( + file, + "is_object(parse_json('{\"a\":\"b\"}'))", + test_ctx.clone(), + ); + run_ast_with_context( + file, + "is_binary(to_binary('abcd')::variant)", + test_ctx.clone(), + ); + run_ast_with_context( + file, + "is_date(to_date('2025-10-11')::variant)", + test_ctx.clone(), + ); + run_ast_with_context( file, "is_timestamp(to_timestamp('2025-05-01 10:00:00')::variant)", - &[], + test_ctx.clone(), ); - run_ast( + run_ast_with_context( file, "is_interval(to_interval('1 year 2 month')::variant)", - &[], + test_ctx.clone(), ); - let columns = &[( - "s", - StringType::from_data(vec![ - "null", - "true", - "123", - "12.34", - "\"ab\"", - "[1,2,3]", - "{\"a\":\"b\"}", - ]), - )]; - run_ast(file, "is_null_value(parse_json(s))", columns); - run_ast(file, "is_boolean(parse_json(s))", columns); - run_ast(file, "is_integer(parse_json(s))", columns); - run_ast(file, "is_float(parse_json(s))", columns); - run_ast(file, "is_string(parse_json(s))", columns); - run_ast(file, "is_array(parse_json(s))", columns); - run_ast(file, "is_object(parse_json(s))", columns); + let test_ctx_with_column = TestContext { + columns: &[( + "s", + StringType::from_data(vec![ + "null", + "true", + "123", + "12.34", + "\"ab\"", + "[1,2,3]", + "{\"a\":\"b\"}", + ]), + )], + func_ctx: FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }, + ..TestContext::default() + }; + run_ast_with_context( + file, + "is_null_value(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "is_boolean(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "is_integer(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "is_float(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "is_string(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context( + file, + "is_array(parse_json(s))", + test_ctx_with_column.clone(), + ); + run_ast_with_context(file, "is_object(parse_json(s))", test_ctx_with_column); } fn test_to_type(file: &mut impl Write) { @@ -1097,16 +1241,44 @@ fn test_json_strip_nulls(file: &mut impl Write) { ); } -fn test_json_typeof(file: &mut impl Write) { - run_ast(file, r#"json_typeof(NULL)"#, &[]); - run_ast(file, r#"json_typeof(parse_json('null'))"#, &[]); - run_ast(file, r#"json_typeof(parse_json('true'))"#, &[]); - run_ast(file, r#"json_typeof(parse_json('"test"'))"#, &[]); - run_ast(file, r#"json_typeof(parse_json('123'))"#, &[]); - run_ast(file, r#"json_typeof(parse_json('-1.12'))"#, &[]); - run_ast(file, r#"json_typeof(parse_json('1.12e10'))"#, &[]); - run_ast(file, r#"json_typeof(parse_json('[1,2,3]'))"#, &[]); - run_ast(file, r#"json_typeof(parse_json('{"a":1,"b":2}'))"#, &[]); +fn test_json_typeof(file: &mut impl Write, enable_extended_json_syntax: bool) { + let func_ctx = FunctionContext { + enable_extended_json_syntax, + ..FunctionContext::default() + }; + let test_ctx = TestContext { + func_ctx: func_ctx.clone(), + ..TestContext::default() + }; + run_ast_with_context(file, r#"json_typeof(NULL)"#, test_ctx.clone()); + run_ast_with_context(file, r#"json_typeof(parse_json('null'))"#, test_ctx.clone()); + run_ast_with_context(file, r#"json_typeof(parse_json('true'))"#, test_ctx.clone()); + run_ast_with_context( + file, + r#"json_typeof(parse_json('"test"'))"#, + test_ctx.clone(), + ); + run_ast_with_context(file, r#"json_typeof(parse_json('123'))"#, test_ctx.clone()); + run_ast_with_context( + file, + r#"json_typeof(parse_json('-1.12'))"#, + test_ctx.clone(), + ); + run_ast_with_context( + file, + r#"json_typeof(parse_json('1.12e10'))"#, + test_ctx.clone(), + ); + run_ast_with_context( + file, + r#"json_typeof(parse_json('[1,2,3]'))"#, + test_ctx.clone(), + ); + run_ast_with_context( + file, + r#"json_typeof(parse_json('{"a":1,"b":2}'))"#, + test_ctx.clone(), + ); } fn test_array_construct(file: &mut impl Write) { diff --git a/src/query/service/src/sessions/query_ctx.rs b/src/query/service/src/sessions/query_ctx.rs index 7601e94dbceb4..14c8c77e05f28 100644 --- a/src/query/service/src/sessions/query_ctx.rs +++ b/src/query/service/src/sessions/query_ctx.rs @@ -947,12 +947,14 @@ impl TableContext for QueryContext { let geometry_format = self.get_settings().get_geometry_output_format()?; let format_null_as_str = self.get_settings().get_format_null_as_str()?; let enable_dst_hour_fix = self.get_settings().get_enable_dst_hour_fix()?; + let enable_extended_json_syntax = self.get_settings().get_enable_extended_json_syntax()?; let format = FormatSettings { timezone, jiff_timezone, geometry_format, enable_dst_hour_fix, format_null_as_str, + enable_extended_json_syntax, }; Ok(format) } @@ -979,6 +981,7 @@ impl TableContext for QueryContext { let geometry_output_format = settings.get_geometry_output_format()?; let parse_datetime_ignore_remainder = settings.get_parse_datetime_ignore_remainder()?; let enable_strict_datetime_parser = settings.get_enable_strict_datetime_parser()?; + let enable_extended_json_syntax = settings.get_enable_extended_json_syntax()?; let week_start = settings.get_week_start()? as u8; let date_format_style = settings.get_date_format_style()?; let query_config = &GlobalConfig::instance().query; @@ -1000,6 +1003,7 @@ impl TableContext for QueryContext { geometry_output_format, parse_datetime_ignore_remainder, enable_strict_datetime_parser, + enable_extended_json_syntax, random_function_seed, week_start, date_format_style, diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index 9b32bbbc233fd..e5389e2126747 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -1361,6 +1361,13 @@ impl DefaultSettings { scope: SettingScope::Both, range: Some(SettingRange::Numeric(0..=1)), }), + ("enable_extended_json_syntax", DefaultSettingValue { + value: UserSettingValue::UInt64(0), + desc: "Set to 1 for entended JSON syntax, support decimal values.", + mode: SettingMode::Both, + scope: SettingScope::Both, + range: Some(SettingRange::Numeric(0..=1)), + }), ]); Ok(Arc::new(DefaultSettings { diff --git a/src/query/settings/src/settings_getter_setter.rs b/src/query/settings/src/settings_getter_setter.rs index 0e9a7617a1ac9..4442f73ee2087 100644 --- a/src/query/settings/src/settings_getter_setter.rs +++ b/src/query/settings/src/settings_getter_setter.rs @@ -1013,4 +1013,8 @@ impl Settings { pub fn get_enable_parallel_union_all(&self) -> Result { Ok(self.try_get_u64("enable_parallel_union_all")? == 1) } + + pub fn get_enable_extended_json_syntax(&self) -> Result { + Ok(self.try_get_u64("enable_extended_json_syntax")? == 1) + } } diff --git a/src/query/storages/parquet/src/parquet_variant_table/recordbatch_to_variant.rs b/src/query/storages/parquet/src/parquet_variant_table/recordbatch_to_variant.rs index 123861e3fda99..5d692cc714fa7 100644 --- a/src/query/storages/parquet/src/parquet_variant_table/recordbatch_to_variant.rs +++ b/src/query/storages/parquet/src/parquet_variant_table/recordbatch_to_variant.rs @@ -25,6 +25,7 @@ pub fn read_record_batch( record_batch: RecordBatch, builder: &mut BinaryColumnBuilder, tz: &TimeZone, + enable_extended_json_syntax: bool, typ: &TableDataType, schema: &DataSchema, ) -> databend_common_exception::Result<()> { @@ -34,7 +35,13 @@ pub fn read_record_batch( } let column = Column::Tuple(columns); for scalar in column.iter() { - cast_scalar_to_variant(scalar, tz, &mut builder.data, Some(typ)); + cast_scalar_to_variant( + scalar, + tz, + enable_extended_json_syntax, + &mut builder.data, + Some(typ), + ); builder.commit_row() } Ok(()) @@ -43,6 +50,7 @@ pub fn read_record_batch( pub fn record_batch_to_block( record_batch: RecordBatch, tz: &TimeZone, + enable_extended_json_syntax: bool, typ: &TableDataType, schema: &DataSchema, ) -> databend_common_exception::Result { @@ -50,7 +58,14 @@ pub fn record_batch_to_block( record_batch.num_rows(), record_batch.get_array_memory_size(), ); - read_record_batch(record_batch, &mut builder, tz, typ, schema)?; + read_record_batch( + record_batch, + &mut builder, + tz, + enable_extended_json_syntax, + typ, + schema, + )?; let column = builder.build(); let num_rows = column.len(); Ok(DataBlock::new( diff --git a/src/query/storages/parquet/src/parquet_variant_table/source.rs b/src/query/storages/parquet/src/parquet_variant_table/source.rs index 25872e424d046..8ab2e27a5906f 100644 --- a/src/query/storages/parquet/src/parquet_variant_table/source.rs +++ b/src/query/storages/parquet/src/parquet_variant_table/source.rs @@ -91,6 +91,7 @@ pub struct ParquetVariantSource { use_logic_type: bool, tz: TimeZone, + enable_extended_json_syntax: bool, } impl ParquetVariantSource { @@ -108,6 +109,7 @@ impl ParquetVariantSource { let settings = ctx.get_settings(); let tz_string = settings.get_timezone()?; + let enable_extended_json_syntax = settings.get_enable_extended_json_syntax()?; let tz = TimeZone::get(&tz_string).map_err(|e| { ErrorCode::InvalidTimezone(format!("[QUERY-CTX] Timezone validation failed: {}", e)) })?; @@ -125,6 +127,7 @@ impl ParquetVariantSource { op_registry, batch_size: 1000, tz, + enable_extended_json_syntax, use_logic_type, }))) } @@ -184,7 +187,13 @@ impl Processor for ParquetVariantSource { } => { if let Some((reader, mut start_row, typ, data_schema)) = vs.front_mut() { if let Some(batch) = reader.next() { - let mut block = record_batch_to_block(batch?, &self.tz, typ, data_schema)?; + let mut block = record_batch_to_block( + batch?, + &self.tz, + self.enable_extended_json_syntax, + typ, + data_schema, + )?; add_internal_columns( &self.internal_columns, location.clone(), @@ -212,8 +221,13 @@ impl Processor for ParquetVariantSource { State::ReadFiles(buffers) => { let mut blocks = Vec::with_capacity(buffers.len()); for (buffer, path) in buffers { - let mut block = - read_small_file(buffer, self.batch_size, &self.tz, self.use_logic_type)?; + let mut block = read_small_file( + buffer, + self.batch_size, + &self.tz, + self.enable_extended_json_syntax, + self.use_logic_type, + )?; if self.is_copy { self.copy_status.add_chunk(path.as_str(), FileStatus { @@ -352,6 +366,7 @@ pub fn read_small_file( bytes: Bytes, batch_size: usize, tz: &TimeZone, + enable_extended_json_syntax: bool, use_logic_type: bool, ) -> databend_common_exception::Result { let len = bytes.len(); @@ -367,7 +382,14 @@ pub fn read_small_file( let mut builder = BinaryColumnBuilder::with_capacity(batch_size, len); for batch in reader { let batch = batch?; - read_record_batch(batch, &mut builder, tz, &typ, &data_schema)?; + read_record_batch( + batch, + &mut builder, + tz, + enable_extended_json_syntax, + &typ, + &data_schema, + )?; } let column = builder.build(); let num_rows = column.len(); diff --git a/tests/sqllogictests/suites/query/functions/02_0000_function_aggregate_mix.test b/tests/sqllogictests/suites/query/functions/02_0000_function_aggregate_mix.test index d7c0ce50583ec..de91c9955918c 100644 --- a/tests/sqllogictests/suites/query/functions/02_0000_function_aggregate_mix.test +++ b/tests/sqllogictests/suites/query/functions/02_0000_function_aggregate_mix.test @@ -505,12 +505,12 @@ NULL query TTTTT select json_array_agg(a), json_array_agg(b), json_array_agg(c), json_array_agg(d), json_array_agg(e), json_array_agg('a') from d ---- -[20.00,10.00,4.23,5.99] ["abc","de","xyz"] [100,200,300] [{"k":"v"},null,"uvw",[1,2,3]] [["a","b"],[],["x","y"],["z"]] ["a","a","a","a"] +[20.0,10.0,4.23,5.99] ["abc","de","xyz"] [100,200,300] [{"k":"v"},null,"uvw",[1,2,3]] [["a","b"],[],["x","y"],["z"]] ["a","a","a","a"] query TTTTT select json_object_agg(b, a), json_object_agg(b, c), json_object_agg(b, d), json_object_agg(b, e), json_object_agg(b, 'a') from d ---- -{"abc":20.00,"de":10.00,"xyz":5.99} {"de":100,"xyz":300} {"abc":{"k":"v"},"de":null,"xyz":[1,2,3]} {"abc":["a","b"],"de":[],"xyz":["z"]} {"abc":"a","de":"a","xyz":"a"} +{"abc":20.0,"de":10.0,"xyz":5.99} {"de":100,"xyz":300} {"abc":{"k":"v"},"de":null,"xyz":[1,2,3]} {"abc":["a","b"],"de":[],"xyz":["z"]} {"abc":"a","de":"a","xyz":"a"} query TTTTT select array_agg(a), array_agg(b), array_agg(c), array_agg(d), array_agg(e), array_agg('a') from d diff --git a/tests/sqllogictests/suites/query/functions/02_0048_function_semi_structureds_parse_json.test b/tests/sqllogictests/suites/query/functions/02_0048_function_semi_structureds_parse_json.test index 164644171fa2d..e43ba4075775e 100644 --- a/tests/sqllogictests/suites/query/functions/02_0048_function_semi_structureds_parse_json.test +++ b/tests/sqllogictests/suites/query/functions/02_0048_function_semi_structureds_parse_json.test @@ -64,6 +64,25 @@ select parse_json('{ "test" : "\\"abc\\"测试⚠️✅❌မြန်မာဘ ---- {"test":"\"abc\"测试⚠️✅❌မြန်မာဘာသာ"} +query TT +select parse_json('99999999999999999999999999999999999999'), parse_json('-99999999.999999999999999999999999999999') +---- +1e38 -100000000.0 + +query T +select parse_json('9999999999999999999999999999999999999999999999999999999999999999999999999999') +---- +1e76 + +statement error 1006 +select parse_json('+1'), parse_json('0000012'), parse_json('1.'), parse_json('.2') + +statement error 1006 +select parse_json('[1,2, ,4]'), parse_json('[,2]'), parse_json('[1,]') + +statement ok +set enable_extended_json_syntax = 1 + query TT select parse_json('99999999999999999999999999999999999999'), parse_json('-99999999.999999999999999999999999999999') ---- @@ -84,6 +103,9 @@ select parse_json('[1,2, ,4]'), parse_json('[,2]'), parse_json('[1,]') ---- [1,2,null,4] [null,2] [1,null] +statement ok +set enable_extended_json_syntax = 0 + statement error 1006 select parse_json('[1,') diff --git a/tests/sqllogictests/suites/query/functions/02_0056_function_semi_structureds_as.test b/tests/sqllogictests/suites/query/functions/02_0056_function_semi_structureds_as.test index 231a780aeb127..effe33152f66c 100644 --- a/tests/sqllogictests/suites/query/functions/02_0056_function_semi_structureds_as.test +++ b/tests/sqllogictests/suites/query/functions/02_0056_function_semi_structureds_as.test @@ -58,6 +58,29 @@ select as_object(parse_json('{"k": 123}')) ---- {"k":123} +query TTTT +select to_binary('abc')::variant, to_binary(to_binary('abc')::variant), to_binary(parse_json('"abc"')), as_binary(to_binary('abc')::variant) +---- +"616263" 363136323633 616263 NULL + +query TTTT +select to_date('2025-01-01')::variant, to_date(to_date('2025-01-01')::variant), to_date(parse_json('"2025-01-01"')), as_date(to_date('2025-01-01')::variant) +---- +"2025-01-01" 2025-01-01 2025-01-01 NULL + +query TTTT +select to_timestamp('2025-01-01 10:00:00')::variant, to_timestamp(to_timestamp('2025-01-01 10:00:00')::variant), to_timestamp(parse_json('"2025-01-01 10:00:00"')), as_timestamp(to_timestamp('2025-01-01 10:00:00')::variant) +---- +"2025-01-01 10:00:00.000000" 2025-01-01 10:00:00.000000 2025-01-01 10:00:00.000000 NULL + +query TTTT +select to_interval('10 months 2 days')::variant, to_interval(to_interval('10 months 2 days')::variant), to_interval(parse_json('"10 months 2 days"')), as_interval(to_interval('10 months 2 days')::variant) +---- +"10 months 2 days" 10 months 2 days 10 months 2 days NULL + +statement ok +set enable_extended_json_syntax = 1 + query TTTT select to_binary('abc')::variant, to_binary(to_binary('abc')::variant), to_binary(parse_json('"abc"')), as_binary(to_binary('abc')::variant) ---- @@ -83,6 +106,9 @@ select to_decimal(10, 2)(parse_json('3.14')), as_decimal(parse_json('3.14')), as ---- 3.14 3 3.14 3.14 +statement ok +set enable_extended_json_syntax = 0 + query B select is_null_value(parse_json('null')) ---- @@ -153,6 +179,29 @@ select is_object(parse_json('["a","b","c"]')) ---- 0 +query B +select is_binary(to_binary('abc')::variant) +---- +0 + +query B +select is_date(to_date('2025-01-01')::variant) +---- +0 + +query B +select is_timestamp(to_timestamp('2025-01-01 10:00:00')::variant) +---- +0 + +query B +select is_interval(to_interval('10 months 2 days')::variant) +---- +0 + +statement ok +set enable_extended_json_syntax = 1 + query B select is_binary(to_binary('abc')::variant) ---- @@ -173,6 +222,8 @@ select is_interval(to_interval('10 months 2 days')::variant) ---- 1 +statement ok +set enable_extended_json_syntax = 0 statement ok DROP DATABASE IF EXISTS db1 diff --git a/tests/sqllogictests/suites/query/functions/02_0065_function_json.test b/tests/sqllogictests/suites/query/functions/02_0065_function_json.test index b1b328649caab..e193d18c72e5f 100644 --- a/tests/sqllogictests/suites/query/functions/02_0065_function_json.test +++ b/tests/sqllogictests/suites/query/functions/02_0065_function_json.test @@ -38,11 +38,22 @@ SELECT json_typeof(parse_json('"test"')) ---- STRING +query TTT +SELECT json_typeof(parse_json('112')), json_typeof(parse_json('-1.12')), json_typeof(parse_json('1.12e10')) +---- +INTEGER DOUBLE DOUBLE + +statement ok +set enable_extended_json_syntax = 1 + query TTT SELECT json_typeof(parse_json('112')), json_typeof(parse_json('-1.12')), json_typeof(parse_json('1.12e10')) ---- INTEGER DECIMAL DOUBLE +statement ok +set enable_extended_json_syntax = 0 + query T SELECT json_typeof(parse_json('[1,2,3]')) ---- From 9a042690720268f1e3391eeae82ec8aeaeb0dc8f Mon Sep 17 00:00:00 2001 From: baishen Date: Sat, 5 Jul 2025 00:40:43 +0800 Subject: [PATCH 2/3] fix --- src/query/settings/src/settings_default.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/query/settings/src/settings_default.rs b/src/query/settings/src/settings_default.rs index e5389e2126747..b73340ce927fc 100644 --- a/src/query/settings/src/settings_default.rs +++ b/src/query/settings/src/settings_default.rs @@ -1363,7 +1363,7 @@ impl DefaultSettings { }), ("enable_extended_json_syntax", DefaultSettingValue { value: UserSettingValue::UInt64(0), - desc: "Set to 1 for entended JSON syntax, support decimal values.", + desc: "Set to 1 for extended JSON syntax (like decimal values).", mode: SettingMode::Both, scope: SettingScope::Both, range: Some(SettingRange::Numeric(0..=1)), From ea9341914d354a1ab30d75634f449191bdb1642b Mon Sep 17 00:00:00 2001 From: baishen Date: Sat, 5 Jul 2025 01:44:44 +0800 Subject: [PATCH 3/3] fix tests --- .../suites/ee/01_ee_system/01_0002_virtual_column.test | 4 ++-- .../01_streaming_load/01_0002_streaming_load_variant.result | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test index c07ccbcef8c6f..1a7dbc8d1da5e 100644 --- a/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test +++ b/tests/sqllogictests/suites/ee/01_ee_system/01_0002_virtual_column.test @@ -262,7 +262,7 @@ query TTTITT show virtual columns where table='tweets'; ---- test_virtual_column tweets data 3000000000 ['create'] String -test_virtual_column tweets data 3000000001 ['geo']['lat'] Jsonb +test_virtual_column tweets data 3000000001 ['geo']['lat'] Float64 test_virtual_column tweets data 3000000002 ['id'] UInt64 test_virtual_column tweets data 3000000003 ['replies'] UInt64 test_virtual_column tweets data 3000000004 ['text'] String @@ -289,7 +289,7 @@ query TTTITT show virtual columns where table='tweets'; ---- test_virtual_column tweets data 3000000000 ['create'] String -test_virtual_column tweets data 3000000001 ['geo']['lat'] Jsonb +test_virtual_column tweets data 3000000001 ['geo']['lat'] Float64 test_virtual_column tweets data 3000000002 ['id'] UInt64 test_virtual_column tweets data 3000000003 ['replies'] UInt64 test_virtual_column tweets data 3000000004 ['text'] String diff --git a/tests/suites/1_stateful/01_streaming_load/01_0002_streaming_load_variant.result b/tests/suites/1_stateful/01_streaming_load/01_0002_streaming_load_variant.result index ed8d0a4e75be1..bd9e4ea758c17 100755 --- a/tests/suites/1_stateful/01_streaming_load/01_0002_streaming_load_variant.result +++ b/tests/suites/1_stateful/01_streaming_load/01_0002_streaming_load_variant.result @@ -1,5 +1,5 @@ -{"id":"csv/json_sample1.csv","stats":{"rows":20,"bytes":647}} -{"id":"csv/json_sample2.csv","stats":{"rows":22,"bytes":721}} +{"id":"csv/json_sample1.csv","stats":{"rows":20,"bytes":645}} +{"id":"csv/json_sample2.csv","stats":{"rows":22,"bytes":719}} 1 null 2 true 3 false