Skip to content

adding config to control Varchar behavior #11090

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
Jun 28, 2024
Merged

adding config to control Varchar behavior #11090

Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
12 commits
Select commit Hold shift + click to select a range
c993fd7
adding config to control Varchar behavior
Lordworms Jun 23, 2024
d81bd76
fix failed tests
Lordworms Jun 23, 2024
a227b93
fix config_md
Lordworms Jun 23, 2024
4458a97
format md
Lordworms Jun 23, 2024
fa3925f
optimize code
Lordworms Jun 28, 2024
aa6ef3b
format md
Lordworms Jun 28, 2024
ea803d8
format md
Lordworms Jun 28, 2024
1ddfe90
adding config
Lordworms Jun 28, 2024
532d977
Tweak documentation
alamb Jun 28, 2024
8c3f4ad
Update sqllogictest
alamb Jun 28, 2024
7e280ca
tweaks strings
alamb Jun 28, 2024
d134b4e
Merge remote-tracking branch 'apache/main' into issue_10743
alamb Jun 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions datafusion/common/src/config.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,12 @@ config_namespace! {
/// MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, and Ansi.
pub dialect: String, default = "generic".to_string()

/// If set to true, the system will return an error when encountering a `Varchar`
/// type with a specified length. This can be useful for enforcing certain schema
/// constraints or maintaining compatibility with systems that do not support
/// length-specified `Varchar` types.
pub error_on_varchar_with_length: bool, default = false
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It might be easier to understand to avoid the double negative here

So perhaps the config flag could be support_varchar_with_length and default to true 🤔

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

sure, I'll fix it


}
}

Expand Down Expand Up @@ -303,6 +309,7 @@ config_namespace! {
/// statistics into the same file groups.
/// Currently experimental
pub split_file_groups_by_statistics: bool, default = false

}
}

Expand Down
1 change: 1 addition & 0 deletions datafusion/core/src/execution/session_state.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -563,6 +563,7 @@ impl SessionState {
ParserOptions {
parse_float_as_decimal: sql_parser_options.parse_float_as_decimal,
enable_ident_normalization: sql_parser_options.enable_ident_normalization,
error_on_varchar_with_length: sql_parser_options.error_on_varchar_with_length,
}
}

Expand Down
9 changes: 8 additions & 1 deletion datafusion/sql/src/planner.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,13 +97,15 @@ pub trait ContextProvider {
pub struct ParserOptions {
pub parse_float_as_decimal: bool,
pub enable_ident_normalization: bool,
pub error_on_varchar_with_length: bool,
}

impl Default for ParserOptions {
fn default() -> Self {
Self {
parse_float_as_decimal: false,
enable_ident_normalization: true,
error_on_varchar_with_length: false,
}
}
}
Expand Down Expand Up @@ -398,12 +400,17 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
SQLDataType::UnsignedInt(_) | SQLDataType::UnsignedInteger(_) | SQLDataType::UnsignedInt4(_) => {
Ok(DataType::UInt32)
}
SQLDataType::Varchar(length) => {
match (length, self.options.error_on_varchar_with_length) {
(Some(_), true) => plan_err!("does not support Varchar with length, please set corresponding parameter"),
_ => Ok(DataType::Utf8),
}
}
SQLDataType::UnsignedBigInt(_) | SQLDataType::UnsignedInt8(_) => Ok(DataType::UInt64),
SQLDataType::Float(_) => Ok(DataType::Float32),
SQLDataType::Real | SQLDataType::Float4 => Ok(DataType::Float32),
SQLDataType::Double | SQLDataType::DoublePrecision | SQLDataType::Float8 => Ok(DataType::Float64),
SQLDataType::Char(_)
| SQLDataType::Varchar(_)
| SQLDataType::Text
| SQLDataType::String(_) => Ok(DataType::Utf8),
SQLDataType::Timestamp(None, tz_info) => {
Expand Down
2 changes: 2 additions & 0 deletions datafusion/sql/tests/sql_integration.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ fn parse_decimals() {
ParserOptions {
parse_float_as_decimal: true,
enable_ident_normalization: false,
error_on_varchar_with_length: false,
},
);
}
Expand Down Expand Up @@ -137,6 +138,7 @@ fn parse_ident_normalization() {
ParserOptions {
parse_float_as_decimal: false,
enable_ident_normalization,
error_on_varchar_with_length: false,
},
);
if plan.is_ok() {
Expand Down
2 changes: 2 additions & 0 deletions datafusion/sqllogictest/test_files/information_schema.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -236,6 +236,7 @@ datafusion.optimizer.skip_failed_rules false
datafusion.optimizer.top_down_join_key_reordering true
datafusion.sql_parser.dialect generic
datafusion.sql_parser.enable_ident_normalization true
datafusion.sql_parser.error_on_varchar_with_length false
datafusion.sql_parser.parse_float_as_decimal false

# show all variables with verbose
Expand Down Expand Up @@ -317,6 +318,7 @@ datafusion.optimizer.skip_failed_rules false When set to true, the logical plan
datafusion.optimizer.top_down_join_key_reordering true When set to true, the physical plan optimizer will run a top down process to reorder the join keys
datafusion.sql_parser.dialect generic Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, and Ansi.
datafusion.sql_parser.enable_ident_normalization true When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted)
datafusion.sql_parser.error_on_varchar_with_length false If set to true, the system will return an error when encountering a `Varchar` type with a specified length. This can be useful for enforcing certain schema constraints or maintaining compatibility with systems that do not support length-specified `Varchar` types.
datafusion.sql_parser.parse_float_as_decimal false When set to true, SQL parser will parse float as decimal type

# show_variable_in_config_options
Expand Down
39 changes: 39 additions & 0 deletions datafusion/sqllogictest/test_files/strings.slt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,3 +78,42 @@ e1
p2
p2e1
p2m1e1

statement ok
set datafusion.sql_parser.error_on_varchar_with_length = true;

# could not process due to config setting
query error
SELECT '12345'::VARCHAR(2);

query error
SELECT s::VARCHAR(2) FROM (VALUES ('12345')) t(s);

statement ok
create table vals(s char) as values('abc'), ('def');

query error
SELECT s::VARCHAR(2) FROM vals

statement ok
set datafusion.sql_parser.error_on_varchar_with_length = false;

# could be done when we diable this setting
query T
SELECT '12345'::VARCHAR(2)
----
12345

query T
SELECT s::VARCHAR(2) FROM (VALUES ('12345')) t(s)
----
12345

query T
SELECT s::VARCHAR(2) FROM vals
----
abc
def

statement ok
drop table vals;
1 change: 1 addition & 0 deletions docs/source/user-guide/configs.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,3 +113,4 @@ Environment variables are read during `SessionConfig` initialisation so they mus
| datafusion.sql_parser.parse_float_as_decimal | false | When set to true, SQL parser will parse float as decimal type |
| datafusion.sql_parser.enable_ident_normalization | true | When set to true, SQL parser will normalize ident (convert ident to lowercase when not quoted) |
| datafusion.sql_parser.dialect | generic | Configure the SQL dialect used by DataFusion's parser; supported values include: Generic, MySQL, PostgreSQL, Hive, SQLite, Snowflake, Redshift, MsSQL, ClickHouse, BigQuery, and Ansi. |
| datafusion.sql_parser.error_on_varchar_with_length | false | If set to true, the system will return an error when encountering a `Varchar` type with a specified length. This can be useful for enforcing certain schema constraints or maintaining compatibility with systems that do not support length-specified `Varchar` types. |