Skip to content

Commit f3d3f90

Browse files
ariesdevilariesdevil
committed
add quote to FileFormatOptions
1 parent e0f2485 commit f3d3f90

File tree

13 files changed

+95
-8
lines changed

13 files changed

+95
-8
lines changed

‎src/meta/proto-conv/src/user_from_to_protobuf_impl.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -509,6 +509,7 @@ impl FromToProto for mt::FileFormatOptions {
509509
escape: p.escape,
510510
compression,
511511
row_tag: p.row_tag,
512+
quote: p.quote,
512513
})
513514
}
514515

@@ -526,6 +527,7 @@ impl FromToProto for mt::FileFormatOptions {
526527
compression,
527528
row_tag: self.row_tag.clone(),
528529
escape: self.escape.clone(),
530+
quote: self.quote.clone(),
529531
})
530532
}
531533
}

‎src/meta/proto-conv/src/util.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,7 @@ const META_CHANGE_LOG: &[(u64, &str)] = &[
7474
21,
7575
"2022-11-24: Add: users.proto/FileFormatOptions::nan_display",
7676
),
77+
(22, "2022-12-13: Add: users.proto/FileFormatOptions::quote"),
7778
];
7879

7980
pub const VER: u64 = META_CHANGE_LOG.last().unwrap().0;

‎src/meta/proto-conv/tests/it/user_proto_conv.rs

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -85,6 +85,7 @@ pub(crate) fn test_fs_stage_info() -> mt::UserStageInfo {
8585
escape: "\\".to_string(),
8686
compression: mt::StageFileCompression::Bz2,
8787
row_tag: "row".to_string(),
88+
quote: "\'\'".to_string(),
8889
},
8990
copy_options: mt::CopyOptions {
9091
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -123,6 +124,7 @@ pub(crate) fn test_s3_stage_info() -> mt::UserStageInfo {
123124
escape: "".to_string(),
124125
compression: mt::StageFileCompression::Bz2,
125126
row_tag: "row".to_string(),
127+
quote: "'".to_string(),
126128
},
127129
copy_options: mt::CopyOptions {
128130
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -163,6 +165,7 @@ pub(crate) fn test_s3_stage_info_v16() -> mt::UserStageInfo {
163165
escape: "".to_string(),
164166
compression: mt::StageFileCompression::Bz2,
165167
row_tag: "".to_string(),
168+
quote: "".to_string(),
166169
},
167170
copy_options: mt::CopyOptions {
168171
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -203,6 +206,7 @@ pub(crate) fn test_s3_stage_info_v14() -> mt::UserStageInfo {
203206
escape: "".to_string(),
204207
compression: mt::StageFileCompression::Bz2,
205208
row_tag: "".to_string(),
209+
quote: "".to_string(),
206210
},
207211
copy_options: mt::CopyOptions {
208212
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -239,6 +243,7 @@ pub(crate) fn test_gcs_stage_info() -> mt::UserStageInfo {
239243
escape: "".to_string(),
240244
compression: mt::StageFileCompression::Bz2,
241245
row_tag: "row".to_string(),
246+
quote: "".to_string(),
242247
},
243248
copy_options: mt::CopyOptions {
244249
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -276,6 +281,7 @@ pub(crate) fn test_oss_stage_info() -> mt::UserStageInfo {
276281
escape: "".to_string(),
277282
compression: mt::StageFileCompression::Bz2,
278283
row_tag: "row".to_string(),
284+
quote: "".to_string(),
279285
},
280286
copy_options: mt::CopyOptions {
281287
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -680,6 +686,7 @@ pub(crate) fn test_internal_stage_info_v17() -> mt::UserStageInfo {
680686
escape: "".to_string(),
681687
compression: mt::StageFileCompression::Bz2,
682688
row_tag: "".to_string(),
689+
quote: "".to_string(),
683690
},
684691
copy_options: mt::CopyOptions {
685692
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -712,6 +719,7 @@ pub(crate) fn test_user_stage_info_v18() -> mt::UserStageInfo {
712719
escape: "".to_string(),
713720
compression: mt::StageFileCompression::Bz2,
714721
row_tag: "".to_string(),
722+
quote: "".to_string(),
715723
},
716724
copy_options: mt::CopyOptions {
717725
on_error: mt::OnErrorMode::SkipFileNum(666),

‎src/meta/proto-conv/tests/it/user_stage.rs

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,53 @@ fn test_user_stage_oss_latest() -> anyhow::Result<()> {
5353
Ok(())
5454
}
5555

56+
#[test]
57+
fn test_user_stage_fs_v22() -> anyhow::Result<()> {
58+
// Encoded data of version 21 of user_stage_fs:
59+
// It is generated with common::test_pb_from_to.
60+
let user_stage_fs_v22 = vec![
61+
10, 17, 102, 115, 58, 47, 47, 100, 105, 114, 47, 116, 111, 47, 102, 105, 108, 101, 115, 26,
62+
25, 10, 23, 18, 21, 10, 13, 47, 100, 105, 114, 47, 116, 111, 47, 102, 105, 108, 101, 115,
63+
160, 6, 22, 168, 6, 1, 34, 37, 8, 1, 16, 128, 8, 26, 1, 124, 34, 2, 47, 47, 40, 2, 50, 1,
64+
92, 58, 3, 114, 111, 119, 66, 3, 78, 97, 78, 74, 2, 39, 39, 160, 6, 22, 168, 6, 1, 42, 10,
65+
10, 3, 32, 154, 5, 16, 142, 8, 24, 1, 50, 4, 116, 101, 115, 116, 160, 6, 22, 168, 6, 1,
66+
];
67+
68+
let want = mt::UserStageInfo {
69+
stage_name: "fs://dir/to/files".to_string(),
70+
stage_type: mt::StageType::LegacyInternal,
71+
stage_params: mt::StageParams {
72+
storage: StorageParams::Fs(StorageFsConfig {
73+
root: "/dir/to/files".to_string(),
74+
}),
75+
},
76+
file_format_options: mt::FileFormatOptions {
77+
format: mt::StageFileFormatType::Json,
78+
skip_header: 1024,
79+
field_delimiter: "|".to_string(),
80+
record_delimiter: "//".to_string(),
81+
nan_display: "NaN".to_string(),
82+
compression: mt::StageFileCompression::Bz2,
83+
escape: "\\".to_string(),
84+
row_tag: "row".to_string(),
85+
quote: "\'\'".to_string(),
86+
},
87+
copy_options: mt::CopyOptions {
88+
on_error: mt::OnErrorMode::SkipFileNum(666),
89+
size_limit: 1038,
90+
split_size: 0,
91+
purge: true,
92+
single: false,
93+
max_file_size: 0,
94+
},
95+
comment: "test".to_string(),
96+
..Default::default()
97+
};
98+
common::test_load_old(func_name!(), user_stage_fs_v22.as_slice(), want)?;
99+
100+
Ok(())
101+
}
102+
56103
#[test]
57104
fn test_user_stage_fs_v21() -> anyhow::Result<()> {
58105
// Encoded data of version 21 of user_stage_fs:
@@ -82,6 +129,7 @@ fn test_user_stage_fs_v21() -> anyhow::Result<()> {
82129
compression: mt::StageFileCompression::Bz2,
83130
escape: "\\".to_string(),
84131
row_tag: "row".to_string(),
132+
quote: "".to_string(),
85133
},
86134
copy_options: mt::CopyOptions {
87135
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -127,6 +175,7 @@ fn test_user_stage_fs_v20() -> anyhow::Result<()> {
127175
compression: mt::StageFileCompression::Bz2,
128176
escape: "\\".to_string(),
129177
row_tag: "row".to_string(),
178+
quote: "".to_string(),
130179
},
131180
copy_options: mt::CopyOptions {
132181
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -173,6 +222,7 @@ fn test_user_stage_fs_v18() -> anyhow::Result<()> {
173222
row_tag: "".to_string(),
174223
escape: "".to_string(),
175224
compression: mt::StageFileCompression::Bz2,
225+
quote: "".to_string(),
176226
},
177227
copy_options: mt::CopyOptions {
178228
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -219,6 +269,7 @@ fn test_user_stage_fs_v16() -> anyhow::Result<()> {
219269
escape: "".to_string(),
220270
compression: mt::StageFileCompression::Bz2,
221271
row_tag: "".to_string(),
272+
quote: "".to_string(),
222273
},
223274
copy_options: mt::CopyOptions {
224275
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -277,6 +328,7 @@ fn test_user_stage_s3_v16() -> anyhow::Result<()> {
277328
escape: "".to_string(),
278329
compression: mt::StageFileCompression::Bz2,
279330
row_tag: "".to_string(),
331+
quote: "".to_string(),
280332
},
281333
copy_options: mt::CopyOptions {
282334
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -329,6 +381,7 @@ fn test_user_stage_gcs_v16() -> anyhow::Result<()> {
329381
escape: "".to_string(),
330382
compression: mt::StageFileCompression::Bz2,
331383
row_tag: "".to_string(),
384+
quote: "".to_string(),
332385
},
333386
copy_options: mt::CopyOptions {
334387
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -383,6 +436,7 @@ fn test_user_stage_oss_v16() -> anyhow::Result<()> {
383436
escape: "".to_string(),
384437
compression: mt::StageFileCompression::Bz2,
385438
row_tag: "".to_string(),
439+
quote: "".to_string(),
386440
},
387441
copy_options: mt::CopyOptions {
388442
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -439,6 +493,7 @@ fn test_user_stage_oss_v13() -> anyhow::Result<()> {
439493
escape: "".to_string(),
440494
compression: mt::StageFileCompression::Bz2,
441495
row_tag: "".to_string(),
496+
quote: "".to_string(),
442497
},
443498
copy_options: mt::CopyOptions {
444499
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -496,6 +551,7 @@ fn test_user_stage_s3_v11() -> anyhow::Result<()> {
496551
escape: "".to_string(),
497552
compression: mt::StageFileCompression::Bz2,
498553
row_tag: "".to_string(),
554+
quote: "".to_string(),
499555
},
500556
copy_options: mt::CopyOptions {
501557
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -551,6 +607,7 @@ fn test_user_stage_s3_v9() -> anyhow::Result<()> {
551607
escape: "".to_string(),
552608
compression: mt::StageFileCompression::Bz2,
553609
row_tag: "".to_string(),
610+
quote: "".to_string(),
554611
},
555612
copy_options: mt::CopyOptions {
556613
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -597,6 +654,7 @@ fn test_user_stage_fs_v6() -> anyhow::Result<()> {
597654
escape: "".to_string(),
598655
compression: mt::StageFileCompression::Bz2,
599656
row_tag: "".to_string(),
657+
quote: "".to_string(),
600658
},
601659
copy_options: mt::CopyOptions {
602660
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -653,6 +711,7 @@ fn test_user_stage_s3_v6() -> anyhow::Result<()> {
653711
escape: "".to_string(),
654712
compression: mt::StageFileCompression::Bz2,
655713
row_tag: "".to_string(),
714+
quote: "".to_string(),
656715
},
657716
copy_options: mt::CopyOptions {
658717
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -705,6 +764,7 @@ fn test_user_stage_gcs_v6() -> anyhow::Result<()> {
705764
escape: "".to_string(),
706765
compression: mt::StageFileCompression::Bz2,
707766
row_tag: "".to_string(),
767+
quote: "".to_string(),
708768
},
709769
copy_options: mt::CopyOptions {
710770
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -750,6 +810,7 @@ fn test_user_stage_fs_v4() -> anyhow::Result<()> {
750810
escape: "".to_string(),
751811
compression: mt::StageFileCompression::Bz2,
752812
row_tag: "".to_string(),
813+
quote: "".to_string(),
753814
},
754815
copy_options: mt::CopyOptions {
755816
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -806,6 +867,7 @@ fn test_user_stage_s3_v4() -> anyhow::Result<()> {
806867
escape: "".to_string(),
807868
compression: mt::StageFileCompression::Bz2,
808869
row_tag: "".to_string(),
870+
quote: "".to_string(),
809871
},
810872
copy_options: mt::CopyOptions {
811873
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -857,6 +919,7 @@ fn test_user_stage_gcs_v4() -> anyhow::Result<()> {
857919
escape: "".to_string(),
858920
compression: mt::StageFileCompression::Bz2,
859921
row_tag: "".to_string(),
922+
quote: "".to_string(),
860923
},
861924
copy_options: mt::CopyOptions {
862925
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -910,6 +973,7 @@ fn test_user_stage_s3_v1() -> anyhow::Result<()> {
910973
escape: "".to_string(),
911974
compression: mt::StageFileCompression::Bz2,
912975
row_tag: "".to_string(),
976+
quote: "".to_string(),
913977
},
914978
copy_options: mt::CopyOptions {
915979
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -958,6 +1022,7 @@ fn test_internal_stage_v17() -> anyhow::Result<()> {
9581022
escape: "".to_string(),
9591023
compression: mt::StageFileCompression::Bz2,
9601024
row_tag: "".to_string(),
1025+
quote: "".to_string(),
9611026
},
9621027
copy_options: mt::CopyOptions {
9631028
on_error: mt::OnErrorMode::SkipFileNum(666),
@@ -1005,6 +1070,7 @@ fn test_user_stage_v18() -> anyhow::Result<()> {
10051070
escape: "".to_string(),
10061071
compression: mt::StageFileCompression::Bz2,
10071072
row_tag: "".to_string(),
1073+
quote: "".to_string(),
10081074
},
10091075
copy_options: mt::CopyOptions {
10101076
on_error: mt::OnErrorMode::SkipFileNum(666),

‎src/meta/protos/proto/user.proto

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -187,6 +187,8 @@ message UserStageInfo {
187187
string row_tag = 7;
188188

189189
string nan_display = 8;
190+
191+
string quote = 9;
190192
}
191193

192194
message OnErrorMode {

‎src/meta/types/src/user_stage.rs

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@ pub struct FileFormatOptions {
200200
pub escape: String,
201201
pub compression: StageFileCompression,
202202
pub row_tag: String,
203+
pub quote: String,
203204
}
204205

205206
impl Default for FileFormatOptions {
@@ -213,6 +214,7 @@ impl Default for FileFormatOptions {
213214
escape: "".to_string(),
214215
compression: StageFileCompression::default(),
215216
row_tag: "row".to_string(),
217+
quote: "".to_string(),
216218
}
217219
}
218220
}

‎src/query/formats/src/field_encoder/csv.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,7 +42,7 @@ impl FieldEncoderCSV {
4242
inf_bytes: INF_BYTES_LOWER.as_bytes().to_vec(),
4343
timezone: options.timezone,
4444
},
45-
quote_char: options.quote.as_bytes()[0],
45+
quote_char: options.stage.quote.as_bytes()[0],
4646
}
4747
}
4848
}

‎src/query/formats/src/file_format_type.rs

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,6 @@ pub trait FileFormatTypeExt {
6161
#[derive(Clone, Debug)]
6262
pub struct FileFormatOptionsExt {
6363
pub stage: FileFormatOptions,
64-
pub quote: String,
6564
pub ident_case_sensitive: bool,
6665
pub headers: usize,
6766
pub json_compact: bool,
@@ -71,7 +70,7 @@ pub struct FileFormatOptionsExt {
7170

7271
impl FileFormatOptionsExt {
7372
pub fn get_quote_char(&self) -> u8 {
74-
self.quote.as_bytes()[0]
73+
self.stage.quote.as_bytes()[0]
7574
}
7675

7776
pub fn get_field_delimiter(&self) -> u8 {
@@ -207,7 +206,6 @@ impl FileFormatTypeExt for StageFileFormatType {
207206
let timezone = parse_timezone(settings)?;
208207
let options = FileFormatOptionsExt {
209208
stage,
210-
quote: "".to_string(),
211209
ident_case_sensitive: false,
212210
headers: 0,
213211
json_compact: false,
@@ -237,10 +235,10 @@ impl FileFormatTypeExt for StageFileFormatType {
237235
|| "get_file_format_options_from_setting",
238236
)?,
239237
row_tag: settings.get_row_tag()?,
238+
quote: settings.get_format_quote()?,
240239
};
241240
let mut options = FileFormatOptionsExt {
242241
stage,
243-
quote: settings.get_format_quote()?,
244242
ident_case_sensitive: settings.get_unquoted_ident_case_sensitive()?,
245243
headers: 0,
246244
json_compact: false,

‎src/query/formats/src/format_option_checker.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ pub trait FormatOptionChecker {
4141

4242
fn check_options(&self, options: &mut FileFormatOptionsExt) -> Result<()> {
4343
self.check_escape(&mut options.stage.escape)?;
44-
self.check_quote(&mut options.quote)?;
44+
self.check_quote(&mut options.stage.quote)?;
4545
self.check_row_tag(&mut options.stage.row_tag)?;
4646
self.check_record_delimiter(&mut options.stage.record_delimiter)?;
4747
self.check_field_delimiter(&mut options.stage.field_delimiter)?;

‎src/query/formats/src/output_format/csv.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,7 @@ impl<const WITH_NAMES: bool, const WITH_TYPES: bool> CSVOutputFormatBase<WITH_NA
4343
field_encoder,
4444
field_delimiter: options.stage.field_delimiter.as_bytes()[0],
4545
record_delimiter: options.stage.record_delimiter.as_bytes().to_vec(),
46-
quote: options.quote.as_bytes()[0],
46+
quote: options.stage.quote.as_bytes()[0],
4747
}
4848
}
4949

0 commit comments

Comments
 (0)