Merge branch 'main' into weekly-72

Author: mergify[bot]

docs/doc/14-sql-commands/10-dml/dml-copy-into-table.md

@@ -160,34 +160,78 @@ formatTypeOptions ::=
   RECORD_DELIMITER = '<character>'
   FIELD_DELIMITER = '<character>'
   SKIP_HEADER = <integer>
+  QUOTE = '<character>'
+  ESCAPE = '<character>'
+  NAN_DISPLAY = '<string>'
+  ROW_TAG = '<string>'
   COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE
 ```
 
-#### `RECORD_DELIMITER = '<character>'`
+#### `TYPE = 'CSV'`
 
-Description: One character that separate records in an input file.
+Comma Separated Values format ([RFC](https://www.rfc-editor.org/rfc/rfc4180)).
 
-Default: `'\n'`
+some notice:
 
-#### `FIELD_DELIMITER = '<character>'`
+1. a string field contains `Quote`|`Escape`|`RECORD_DELIMITER`|`RECORD_DELIMITER` must be quoted.
+2. no character is escaped except `Quote` in quoted string.
+3. no space between `FIELD_DELIMITER` and `Quote`.
+4. no trailing `FIELD_DELIMITER` for a record.
+5. Array/Struct field is serialized to a string as in SQL, and then the resulting string is output to CSV in quotes.
+6. if you are generating CSV via programing, we highly recommend you to use the CSV lib of the programing language.
+7. for text file unloaded from [MySQL](https://dev.mysql.com/doc/refman/8.0/en/load-data.html), the default format is
+   TSV in databend. it is valid CSV only if  `ESCAPED BY` is empty and `ENCLOSED BY` is not empty.
 
-Description: One character that separate fields in an input file.
+##### `RECORD_DELIMITER = '<character>'`
 
-Default: `','` (comma)
+**Description**: One character that separate records in an input file.
+**Supported Values**: `\r\n` or One character including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+**Default**: `\n`
 
-#### `SKIP_HEADER = '<integer>'`
+##### `FIELD_DELIMITER = '<character>'`
 
-Description: Number of lines at the start of the file to skip.
+**Description**: One character that separate fields in an input file.
+**Supported Values**: One character only, including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+**Default**: `,` (comma)
 
-Default: `0`
+##### `Quote = '<character>'`
 
-#### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+**Description**: One character to quote strings in CSV file.
 
-Description: String that represents the compression algorithm.
+for data loading, quote is not necessary unless a string contains `Quote`|`Escape`|`RECORD_DELIMITER`|`RECORD_DELIMITER`
 
-Default: `NONE`
+**Supported Values**: `\'` or `\"`.
 
-Values:
+**Default**: `\"`
+
+##### `ESCAPE = '<character>'`
+
+**Description**: One character to escape quote in quoted strings.
+
+**Supported Values**: `\'` or `\"` or `\\`.
+
+**Default**: `\"`
+
+##### `SKIP_HEADER = '<integer>'`
+
+**Use**: Data loading only.
+
+**Description**: Number of lines at the start of the file to skip.
+
+**Default**: `0`
+
+##### `NAN_DISPLAY = '<string>'`
+
+**Supported Values**: must be literal `'nan'` or `'null'` (case-insensitive)
+**Default**: `'NaN'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+**Description**: String that represents the compression algorithm.
+
+**Default**: `NONE`
+
+**Supported Values**:
 
 | Values        | Notes                                                           |
 | ------------- | --------------------------------------------------------------- |
@@ -201,6 +245,55 @@ Values:
 | `XZ`          |                                                                 |
 | `NONE`        | Indicates that the files have not been compressed.              |
 
+#### `TYPE = 'TSV'`
+
+1. these characters are escaped: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\\`, `\'`, `RECORD_DELIMITER`,`FIELD_DELIMITER`.
+2. quoting/enclosing is not support now.
+3. Array/Struct field is serialized to a string as in SQL, and then the resulting string is output to CSV in quotes.
+4. Null is serialized as `\N`
+
+##### `RECORD_DELIMITER = '<character>'`
+
+**Description**: One character that separate records in an input file.
+
+**Supported Values**: `\r\n` or One character including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+
+**Default**: `\n`
+
+##### `FIELD_DELIMITER = '<character>'`
+
+**Description**: One character that separate fields in an input file.
+
+**Supported Values**: One character only, including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+
+**Default**: `\t` (TAB)
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+#### `TYPE = 'NDJSON'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+#### `TYPE = 'XML'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+##### `ROW_TAG` = `<string>`
+
+**Description**: used to select XML elements to be decoded as a record.
+
+**Default**: `'row'`
+
+#### `TYPE = 'Parquet'`
+
+No options available now.
+
 ### copyOptions
 
 ```

src/common/base/src/mem_allocator/mmap_allocator.rs

@@ -51,14 +51,14 @@ pub mod linux {
         #[inline(always)]
         fn mmap_alloc(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
             debug_assert!(layout.align() <= page_size());
+            ThreadTracker::alloc(layout.size() as i64);
             const PROT: i32 = libc::PROT_READ | libc::PROT_WRITE;
             const FLAGS: i32 = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_POPULATE;
             let addr = unsafe { libc::mmap(null_mut(), layout.size(), PROT, FLAGS, -1, 0) };
             if addr == libc::MAP_FAILED {
                 return Err(AllocError);
             }
             let addr = NonNull::new(addr as *mut ()).ok_or(AllocError)?;
-            ThreadTracker::alloc(layout.size() as i64);
             Ok(NonNull::<[u8]>::from_raw_parts(addr, layout.size()))
         }
 

src/common/storage/src/config.rs

@@ -451,6 +451,7 @@ pub struct StorageMokaConfig {
 }
 
 impl Default for StorageMokaConfig {
+    #[no_sanitize(address)]
     fn default() -> Self {
         Self {
             // Use 1G as default.

src/common/storage/src/lib.rs

@@ -28,6 +28,8 @@
 //! - Table snapshots, segments cache must be stored accessed via cache operator.
 //! - Intermediate data generated by query could be stored by temporary operator.
 
+#![feature(no_sanitize)]
+
 mod config;
 pub use config::CacheConfig;
 pub use config::ShareTableConfig;

src/query/ast/src/ast/statements/table.rs

@@ -30,6 +30,7 @@ use crate::ast::UriLocation;
 
 #[derive(Debug, Clone, PartialEq)] // Tables
 pub struct ShowTablesStmt<'a> {
+    pub catalog: Option<Identifier<'a>>,
     pub database: Option<Identifier<'a>>,
     pub full: bool,
     pub limit: Option<ShowLimit<'a>>,
@@ -47,7 +48,11 @@ impl Display for ShowTablesStmt<'_> {
             write!(f, " HISTORY")?;
         }
         if let Some(database) = &self.database {
-            write!(f, " FROM {database}")?;
+            write!(f, " FROM ")?;
+            if let Some(catalog) = &self.catalog {
+                write!(f, "{catalog}.",)?;
+            }
+            write!(f, "{database}")?;
         }
         if let Some(limit) = &self.limit {
             write!(f, " {limit}")?;

src/query/ast/src/parser/statement.rs

@@ -332,11 +332,17 @@ pub fn statement(i: Input) -> IResult<StatementMsg> {
     );
     let show_tables = map(
         rule! {
-            SHOW ~ FULL? ~ TABLES ~ HISTORY? ~ ( ( FROM | IN ) ~ ^#ident )? ~ #show_limit?
+            SHOW ~ FULL? ~ TABLES ~ HISTORY? ~ ( ( FROM | IN ) ~ #peroid_separated_idents_1_to_2 )? ~ #show_limit?
         },
-        |(_, opt_full, _, opt_history, opt_database, limit)| {
+        |(_, opt_full, _, opt_history, ctl_db, limit)| {
+            let (catalog, database) = match ctl_db {
+                Some((_, (Some(c), d))) => (Some(c), Some(d)),
+                Some((_, (None, d))) => (None, Some(d)),
+                _ => (None, None),
+            };
             Statement::ShowTables(ShowTablesStmt {
-                database: opt_database.map(|(_, database)| database),
+                catalog,
+                database,
                 full: opt_full.is_some(),
                 limit,
                 with_history: opt_history.is_some(),

src/query/ast/tests/it/parser.rs

@@ -67,6 +67,9 @@ fn test_statement() {
         r#"show databases format TabSeparatedWithNamesAndTypes;"#,
         r#"show tables"#,
         r#"show tables format TabSeparatedWithNamesAndTypes;"#,
+        r#"show full tables"#,
+        r#"show full tables from db"#,
+        r#"show full tables from ctl.db"#,
         r#"show processlist;"#,
         r#"show create table a.b;"#,
         r#"show create table a.b format TabSeparatedWithNamesAndTypes;"#,

src/query/ast/tests/it/testdata/statement.txt

@@ -37,6 +37,7 @@ SHOW TABLES
 ---------- AST ------------
 ShowTables(
     ShowTablesStmt {
+        catalog: None,
         database: None,
         full: false,
         limit: None,
@@ -52,6 +53,7 @@ SHOW TABLES
 ---------- AST ------------
 ShowTables(
     ShowTablesStmt {
+        catalog: None,
         database: None,
         full: false,
         limit: None,
@@ -64,6 +66,72 @@ ShowTables(
 Some(
     "TabSeparatedWithNamesAndTypes",
 )
+---------- Input ----------
+show full tables
+---------- Output ---------
+SHOW FULL TABLES
+---------- AST ------------
+ShowTables(
+    ShowTablesStmt {
+        catalog: None,
+        database: None,
+        full: true,
+        limit: None,
+        with_history: false,
+    },
+)
+
+
+---------- Input ----------
+show full tables from db
+---------- Output ---------
+SHOW FULL TABLES FROM db
+---------- AST ------------
+ShowTables(
+    ShowTablesStmt {
+        catalog: None,
+        database: Some(
+            Identifier {
+                name: "db",
+                quote: None,
+                span: Ident(22..24),
+            },
+        ),
+        full: true,
+        limit: None,
+        with_history: false,
+    },
+)
+
+
+---------- Input ----------
+show full tables from ctl.db
+---------- Output ---------
+SHOW FULL TABLES FROM ctl.db
+---------- AST ------------
+ShowTables(
+    ShowTablesStmt {
+        catalog: Some(
+            Identifier {
+                name: "ctl",
+                quote: None,
+                span: Ident(22..25),
+            },
+        ),
+        database: Some(
+            Identifier {
+                name: "db",
+                quote: None,
+                span: Ident(26..28),
+            },
+        ),
+        full: true,
+        limit: None,
+        with_history: false,
+    },
+)
+
+
 ---------- Input ----------
 show processlist;
 ---------- Output ---------

src/query/config/src/lib.rs

@@ -12,6 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
+#![feature(no_sanitize)]
+
 /// Config mods provide config support.
 ///
 /// We are providing two config types:

src/query/config/src/outer_v0.rs

@@ -136,6 +136,7 @@ impl Config {
     /// with_args is to control whether we need to load from args or not.
     /// We should set this to false during tests because we don't want
     /// our test binary to parse cargo's args.
+    #[no_sanitize(address)]
     pub fn load(with_args: bool) -> Result<Self> {
         let mut arg_conf = Self::default();