Merge branch 'multiple-catalogs/show-tables' of github.com:ClSlaid/databend into multiple-catalogs/show-tables

Author: ClSlaid

.github/actions/artifact_upload/action.yml

@@ -39,6 +39,13 @@ runs:
         name: ${{ inputs.profile }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
         path: ./target/${{ inputs.target }}/${{ inputs.profile }}/databend-*
 
+    - name: Upload open-sharing to github
+      if: steps.info.outputs.src == 'github'
+      uses: actions/upload-artifact@v3
+      with:
+        name: ${{ inputs.profile }}-${{ inputs.sha }}-${{ inputs.target }}-${{ inputs.category }}
+        path: ./target/${{ inputs.target }}/${{ inputs.profile }}/open-sharing
+
     - name: Upload artifacts to s3
       if: steps.info.outputs.src == 's3'
       shell: bash

Cargo.lock

@@ -160,34 +160,78 @@ formatTypeOptions ::=
   RECORD_DELIMITER = '<character>'
   FIELD_DELIMITER = '<character>'
   SKIP_HEADER = <integer>
+  QUOTE = '<character>'
+  ESCAPE = '<character>'
+  NAN_DISPLAY = '<string>'
+  ROW_TAG = '<string>'
   COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE
 ```
 
-#### `RECORD_DELIMITER = '<character>'`
+#### `TYPE = 'CSV'`
 
-Description: One character that separate records in an input file.
+Comma Separated Values format ([RFC](https://www.rfc-editor.org/rfc/rfc4180)).
 
-Default: `'\n'`
+some notice:
 
-#### `FIELD_DELIMITER = '<character>'`
+1. a string field contains `Quote`|`Escape`|`RECORD_DELIMITER`|`RECORD_DELIMITER` must be quoted.
+2. no character is escaped except `Quote` in quoted string.
+3. no space between `FIELD_DELIMITER` and `Quote`.
+4. no trailing `FIELD_DELIMITER` for a record.
+5. Array/Struct field is serialized to a string as in SQL, and then the resulting string is output to CSV in quotes.
+6. if you are generating CSV via programing, we highly recommend you to use the CSV lib of the programing language.
+7. for text file unloaded from [MySQL](https://dev.mysql.com/doc/refman/8.0/en/load-data.html), the default format is
+   TSV in databend. it is valid CSV only if  `ESCAPED BY` is empty and `ENCLOSED BY` is not empty.
 
-Description: One character that separate fields in an input file.
+##### `RECORD_DELIMITER = '<character>'`
 
-Default: `','` (comma)
+**Description**: One character that separate records in an input file.
+**Supported Values**: `\r\n` or One character including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+**Default**: `\n`
 
-#### `SKIP_HEADER = '<integer>'`
+##### `FIELD_DELIMITER = '<character>'`
 
-Description: Number of lines at the start of the file to skip.
+**Description**: One character that separate fields in an input file.
+**Supported Values**: One character only, including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+**Default**: `,` (comma)
 
-Default: `0`
+##### `Quote = '<character>'`
 
-#### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+**Description**: One character to quote strings in CSV file.
 
-Description: String that represents the compression algorithm.
+for data loading, quote is not necessary unless a string contains `Quote`|`Escape`|`RECORD_DELIMITER`|`RECORD_DELIMITER`
 
-Default: `NONE`
+**Supported Values**: `\'` or `\"`.
 
-Values:
+**Default**: `\"`
+
+##### `ESCAPE = '<character>'`
+
+**Description**: One character to escape quote in quoted strings.
+
+**Supported Values**: `\'` or `\"` or `\\`.
+
+**Default**: `\"`
+
+##### `SKIP_HEADER = '<integer>'`
+
+**Use**: Data loading only.
+
+**Description**: Number of lines at the start of the file to skip.
+
+**Default**: `0`
+
+##### `NAN_DISPLAY = '<string>'`
+
+**Supported Values**: must be literal `'nan'` or `'null'` (case-insensitive)
+**Default**: `'NaN'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+**Description**: String that represents the compression algorithm.
+
+**Default**: `NONE`
+
+**Supported Values**:
 
 | Values        | Notes                                                           |
 | ------------- | --------------------------------------------------------------- |
@@ -201,6 +245,55 @@ Values:
 | `XZ`          |                                                                 |
 | `NONE`        | Indicates that the files have not been compressed.              |
 
+#### `TYPE = 'TSV'`
+
+1. these characters are escaped: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\\`, `\'`, `RECORD_DELIMITER`,`FIELD_DELIMITER`.
+2. quoting/enclosing is not support now.
+3. Array/Struct field is serialized to a string as in SQL, and then the resulting string is output to CSV in quotes.
+4. Null is serialized as `\N`
+
+##### `RECORD_DELIMITER = '<character>'`
+
+**Description**: One character that separate records in an input file.
+
+**Supported Values**: `\r\n` or One character including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+
+**Default**: `\n`
+
+##### `FIELD_DELIMITER = '<character>'`
+
+**Description**: One character that separate fields in an input file.
+
+**Supported Values**: One character only, including escaped char: `\b`, `\f`, `\r`, `\n`, `\t`, `\0`, `\xHH`
+
+**Default**: `\t` (TAB)
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+#### `TYPE = 'NDJSON'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+#### `TYPE = 'XML'`
+
+##### `COMPRESSION = AUTO | GZIP | BZ2 | BROTLI | ZSTD | DEFLATE | RAW_DEFLATE | XZ | NONE`
+
+same as `COMPRESSION` in `TYPE = 'CSV'`
+
+##### `ROW_TAG` = `<string>`
+
+**Description**: used to select XML elements to be decoded as a record.
+
+**Default**: `'row'`
+
+#### `TYPE = 'Parquet'`
+
+No options available now.
+
 ### copyOptions
 
 ```

docs/doc/14-sql-commands/10-dml/dml-copy-into-table.md

@@ -56,7 +56,7 @@ impl SingletonType {
                 let guard = LOCAL.wait().read();
                 let v: &T = guard
                     .get(&thread_name)
-                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated"))
+                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated, don't worry if we are in dropping"))
                     .get();
                 v.clone()
             }
@@ -75,7 +75,7 @@ impl SingletonType {
                 let guard = LOCAL.wait().read();
                 guard
                     .get(&thread_name)
-                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated"))
+                    .unwrap_or_else(|| panic!("thread {thread_name} is not initiated, don't worry if we are in dropping"))
                     .set(value)
             }
         }

src/common/base/src/base/singleton_instance.rs

@@ -51,14 +51,14 @@ pub mod linux {
         #[inline(always)]
         fn mmap_alloc(&self, layout: Layout) -> Result<NonNull<[u8]>, AllocError> {
             debug_assert!(layout.align() <= page_size());
+            ThreadTracker::alloc(layout.size() as i64);
             const PROT: i32 = libc::PROT_READ | libc::PROT_WRITE;
             const FLAGS: i32 = libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_POPULATE;
             let addr = unsafe { libc::mmap(null_mut(), layout.size(), PROT, FLAGS, -1, 0) };
             if addr == libc::MAP_FAILED {
                 return Err(AllocError);
             }
             let addr = NonNull::new(addr as *mut ()).ok_or(AllocError)?;
-            ThreadTracker::alloc(layout.size() as i64);
             Ok(NonNull::<[u8]>::from_raw_parts(addr, layout.size()))
         }
 

src/common/base/src/mem_allocator/mmap_allocator.rs

@@ -55,10 +55,6 @@ pub use operator::init_operator;
 pub use operator::CacheOperator;
 pub use operator::DataOperator;
 
-mod location;
-pub use location::parse_uri_location;
-pub use location::UriLocation;
-
 mod metrics;
 pub use metrics::StorageMetrics;
 pub use metrics::StorageMetricsLayer;

src/common/storage/src/lib.rs

@@ -18,8 +18,8 @@ use std::fmt::Formatter;
 use crate::ast::write_comma_separated_list;
 use crate::ast::write_period_separated_list;
 use crate::ast::Expr;
+use crate::ast::FileLocation;
 use crate::ast::Identifier;
-use crate::ast::StageLocation;
 use crate::parser::token::Token;
 
 /// Root node of a query tree
@@ -178,7 +178,7 @@ pub enum TableReference<'a> {
     },
     Stage {
         span: &'a [Token<'a>],
-        location: StageLocation,
+        location: FileLocation,
         files: Vec<String>,
         alias: Option<TableAlias<'a>>,
     },

src/query/ast/src/ast/query.rs

@@ -213,6 +213,25 @@ impl Display for StageLocation {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum FileLocation {
+    Stage(StageLocation),
+    Uri(UriLocation),
+}
+
+impl Display for FileLocation {
+    fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
+        match self {
+            FileLocation::Uri(loc) => {
+                write!(f, "{}", loc)
+            }
+            FileLocation::Stage(loc) => {
+                write!(f, "{}", loc)
+            }
+        }
+    }
+}
+
 pub enum CopyOption {
     Files(Vec<String>),
     Pattern(String),

src/query/ast/src/ast/statements/copy.rs

@@ -22,6 +22,7 @@ use pratt::PrattParser;
 use pratt::Precedence;
 
 use super::stage::stage_location;
+use super::stage::uri_location;
 use crate::ast::*;
 use crate::input::Input;
 use crate::input::WithSpan;
@@ -259,7 +260,7 @@ pub enum TableReferenceElement<'a> {
     JoinCondition(JoinCondition<'a>),
     Group(TableReference<'a>),
     Stage {
-        location: StageLocation,
+        location: FileLocation,
         files: Vec<String>,
         alias: Option<TableAlias<'a>>,
     },
@@ -325,9 +326,27 @@ pub fn table_reference_element(i: Input) -> IResult<WithSpan<TableReferenceEleme
         |(_, table_ref, _)| TableReferenceElement::Group(table_ref),
     );
 
+    let stage_location = |i| {
+        map_res(
+            rule! {
+                #stage_location
+            },
+            |v| Ok(FileLocation::Stage(v)),
+        )(i)
+    };
+
+    let uri_location = |i| {
+        map_res(
+            rule! {
+                #uri_location
+            },
+            |v| Ok(FileLocation::Uri(v)),
+        )(i)
+    };
+
     let aliased_stage = map(
         rule! {
-            #stage_location ~ ( FILES ~ "=" ~ "(" ~ #comma_separated_list0(literal_string) ~ ")")? ~ #table_alias?
+            (#stage_location | #uri_location) ~ ( FILES ~ "=" ~ "(" ~ #comma_separated_list0(literal_string) ~ ")")? ~ #table_alias?
         },
         |(location, files, alias)| TableReferenceElement::Stage {
             location,

src/query/ast/src/parser/query.rs

@@ -552,7 +552,9 @@ impl QueryCoordinator {
             }
         }
 
-        let executor_settings = ExecutorSettings::try_create(&info.query_ctx.get_settings())?;
+        let query_id = info.query_ctx.get_id();
+        let executor_settings =
+            ExecutorSettings::try_create(&info.query_ctx.get_settings(), query_id)?;
 
         let executor = PipelineCompleteExecutor::from_pipelines(pipelines, executor_settings)?;