Skip to content

Commit 6c270ba

Browse files
mergify[bot]mergify[bot]
authored
Merge pull request #9198 from youngsofun/stage
refactor(parser): extract stage related rules.
2 parents 4533c87 + 6252e4a commit 6c270ba

File tree

4 files changed

+126
-98
lines changed

4 files changed

+126
-98
lines changed

‎src/query/ast/src/parser/mod.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ pub mod expr;
1616
#[allow(clippy::module_inception)]
1717
mod parser;
1818
pub mod query;
19+
mod stage;
1920
pub mod statement;
2021
pub mod token;
2122
pub mod unescape;

‎src/query/ast/src/parser/query.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ use pratt::Associativity;
2121
use pratt::PrattParser;
2222
use pratt::Precedence;
2323

24-
use super::statement::stage_location;
24+
use super::stage::stage_location;
2525
use crate::ast::*;
2626
use crate::input::Input;
2727
use crate::input::WithSpan;

‎src/query/ast/src/parser/stage.rs

Lines changed: 123 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,123 @@
1+
// Copyright 2022 Datafuse Labs.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
use std::collections::BTreeMap;
15+
16+
use nom::branch::alt;
17+
use nom::combinator::map;
18+
use url::Url;
19+
20+
use crate::ast::StageLocation;
21+
use crate::ast::UriLocation;
22+
use crate::input::Input;
23+
use crate::parser::expr::*;
24+
use crate::parser::token::*;
25+
use crate::rule;
26+
use crate::util::*;
27+
use crate::ErrorKind;
28+
29+
pub fn ident_to_string(i: Input) -> IResult<String> {
30+
map_res(ident, |ident| Ok(ident.name))(i)
31+
}
32+
33+
pub fn u64_to_string(i: Input) -> IResult<String> {
34+
map(literal_u64, |v| v.to_string())(i)
35+
}
36+
37+
pub fn parameter_to_string(i: Input) -> IResult<String> {
38+
map(
39+
rule! { ( #literal_string | #ident_to_string | #u64_to_string ) },
40+
|parameter| parameter,
41+
)(i)
42+
}
43+
44+
// parse: (k = v ...)* into a map
45+
pub fn options(i: Input) -> IResult<BTreeMap<String, String>> {
46+
let ident_with_format = alt((
47+
ident_to_string,
48+
map(rule! { FORMAT }, |_| "FORMAT".to_string()),
49+
));
50+
51+
map(
52+
rule! {
53+
"(" ~ ( #ident_with_format ~ "=" ~ #parameter_to_string )* ~ ")"
54+
},
55+
|(_, opts, _)| {
56+
BTreeMap::from_iter(opts.iter().map(|(k, _, v)| (k.to_lowercase(), v.clone())))
57+
},
58+
)(i)
59+
}
60+
61+
pub fn stage_location(i: Input) -> IResult<StageLocation> {
62+
map_res(at_string, |location| {
63+
let parsed = location.splitn(2, '/').collect::<Vec<_>>();
64+
let name = parsed[0].to_string();
65+
let path = if parsed.len() == 1 {
66+
"/".to_string()
67+
} else {
68+
format!("/{}", parsed[1])
69+
};
70+
Ok(StageLocation { name, path })
71+
})(i)
72+
}
73+
74+
/// Parse input into `UriLocation`
75+
pub fn uri_location(i: Input) -> IResult<UriLocation> {
76+
map_res(
77+
rule! {
78+
#literal_string
79+
~ (CONNECTION ~ "=" ~ #options)?
80+
~ (CREDENTIALS ~ "=" ~ #options)?
81+
~ (ENCRYPTION ~ "=" ~ #options)?
82+
},
83+
|(location, connection_opt, credentials_opt, encryption_opt)| {
84+
// fs location is not a valid url, let's check it in advance.
85+
if let Some(path) = location.strip_prefix("fs://") {
86+
return Ok(UriLocation {
87+
protocol: "fs".to_string(),
88+
name: "".to_string(),
89+
path: path.to_string(),
90+
connection: BTreeMap::default(),
91+
});
92+
}
93+
94+
let parsed =
95+
Url::parse(&location).map_err(|_| ErrorKind::Other("invalid uri location"))?;
96+
97+
// TODO: We will use `CONNECTION` to replace `CREDENTIALS` and `ENCRYPTION`.
98+
let mut conn = connection_opt.map(|v| v.2).unwrap_or_default();
99+
conn.extend(credentials_opt.map(|v| v.2).unwrap_or_default());
100+
conn.extend(encryption_opt.map(|v| v.2).unwrap_or_default());
101+
102+
Ok(UriLocation {
103+
protocol: parsed.scheme().to_string(),
104+
name: parsed
105+
.host_str()
106+
.map(|hostname| {
107+
if let Some(port) = parsed.port() {
108+
format!("{}:{}", hostname, port)
109+
} else {
110+
hostname.to_string()
111+
}
112+
})
113+
.ok_or(ErrorKind::Other("invalid uri location"))?,
114+
path: if parsed.path().is_empty() {
115+
"/".to_string()
116+
} else {
117+
parsed.path().to_string()
118+
},
119+
connection: conn,
120+
})
121+
},
122+
)(i)
123+
}

‎src/query/ast/src/parser/statement.rs

Lines changed: 1 addition & 97 deletions
Original file line numberDiff line numberDiff line change
@@ -28,12 +28,12 @@ use nom::combinator::consumed;
2828
use nom::combinator::map;
2929
use nom::combinator::value;
3030
use nom::Slice;
31-
use url::Url;
3231

3332
use crate::ast::*;
3433
use crate::input::Input;
3534
use crate::parser::expr::*;
3635
use crate::parser::query::*;
36+
use crate::parser::stage::*;
3737
use crate::parser::token::*;
3838
use crate::rule;
3939
use crate::util::*;
@@ -1532,70 +1532,6 @@ pub fn copy_unit(i: Input) -> IResult<CopyUnit> {
15321532
)(i)
15331533
}
15341534

1535-
pub fn stage_location(i: Input) -> IResult<StageLocation> {
1536-
map_res(at_string, |location| {
1537-
let parsed = location.splitn(2, '/').collect::<Vec<_>>();
1538-
let name = parsed[0].to_string();
1539-
let path = if parsed.len() == 1 {
1540-
"/".to_string()
1541-
} else {
1542-
format!("/{}", parsed[1])
1543-
};
1544-
Ok(StageLocation { name, path })
1545-
})(i)
1546-
}
1547-
1548-
/// Parse input into `UriLocation`
1549-
pub fn uri_location(i: Input) -> IResult<UriLocation> {
1550-
map_res(
1551-
rule! {
1552-
#literal_string
1553-
~ (CONNECTION ~ "=" ~ #options)?
1554-
~ (CREDENTIALS ~ "=" ~ #options)?
1555-
~ (ENCRYPTION ~ "=" ~ #options)?
1556-
},
1557-
|(location, connection_opt, credentials_opt, encryption_opt)| {
1558-
// fs location is not a valid url, let's check it in advance.
1559-
if let Some(path) = location.strip_prefix("fs://") {
1560-
return Ok(UriLocation {
1561-
protocol: "fs".to_string(),
1562-
name: "".to_string(),
1563-
path: path.to_string(),
1564-
connection: BTreeMap::default(),
1565-
});
1566-
}
1567-
1568-
let parsed =
1569-
Url::parse(&location).map_err(|_| ErrorKind::Other("invalid uri location"))?;
1570-
1571-
// TODO: We will use `CONNECTION` to replace `CREDENTIALS` and `ENCRYPTION`.
1572-
let mut conn = connection_opt.map(|v| v.2).unwrap_or_default();
1573-
conn.extend(credentials_opt.map(|v| v.2).unwrap_or_default());
1574-
conn.extend(encryption_opt.map(|v| v.2).unwrap_or_default());
1575-
1576-
Ok(UriLocation {
1577-
protocol: parsed.scheme().to_string(),
1578-
name: parsed
1579-
.host_str()
1580-
.map(|hostname| {
1581-
if let Some(port) = parsed.port() {
1582-
format!("{}:{}", hostname, port)
1583-
} else {
1584-
hostname.to_string()
1585-
}
1586-
})
1587-
.ok_or(ErrorKind::Other("invalid uri location"))?,
1588-
path: if parsed.path().is_empty() {
1589-
"/".to_string()
1590-
} else {
1591-
parsed.path().to_string()
1592-
},
1593-
connection: conn,
1594-
})
1595-
},
1596-
)(i)
1597-
}
1598-
15991535
pub fn show_limit(i: Input) -> IResult<ShowLimit> {
16001536
let limit_like = map(
16011537
rule! {
@@ -1773,38 +1709,6 @@ pub fn copy_option(i: Input) -> IResult<CopyOption> {
17731709
))(i)
17741710
}
17751711

1776-
pub fn ident_to_string(i: Input) -> IResult<String> {
1777-
map_res(ident, |ident| Ok(ident.name))(i)
1778-
}
1779-
1780-
pub fn u64_to_string(i: Input) -> IResult<String> {
1781-
map(literal_u64, |v| v.to_string())(i)
1782-
}
1783-
1784-
pub fn parameter_to_string(i: Input) -> IResult<String> {
1785-
map(
1786-
rule! { ( #literal_string | #ident_to_string | #u64_to_string ) },
1787-
|parameter| parameter,
1788-
)(i)
1789-
}
1790-
1791-
// parse: (k = v ...)* into a map
1792-
pub fn options(i: Input) -> IResult<BTreeMap<String, String>> {
1793-
let ident_with_format = alt((
1794-
ident_to_string,
1795-
map(rule! { FORMAT }, |_| "FORMAT".to_string()),
1796-
));
1797-
1798-
map(
1799-
rule! {
1800-
"(" ~ ( #ident_with_format ~ "=" ~ #parameter_to_string )* ~ ")"
1801-
},
1802-
|(_, opts, _)| {
1803-
BTreeMap::from_iter(opts.iter().map(|(k, _, v)| (k.to_lowercase(), v.clone())))
1804-
},
1805-
)(i)
1806-
}
1807-
18081712
pub fn presign_action(i: Input) -> IResult<PresignAction> {
18091713
alt((
18101714
value(PresignAction::Download, rule! { DOWNLOAD }),

0 commit comments

Comments
 (0)