Skip to content

Commit aed99cf

Browse files
authored
Merge branch 'main' into fix-jaeger-filter
2 parents f6ab272 + 2c56c9a commit aed99cf

File tree

12 files changed

+360
-136
lines changed

12 files changed

+360
-136
lines changed

docs/doc/60-contributing/02-roadmap.md

Lines changed: 31 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -12,41 +12,51 @@ This is Databend Roadmap 2022 :rocket:, sync from the [#3706](https://github.com
1212

1313
# Main tasks
1414

15-
### 1. Query
15+
Roadmap 2021: https://github.com/datafuselabs/databend/issues/746
16+
17+
# Main tasks
18+
19+
### 1. Query
1620

1721

1822
| Task | Status | Release Target | Comments |
1923
| ----------------------------------------------- | --------- | -------------- | --------------- |
20-
| [Query Cluster Track #747](https://github.com/datafuselabs/databend/issues/747) | PROGRESS | | |
21-
| [RBAC Privileges #2793](https://github.com/datafuselabs/databend/issues/2793) | PROGRESS | | |
22-
| [ New Planner Framework #1217](https://github.com/datafuselabs/databend/issues/1218)| PROGRESS | | [RFC](https://databend.rs/doc/contributing/rfcs/new-sql-planner-framework)|
23-
| [ Database Sharing #3430](https://github.com/datafuselabs/databend/issues/3430)| PROGRESS | | |
24-
| [ STAGE Command #2976](https://github.com/datafuselabs/databend/issues/2976)| PROGRESS | | |
25-
| [ COPY Command #4104](https://github.com/datafuselabs/databend/issues/4104)| PROGRESS | | |
24+
| [Query Cluster Track #747](https://github.com/datafuselabs/databend/issues/747) | DONE | | |
25+
| [RBAC Privileges #2793](https://github.com/datafuselabs/databend/issues/2793) | DONE | | |
26+
| [ New Planner Framework #1217](https://github.com/datafuselabs/databend/issues/1218)| DONE | | [RFC](https://databend.rs/doc/contributing/rfcs/new-sql-planner-framework)|
27+
| [ Database Sharing #3430](https://github.com/datafuselabs/databend/issues/3430)| DONE | | |
28+
| [ STAGE Command #2976](https://github.com/datafuselabs/databend/issues/2976)| DONE | | |
29+
| [ COPY Command #4104](https://github.com/datafuselabs/databend/issues/4104)| DONE | | |
2630
| [Index Design #3711](https://github.com/datafuselabs/databend/issues/3711) | PROGRESS | | |
27-
| [Push-Based + Pull-Based processor](https://github.com/datafuselabs/databend/issues/3379)| PROGRESS | | |
28-
| [Semi-structured Data Types #3916](https://github.com/datafuselabs/databend/issues/3916) | PROGRESS | | |
31+
| [Push-Based + Pull-Based processor](https://github.com/datafuselabs/databend/issues/3379)| DONE | | |
32+
| [Semi-structured Data Types #3916](https://github.com/datafuselabs/databend/issues/3916) | DONE | | |
33+
| [Table Cluster Key #4268](https://github.com/datafuselabs/databend/issues/4268) | DONE | | |
34+
| Transactions | DONE | | |
2935
| [Support Fulltext Index #3915](https://github.com/datafuselabs/databend/issues/3915) | PLANNING | | |
30-
| [Table Cluster Key #4268](https://github.com/datafuselabs/databend/issues/4268) | PLANNING | | |
31-
| Tansactions | PLANNING | | |
32-
| Window Functions | PLANNING | | |
33-
| Lambda Functions | PLANNING | | |
34-
| Array Functions | PLANNING | | |
36+
| [Hive External Data Source #4826](https://github.com/datafuselabs/databend/issues/4826) | DONE | | |
37+
| [Window Functions](https://github.com/datafuselabs/databend/issues/4653) | DONE | | |
38+
| Lambda Functions | DONE | | |
39+
| Array Functions | DONE | | |
3540
| Compile Aggregate Functions(JIT) | PLANNING | | |
36-
| Common Table Expressions | PLANNING | | [MySQL CTE](https://dev.mysql.com/doc/refman/8.0/en/with.html#common-table-expressions) |
37-
| External Cache | PLANNING | | |
41+
| [Common Table Expressions #6246](https://github.com/datafuselabs/databend/issues/6246) | DONE | | [MySQL CTE](https://dev.mysql.com/doc/refman/8.0/en/with.html#common-table-expressions) |
42+
| [External Cache](https://github.com/datafuselabs/databend/issues/6786) #6786 | PROGRESS | | |
3843
| External Table | PLANNING | | [Snowflake ET](https://docs.snowflake.com/en/sql-reference/sql/create-external-table.html)|
39-
| Update&Delete | PLANNING | | |
44+
| Delete | DONE | | |
45+
| Update | PROGRESS | | |
4046
| Streaming Ingestion | PLANNING | | |
41-
| Streaming Analytics | PLANNING | | |
47+
| [Resource Quota](https://github.com/datafuselabs/databend/issues/6935) | PROGRESS | | |
48+
| [LakeHouse](https://github.com/datafuselabs/databend/issues/7592) | PROGRESS | | v0.9|
4249

4350

4451
### 2. Testing
4552

4653
| Task | Status | Release Target | Comments |
4754
| ----------------------------------------------- | --------- | -------------- | --------------- |
48-
| [ Continuous Benchmarking #3084](https://github.com/datafuselabs/databend/issues/3084) | PROGRESS | | |
55+
| [ Continuous Benchmarking #3084](https://github.com/datafuselabs/databend/issues/3084) | DONE | | https://perf.databend.rs |
56+
4957

5058
# Releases
51-
- [x] #2525
52-
- [x] #2257
59+
- [x] [Release proposal: Nightly v0.8 #4591](https://github.com/datafuselabs/databend/issues/4591)
60+
- [x] [Release proposal: Nightly v0.7 #3428](https://github.com/datafuselabs/databend/issues/3428)
61+
- [x] [Release proposal: Nightly v0.6 #2525](https://github.com/datafuselabs/databend/issues/2525)
62+
- [x] [Release proposal: Nightly v0.5 #2257](https://github.com/datafuselabs/databend/issues/2257)

docs/doc/60-contributing/03-rfcs/20220809-share.md

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,8 @@ To support creating and managing shares, Databend provides the following set of
7676
- Describe Share
7777
- Show Grants
7878
- Create Database, Table FROM Share
79+
- Select data from the Share DB tables
80+
- Show shared database tables
7981

8082
### Create Share
8183

@@ -210,6 +212,33 @@ Syntax:
210212
CREATE DATABASE <name> FROM SHARE <provider_tenant>.<share_name>
211213
```
212214

215+
216+
217+
### Select data from the Share DB tables
218+
219+
After tenants have created a database from a shared database, tenants can select data from the shared table like a normal table, before that the providers MUST have permitted access permission to the shared database and table.
220+
221+
Syntax:
222+
223+
```sql
224+
Select * from <share_db_name>.<table_name>
225+
```
226+
227+
228+
229+
### Show shared database tables
230+
231+
After tenants have created a database from a shared database, tenants can show tables from the shared database, it only outputs the tables which have been permitted to access.
232+
233+
Syntax:
234+
235+
```sql
236+
use <share_db_name>;
237+
show tables;
238+
```
239+
240+
241+
213242
## Example of Using Share with SQL
214243

215244
To create a share using SQL:

src/meta/client/src/grpc_action.rs

Lines changed: 4 additions & 38 deletions
Original file line numberDiff line numberDiff line change
@@ -68,36 +68,16 @@ pub enum MetaGrpcReq {
6868
ListKV(ListKVReq),
6969
}
7070

71-
impl From<MetaGrpcWriteReq> for MetaGrpcReq {
72-
fn from(r: MetaGrpcWriteReq) -> Self {
73-
match r {
74-
MetaGrpcWriteReq::UpsertKV(x) => x.into(),
75-
}
76-
}
77-
}
78-
79-
impl From<MetaGrpcReadReq> for MetaGrpcReq {
80-
fn from(r: MetaGrpcReadReq) -> Self {
81-
match r {
82-
MetaGrpcReadReq::GetKV(x) => x.into(),
83-
MetaGrpcReadReq::MGetKV(x) => x.into(),
84-
MetaGrpcReadReq::ListKV(x) => x.into(),
85-
}
86-
}
87-
}
88-
89-
/// Try convert tonic::Request<RaftRequest> to DoActionAction.
90-
impl TryInto<MetaGrpcWriteReq> for Request<RaftRequest> {
71+
impl TryInto<MetaGrpcReq> for Request<RaftRequest> {
9172
type Error = tonic::Status;
9273

93-
fn try_into(self) -> Result<MetaGrpcWriteReq, Self::Error> {
74+
fn try_into(self) -> Result<MetaGrpcReq, Self::Error> {
9475
let raft_request = self.into_inner();
9576

96-
// Decode DoActionAction from flight request body.
9777
let json_str = raft_request.data.as_str();
98-
let action = serde_json::from_str::<MetaGrpcWriteReq>(json_str)
78+
let req = serde_json::from_str::<MetaGrpcReq>(json_str)
9979
.map_err(|e| tonic::Status::internal(e.to_string()))?;
100-
Ok(action)
80+
Ok(req)
10181
}
10282
}
10383

@@ -110,7 +90,6 @@ impl tonic::IntoRequest<RaftRequest> for MetaGrpcWriteReq {
11090
}
11191
}
11292

113-
/// Try convert DoActionAction to tonic::Request<RaftRequest>.
11493
impl TryInto<Request<RaftRequest>> for MetaGrpcWriteReq {
11594
type Error = serde_json::Error;
11695

@@ -124,19 +103,6 @@ impl TryInto<Request<RaftRequest>> for MetaGrpcWriteReq {
124103
}
125104
}
126105

127-
impl TryInto<MetaGrpcReadReq> for Request<RaftRequest> {
128-
type Error = tonic::Status;
129-
130-
fn try_into(self) -> Result<MetaGrpcReadReq, Self::Error> {
131-
let raft_req = self.into_inner();
132-
133-
let json_str = raft_req.data.as_str();
134-
let action = serde_json::from_str::<MetaGrpcReadReq>(json_str)
135-
.map_err(|e| tonic::Status::internal(e.to_string()))?;
136-
Ok(action)
137-
}
138-
}
139-
140106
impl TryInto<Request<RaftRequest>> for MetaGrpcReadReq {
141107
type Error = serde_json::Error;
142108

src/meta/client/tests/it/grpc_server.rs

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,10 @@ impl MetaService for GrpcServiceForTestImpl {
5555
Ok(Response::new(Box::pin(output)))
5656
}
5757

58+
async fn kv_api(&self, _request: Request<RaftRequest>) -> Result<Response<RaftReply>, Status> {
59+
Err(Status::unimplemented("Not yet implemented"))
60+
}
61+
5862
async fn write_msg(
5963
&self,
6064
_request: Request<RaftRequest>,

src/meta/service/src/api/grpc/grpc_service.rs

Lines changed: 44 additions & 73 deletions
Original file line numberDiff line numberDiff line change
@@ -22,9 +22,7 @@ use common_base::base::tokio::sync::mpsc;
2222
use common_grpc::GrpcClaim;
2323
use common_grpc::GrpcToken;
2424
use common_meta_api::KVApi;
25-
use common_meta_client::MetaGrpcReadReq;
2625
use common_meta_client::MetaGrpcReq;
27-
use common_meta_client::MetaGrpcWriteReq;
2826
use common_meta_types::protobuf::meta_service_server::MetaService;
2927
use common_meta_types::protobuf::ClientInfo;
3028
use common_meta_types::protobuf::Empty;
@@ -39,6 +37,7 @@ use common_meta_types::protobuf::WatchRequest;
3937
use common_meta_types::protobuf::WatchResponse;
4038
use common_meta_types::TxnReply;
4139
use common_meta_types::TxnRequest;
40+
use common_metrics::counter::WithCount;
4241
use futures::StreamExt;
4342
use prost::Message;
4443
use tokio_stream;
@@ -52,10 +51,10 @@ use tracing::info;
5251

5352
use crate::meta_service::meta_service_impl::GrpcStream;
5453
use crate::meta_service::MetaNode;
55-
use crate::metrics::add_meta_metrics_meta_request_inflights;
5654
use crate::metrics::incr_meta_metrics_meta_recv_bytes;
5755
use crate::metrics::incr_meta_metrics_meta_request_result;
5856
use crate::metrics::incr_meta_metrics_meta_sent_bytes;
57+
use crate::metrics::RequestInFlight;
5958
use crate::version::from_digit_ver;
6059
use crate::version::to_digit_ver;
6160
use crate::version::METASRV_SEMVER;
@@ -87,34 +86,7 @@ impl MetaServiceImpl {
8786
Ok(claim)
8887
}
8988

90-
pub async fn execute_kv_req(&self, req: MetaGrpcReq) -> RaftReply {
91-
// To keep the code IDE-friendly, we manually expand the enum variants and dispatch them one by one
92-
93-
match req {
94-
MetaGrpcReq::UpsertKV(a) => {
95-
let res = self.meta_node.upsert_kv(a).await;
96-
incr_meta_metrics_meta_request_result(res.is_ok());
97-
RaftReply::from(res)
98-
}
99-
MetaGrpcReq::GetKV(a) => {
100-
let res = self.meta_node.get_kv(&a.key).await;
101-
incr_meta_metrics_meta_request_result(res.is_ok());
102-
RaftReply::from(res)
103-
}
104-
MetaGrpcReq::MGetKV(a) => {
105-
let res = self.meta_node.mget_kv(&a.keys).await;
106-
incr_meta_metrics_meta_request_result(res.is_ok());
107-
RaftReply::from(res)
108-
}
109-
MetaGrpcReq::ListKV(a) => {
110-
let res = self.meta_node.prefix_list_kv(&a.prefix).await;
111-
incr_meta_metrics_meta_request_result(res.is_ok());
112-
RaftReply::from(res)
113-
}
114-
}
115-
}
116-
117-
pub async fn execute_txn(&self, req: TxnRequest) -> TxnReply {
89+
async fn execute_txn(&self, req: TxnRequest) -> TxnReply {
11890
let ret = self.meta_node.transaction(req).await;
11991
incr_meta_metrics_meta_request_result(ret.is_ok());
12092

@@ -188,51 +160,48 @@ impl MetaService for MetaServiceImpl {
188160
}
189161
}
190162

191-
async fn write_msg(
192-
&self,
193-
request: Request<RaftRequest>,
194-
) -> Result<Response<RaftReply>, Status> {
195-
self.check_token(request.metadata())?;
196-
common_tracing::extract_remote_span_as_parent(&request);
197-
198-
incr_meta_metrics_meta_recv_bytes(request.get_ref().encoded_len() as u64);
199-
200-
let req: MetaGrpcWriteReq = request.try_into()?;
201-
let req: MetaGrpcReq = req.into();
202-
203-
add_meta_metrics_meta_request_inflights(1);
204-
205-
info!("Receive write_action: {:?}", req);
206-
207-
let body = self.execute_kv_req(req).await;
208-
209-
add_meta_metrics_meta_request_inflights(-1);
210-
211-
incr_meta_metrics_meta_sent_bytes(body.encoded_len() as u64);
212-
213-
Ok(Response::new(body))
163+
async fn write_msg(&self, r: Request<RaftRequest>) -> Result<Response<RaftReply>, Status> {
164+
self.kv_api(r).await
214165
}
215166

216-
async fn read_msg(&self, request: Request<RaftRequest>) -> Result<Response<RaftReply>, Status> {
217-
self.check_token(request.metadata())?;
218-
common_tracing::extract_remote_span_as_parent(&request);
219-
220-
incr_meta_metrics_meta_recv_bytes(request.get_ref().encoded_len() as u64);
221-
222-
let req: MetaGrpcReadReq = request.try_into()?;
223-
let req: MetaGrpcReq = req.into();
167+
async fn read_msg(&self, r: Request<RaftRequest>) -> Result<Response<RaftReply>, Status> {
168+
self.kv_api(r).await
169+
}
224170

225-
add_meta_metrics_meta_request_inflights(1);
171+
async fn kv_api(&self, r: Request<RaftRequest>) -> Result<Response<RaftReply>, Status> {
172+
let _guard = WithCount::new((), RequestInFlight);
226173

227-
info!("Receive read_action: {:?}", req);
174+
self.check_token(r.metadata())?;
175+
common_tracing::extract_remote_span_as_parent(&r);
176+
incr_meta_metrics_meta_recv_bytes(r.get_ref().encoded_len() as u64);
228177

229-
let res = self.execute_kv_req(req).await;
178+
let req: MetaGrpcReq = r.try_into()?;
179+
info!("Received MetaGrpcReq: {:?}", req);
230180

231-
add_meta_metrics_meta_request_inflights(-1);
181+
let m = &self.meta_node;
182+
let reply = match req {
183+
MetaGrpcReq::UpsertKV(a) => {
184+
let res = m.upsert_kv(a).await;
185+
RaftReply::from(res)
186+
}
187+
MetaGrpcReq::GetKV(a) => {
188+
let res = m.get_kv(&a.key).await;
189+
RaftReply::from(res)
190+
}
191+
MetaGrpcReq::MGetKV(a) => {
192+
let res = m.mget_kv(&a.keys).await;
193+
RaftReply::from(res)
194+
}
195+
MetaGrpcReq::ListKV(a) => {
196+
let res = m.prefix_list_kv(&a.prefix).await;
197+
RaftReply::from(res)
198+
}
199+
};
232200

233-
incr_meta_metrics_meta_sent_bytes(res.encoded_len() as u64);
201+
incr_meta_metrics_meta_request_result(reply.error.is_empty());
202+
incr_meta_metrics_meta_sent_bytes(reply.encoded_len() as u64);
234203

235-
Ok(Response::new(res))
204+
Ok(Response::new(reply))
236205
}
237206

238207
type ExportStream =
@@ -246,12 +215,12 @@ impl MetaService for MetaServiceImpl {
246215
&self,
247216
_request: Request<common_meta_types::protobuf::Empty>,
248217
) -> Result<Response<Self::ExportStream>, Status> {
249-
let meta_node = &self.meta_node;
218+
let _guard = WithCount::new((), RequestInFlight);
250219

220+
let meta_node = &self.meta_node;
251221
let res = meta_node.sto.export().await?;
252222

253223
let stream = ExportStream { data: res };
254-
255224
let s = stream.map(|strings| Ok(ExportedChunk { data: strings }));
256225

257226
Ok(Response::new(Box::pin(s)))
@@ -280,7 +249,7 @@ impl MetaService for MetaServiceImpl {
280249
) -> Result<Response<TxnReply>, Status> {
281250
self.check_token(request.metadata())?;
282251
incr_meta_metrics_meta_recv_bytes(request.get_ref().encoded_len() as u64);
283-
add_meta_metrics_meta_request_inflights(1);
252+
let _guard = WithCount::new((), RequestInFlight);
284253

285254
common_tracing::extract_remote_span_as_parent(&request);
286255

@@ -289,8 +258,6 @@ impl MetaService for MetaServiceImpl {
289258
info!("Receive txn_request: {:?}", request);
290259

291260
let body = self.execute_txn(request).await;
292-
add_meta_metrics_meta_request_inflights(-1);
293-
294261
incr_meta_metrics_meta_sent_bytes(body.encoded_len() as u64);
295262

296263
Ok(Response::new(body))
@@ -301,6 +268,8 @@ impl MetaService for MetaServiceImpl {
301268
request: Request<MemberListRequest>,
302269
) -> Result<Response<MemberListReply>, Status> {
303270
self.check_token(request.metadata())?;
271+
let _guard = WithCount::new((), RequestInFlight);
272+
304273
let meta_node = &self.meta_node;
305274
let members = meta_node.get_meta_addrs().await.map_err(|e| {
306275
Status::internal(format!("Cannot get metasrv member list, error: {:?}", e))
@@ -316,6 +285,8 @@ impl MetaService for MetaServiceImpl {
316285
&self,
317286
request: Request<Empty>,
318287
) -> Result<Response<ClientInfo>, Status> {
288+
let _guard = WithCount::new((), RequestInFlight);
289+
319290
let r = request.remote_addr();
320291
if let Some(addr) = r {
321292
let resp = ClientInfo {

0 commit comments

Comments
 (0)