Skip to content

Commit 04a4929

Browse files
drmingdrmerdrmingdrmer
authored
chore: simplify create_data_mask (#16309)
* chore: move types in mod seqv into separate files * refactor: get_background_job() returns SeqV<T> This way the caller will be able to update the record against the exact seq number. * refactor: add kvapi::Pair * chore: simplify create_data_mask
1 parent 4a6ac70 commit 04a4929

File tree

89 files changed

+357
-291
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+357
-291
lines changed

‎src/meta/api/src/background_api_impl.rs

Lines changed: 12 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -43,23 +43,20 @@ use databend_common_meta_app::KeyWithTenant;
4343
use databend_common_meta_kvapi::kvapi;
4444
use databend_common_meta_kvapi::kvapi::Key;
4545
use databend_common_meta_kvapi::kvapi::UpsertKVReq;
46+
use databend_common_meta_types::seq_value::SeqValue;
4647
use databend_common_meta_types::ConditionResult::Eq;
4748
use databend_common_meta_types::InvalidReply;
48-
use databend_common_meta_types::MatchSeq;
4949
use databend_common_meta_types::MatchSeq::Any;
5050
use databend_common_meta_types::MetaError;
5151
use databend_common_meta_types::MetaSpec;
5252
use databend_common_meta_types::Operation;
53-
use databend_common_meta_types::SeqValue;
5453
use databend_common_meta_types::TxnRequest;
55-
use databend_common_meta_types::With;
5654
use fastrace::func_name;
5755
use log::debug;
5856

5957
use crate::background_api::BackgroundApi;
6058
use crate::deserialize_struct;
6159
use crate::fetch_id;
62-
use crate::get_pb_value;
6360
use crate::get_u64_value;
6461
use crate::kv_app_error::KVAppError;
6562
use crate::kv_pb_api::KVPbApi;
@@ -200,10 +197,10 @@ impl<KV: kvapi::KVApi<Error = MetaError>> BackgroundApi for KV {
200197

201198
let name_key = &req.name;
202199

203-
let (id_ident, _, job) =
200+
let (id_ident, seq_joq) =
204201
get_background_job_or_error(self, name_key, format!("get_: {:?}", name_key)).await?;
205202

206-
Ok(GetBackgroundJobReply::new(id_ident, job))
203+
Ok(GetBackgroundJobReply::new(id_ident, seq_joq))
207204
}
208205

209206
#[fastrace::trace]
@@ -315,14 +312,15 @@ async fn get_background_job_or_error(
315312
kv_api: &(impl kvapi::KVApi<Error = MetaError> + ?Sized),
316313
name_ident: &BackgroundJobIdent,
317314
_msg: impl Display,
318-
) -> Result<(BackgroundJobIdIdent, u64, BackgroundJobInfo), KVAppError> {
315+
) -> Result<kvapi::Pair<BackgroundJobIdIdent>, KVAppError> {
319316
let id_ident = get_background_job_id(kv_api, name_ident).await?;
320317

321-
let (id_seq, job_info) = get_pb_value(kv_api, &id_ident).await?;
322-
assert_background_job_exist(id_seq, name_ident)?;
318+
let seq_job = kv_api
319+
.get_pb(&id_ident)
320+
.await?
321+
.ok_or_else(|| unknown_background_job(name_ident))?;
323322

324-
// Safe unwrap(): background_job_seq > 0 implies background_job is not None.
325-
Ok((id_ident, id_seq, job_info.unwrap()))
323+
Ok((id_ident, seq_job))
326324
}
327325

328326
/// Return OK if a db_id or db_meta exists by checking the seq.
@@ -354,15 +352,15 @@ async fn update_background_job<F: FnOnce(&mut BackgroundJobInfo) -> bool>(
354352
mutation: F,
355353
) -> Result<UpdateBackgroundJobReply, KVAppError> {
356354
debug!(req :? =(name); "BackgroundApi: {}", func_name!());
357-
let (id_ident, id_val_seq, mut info) =
355+
let (id_ident, mut seq_job) =
358356
get_background_job_or_error(kv_api, name, "update_background_job").await?;
359357

360-
let should_update = mutation(&mut info);
358+
let should_update = mutation(&mut seq_job.data);
361359
if !should_update {
362360
return Ok(UpdateBackgroundJobReply::new(id_ident.clone()));
363361
}
364362

365-
let req = UpsertPB::update(id_ident.clone(), info).with(MatchSeq::Exact(id_val_seq));
363+
let req = UpsertPB::update_exact(id_ident.clone(), seq_job);
366364
let resp = kv_api.upsert_pb(&req).await?;
367365

368366
assert!(resp.is_changed());

‎src/meta/api/src/background_api_test_suite.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -231,7 +231,7 @@ impl BackgroundApiTestSuite {
231231
let res = res.unwrap();
232232
assert_eq!(
233233
BackgroundJobState::RUNNING,
234-
res.info.job_status.unwrap().job_state,
234+
res.info.data.job_status.unwrap().job_state,
235235
"first state is started"
236236
);
237237
}
@@ -320,7 +320,7 @@ impl BackgroundApiTestSuite {
320320
);
321321
assert_eq!(
322322
Some("newid".to_string()),
323-
res.info.job_status.unwrap().last_task_id
323+
res.info.data.job_status.unwrap().last_task_id
324324
)
325325
}
326326

‎src/meta/api/src/crud/mod.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -28,11 +28,11 @@ use databend_common_meta_app::tenant_key::resource::TenantResource;
2828
use databend_common_meta_kvapi::kvapi;
2929
use databend_common_meta_kvapi::kvapi::DirName;
3030
use databend_common_meta_kvapi::kvapi::ValueWithName;
31+
use databend_common_meta_types::seq_value::SeqV;
32+
use databend_common_meta_types::seq_value::SeqValue;
3133
use databend_common_meta_types::MatchSeq;
3234
use databend_common_meta_types::MatchSeqExt;
3335
use databend_common_meta_types::MetaError;
34-
use databend_common_meta_types::SeqV;
35-
use databend_common_meta_types::SeqValue;
3636
use databend_common_meta_types::With;
3737
use databend_common_proto_conv::FromToProto;
3838
pub use errors::CrudError;

‎src/meta/api/src/data_mask_api_impl.rs

Lines changed: 45 additions & 60 deletions
Original file line numberDiff line numberDiff line change
@@ -34,11 +34,8 @@ use databend_common_meta_app::schema::TableId;
3434
use databend_common_meta_app::schema::TableMeta;
3535
use databend_common_meta_app::KeyWithTenant;
3636
use databend_common_meta_kvapi::kvapi;
37-
use databend_common_meta_types::ConditionResult::Eq;
3837
use databend_common_meta_types::MetaError;
3938
use databend_common_meta_types::SeqValue;
40-
use databend_common_meta_types::TxnCondition;
41-
use databend_common_meta_types::TxnOp;
4239
use databend_common_meta_types::TxnRequest;
4340
use fastrace::func_name;
4441
use log::debug;
@@ -54,7 +51,6 @@ use crate::serialize_struct;
5451
use crate::serialize_u64;
5552
use crate::txn_backoff::txn_backoff;
5653
use crate::txn_cond_eq_seq;
57-
use crate::txn_cond_seq;
5854
use crate::txn_op_del;
5955
use crate::txn_op_put;
6056

@@ -78,8 +74,7 @@ impl<KV: kvapi::KVApi<Error = MetaError>> DatamaskApi for KV {
7874
let (seq, id) = get_u64_value(self, name_ident).await?;
7975
debug!(seq = seq, id = id, name_key :? =(name_ident); "create_data_mask");
8076

81-
let mut condition = vec![];
82-
let mut if_then = vec![];
77+
let mut txn = TxnRequest::default();
8378

8479
if seq > 0 {
8580
match req.create_option {
@@ -99,8 +94,7 @@ impl<KV: kvapi::KVApi<Error = MetaError>> DatamaskApi for KV {
9994
false,
10095
false,
10196
func_name!(),
102-
&mut condition,
103-
&mut if_then,
97+
&mut txn,
10498
)
10599
.await?;
106100
}
@@ -126,20 +120,14 @@ impl<KV: kvapi::KVApi<Error = MetaError>> DatamaskApi for KV {
126120
{
127121
let meta: DatamaskMeta = req.clone().into();
128122
let id_list = MaskpolicyTableIdList::default();
129-
condition.push(txn_cond_seq(name_ident, Eq, seq));
130-
if_then.extend( vec![
123+
txn.condition.push(txn_cond_eq_seq(name_ident, seq));
124+
txn.if_then.extend( vec![
131125
txn_op_put(name_ident, serialize_u64(id)?), // name -> db_id
132126
txn_op_put(&id_ident, serialize_struct(&meta)?), // id -> meta
133127
txn_op_put(&id_list_key, serialize_struct(&id_list)?), /* data mask name -> id_list */
134128
]);
135129

136-
let txn_req = TxnRequest {
137-
condition,
138-
if_then,
139-
else_then: vec![],
140-
};
141-
142-
let (succ, _responses) = send_txn(self, txn_req).await?;
130+
let (succ, _responses) = send_txn(self, txn).await?;
143131

144132
debug!(
145133
name :? =(name_ident),
@@ -166,25 +154,19 @@ impl<KV: kvapi::KVApi<Error = MetaError>> DatamaskApi for KV {
166154
loop {
167155
trials.next().unwrap()?.await;
168156

169-
let mut condition = vec![];
170-
let mut if_then = vec![];
157+
let mut txn = TxnRequest::default();
171158

172159
construct_drop_mask_policy_operations(
173160
self,
174161
name_key,
175162
req.if_exists,
176163
true,
177164
func_name!(),
178-
&mut condition,
179-
&mut if_then,
165+
&mut txn,
180166
)
181167
.await?;
182-
let txn_req = TxnRequest {
183-
condition,
184-
if_then,
185-
else_then: vec![],
186-
};
187-
let (succ, _responses) = send_txn(self, txn_req).await?;
168+
169+
let (succ, _responses) = send_txn(self, txn).await?;
188170

189171
debug!(
190172
succ = succ;
@@ -253,35 +235,38 @@ pub fn assert_data_mask_exist(
253235
async fn clear_table_column_mask_policy(
254236
kv_api: &(impl kvapi::KVApi<Error = MetaError> + ?Sized),
255237
name_ident: &DataMaskNameIdent,
256-
condition: &mut Vec<TxnCondition>,
257-
if_then: &mut Vec<TxnOp>,
238+
txn: &mut TxnRequest,
258239
) -> Result<(), KVAppError> {
259240
let id_list_key = MaskPolicyTableIdListIdent::new_from(name_ident.clone());
260241

261-
let (id_list_seq, id_list_opt): (_, Option<MaskpolicyTableIdList>) =
262-
get_pb_value(kv_api, &id_list_key).await?;
263-
if let Some(id_list) = id_list_opt {
264-
condition.push(txn_cond_seq(&id_list_key, Eq, id_list_seq));
265-
if_then.push(txn_op_del(&id_list_key));
266-
267-
// remove mask policy from table meta
268-
for table_id in id_list.id_list.into_iter() {
269-
let tbid = TableId { table_id };
270-
271-
let (tb_meta_seq, table_meta_opt): (_, Option<TableMeta>) =
272-
get_pb_value(kv_api, &tbid).await?;
273-
if let Some(mut table_meta) = table_meta_opt {
274-
if let Some(column_mask_policy) = table_meta.column_mask_policy {
275-
let new_column_mask_policy = column_mask_policy
276-
.into_iter()
277-
.filter(|(_, name)| name != name_ident.name())
278-
.collect();
279-
280-
table_meta.column_mask_policy = Some(new_column_mask_policy);
281-
282-
condition.push(txn_cond_seq(&tbid, Eq, tb_meta_seq));
283-
if_then.push(txn_op_put(&tbid, serialize_struct(&table_meta)?));
284-
}
242+
let seq_id_list = kv_api.get_pb(&id_list_key).await?;
243+
244+
let Some(seq_id_list) = seq_id_list else {
245+
return Ok(());
246+
};
247+
248+
txn.condition
249+
.push(txn_cond_eq_seq(&id_list_key, seq_id_list.seq));
250+
txn.if_then.push(txn_op_del(&id_list_key));
251+
252+
// remove mask policy from table meta
253+
for table_id in seq_id_list.data.id_list.into_iter() {
254+
let tbid = TableId { table_id };
255+
256+
let (tb_meta_seq, table_meta_opt): (_, Option<TableMeta>) =
257+
get_pb_value(kv_api, &tbid).await?;
258+
if let Some(mut table_meta) = table_meta_opt {
259+
if let Some(column_mask_policy) = table_meta.column_mask_policy {
260+
let new_column_mask_policy = column_mask_policy
261+
.into_iter()
262+
.filter(|(_, name)| name != name_ident.name())
263+
.collect();
264+
265+
table_meta.column_mask_policy = Some(new_column_mask_policy);
266+
267+
txn.condition.push(txn_cond_eq_seq(&tbid, tb_meta_seq));
268+
txn.if_then
269+
.push(txn_op_put(&tbid, serialize_struct(&table_meta)?));
285270
}
286271
}
287272
}
@@ -295,8 +280,7 @@ async fn construct_drop_mask_policy_operations(
295280
drop_if_exists: bool,
296281
if_delete: bool,
297282
ctx: &str,
298-
condition: &mut Vec<TxnCondition>,
299-
if_then: &mut Vec<TxnOp>,
283+
txn: &mut TxnRequest,
300284
) -> Result<(), KVAppError> {
301285
let result = get_data_mask_or_err(
302286
kv_api,
@@ -320,13 +304,14 @@ async fn construct_drop_mask_policy_operations(
320304

321305
let id_ident = DataMaskIdIdent::new(name_key.tenant(), id);
322306

323-
condition.push(txn_cond_eq_seq(&id_ident, data_mask_seq));
324-
if_then.push(txn_op_del(&id_ident));
307+
txn.condition
308+
.push(txn_cond_eq_seq(&id_ident, data_mask_seq));
309+
txn.if_then.push(txn_op_del(&id_ident));
325310

326311
if if_delete {
327-
condition.push(txn_cond_eq_seq(name_key, id_seq));
328-
if_then.push(txn_op_del(name_key));
329-
clear_table_column_mask_policy(kv_api, name_key, condition, if_then).await?;
312+
txn.condition.push(txn_cond_eq_seq(name_key, id_seq));
313+
txn.if_then.push(txn_op_del(name_key));
314+
clear_table_column_mask_policy(kv_api, name_key, txn).await?;
330315
}
331316

332317
debug!(

‎src/meta/api/src/kv_pb_api/codec.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,9 @@
1515
use databend_common_meta_kvapi::kvapi;
1616
use databend_common_meta_kvapi::kvapi::NonEmptyItem;
1717
use databend_common_meta_types::protobuf::StreamItem;
18+
use databend_common_meta_types::seq_value::SeqV;
1819
use databend_common_meta_types::Change;
1920
use databend_common_meta_types::Operation;
20-
use databend_common_meta_types::SeqV;
2121
use databend_common_proto_conv::FromToProto;
2222

2323
use crate::kv_pb_api::errors::NoneValue;

‎src/meta/api/src/kv_pb_api/mod.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,8 @@ use databend_common_meta_kvapi::kvapi::DirName;
2626
use databend_common_meta_kvapi::kvapi::KVApi;
2727
use databend_common_meta_kvapi::kvapi::NonEmptyItem;
2828
use databend_common_meta_types::protobuf::StreamItem;
29+
use databend_common_meta_types::seq_value::SeqV;
2930
use databend_common_meta_types::Change;
30-
use databend_common_meta_types::SeqV;
3131
use databend_common_meta_types::UpsertKV;
3232
use databend_common_proto_conv::FromToProto;
3333
use futures::future::FutureExt;
@@ -341,9 +341,9 @@ mod tests {
341341
use databend_common_meta_kvapi::kvapi::UpsertKVReply;
342342
use databend_common_meta_kvapi::kvapi::UpsertKVReq;
343343
use databend_common_meta_types::protobuf::StreamItem;
344+
use databend_common_meta_types::seq_value::SeqV;
345+
use databend_common_meta_types::seq_value::SeqValue;
344346
use databend_common_meta_types::MetaError;
345-
use databend_common_meta_types::SeqV;
346-
use databend_common_meta_types::SeqValue;
347347
use databend_common_meta_types::TxnReply;
348348
use databend_common_meta_types::TxnRequest;
349349
use databend_common_proto_conv::FromToProto;

‎src/meta/api/src/kv_pb_api/upsert_pb.rs

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ use databend_common_meta_kvapi::kvapi;
1818
use databend_common_meta_types::MatchSeq;
1919
use databend_common_meta_types::MetaSpec;
2020
use databend_common_meta_types::Operation;
21+
use databend_common_meta_types::SeqV;
2122
use databend_common_meta_types::With;
2223

2324
#[derive(Clone, Debug)]
@@ -70,6 +71,16 @@ impl<K: kvapi::Key> UpsertPB<K> {
7071
}
7172
}
7273

74+
/// Update the value only when the seq matches exactly. Note that the meta is not copied.
75+
pub fn update_exact(key: K, value: SeqV<K::ValueType>) -> Self {
76+
Self {
77+
key,
78+
seq: MatchSeq::Exact(value.seq),
79+
value: Operation::Update(value.data),
80+
value_meta: None,
81+
}
82+
}
83+
7384
pub fn delete(key: K) -> Self {
7485
Self {
7586
key,

‎src/meta/api/src/schema_api.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,9 +104,9 @@ use databend_common_meta_app::schema::UpdateVirtualColumnReq;
104104
use databend_common_meta_app::schema::UpsertTableOptionReply;
105105
use databend_common_meta_app::schema::UpsertTableOptionReq;
106106
use databend_common_meta_app::schema::VirtualColumnMeta;
107+
use databend_common_meta_types::seq_value::SeqV;
107108
use databend_common_meta_types::MetaError;
108109
use databend_common_meta_types::MetaId;
109-
use databend_common_meta_types::SeqV;
110110

111111
use crate::kv_app_error::KVAppError;
112112

‎src/meta/api/src/schema_api_impl.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -207,19 +207,19 @@ use databend_common_meta_kvapi::kvapi::DirName;
207207
use databend_common_meta_kvapi::kvapi::Key;
208208
use databend_common_meta_kvapi::kvapi::UpsertKVReq;
209209
use databend_common_meta_types::protobuf as pb;
210+
use databend_common_meta_types::seq_value::KVMeta;
211+
use databend_common_meta_types::seq_value::SeqV;
212+
use databend_common_meta_types::seq_value::SeqValue;
210213
use databend_common_meta_types::txn_op::Request;
211214
use databend_common_meta_types::txn_op_response::Response;
212215
use databend_common_meta_types::ConditionResult;
213216
use databend_common_meta_types::InvalidReply;
214-
use databend_common_meta_types::KVMeta;
215217
use databend_common_meta_types::MatchSeq;
216218
use databend_common_meta_types::MatchSeqExt;
217219
use databend_common_meta_types::MetaError;
218220
use databend_common_meta_types::MetaId;
219221
use databend_common_meta_types::MetaNetworkError;
220222
use databend_common_meta_types::Operation;
221-
use databend_common_meta_types::SeqV;
222-
use databend_common_meta_types::SeqValue;
223223
use databend_common_meta_types::TxnCondition;
224224
use databend_common_meta_types::TxnGetRequest;
225225
use databend_common_meta_types::TxnGetResponse;

0 commit comments

Comments
 (0)