Skip to content

Commit 77248da

Browse files
drmingdrmerdrmingdrmer
authored
refactor: replace absolute expiration time with relative TTL in schema API (#16268)
* refactor: replace absolute expiration time with relative TTL in schema API The use of absolute expiration time can lead to issues due to local time discrepancies when calling `SchemaAPI`. If the local time lags behind the meta-service leader's time, records may expire prematurely. By switching to a relative expiration time (`ttl`), the expiration is calculated by the meta-service leader, which synchronizes the time through the Raft log. This ensures that records expire correctly. * chore: enable INFO log for meta to address the TableLock issue https://github.com/datafuselabs/databend/actions/runs/10389326033/job/28775319145?pr=16207#step:4:1201 * chore: SchemaApi::update_multi_table_meta() should use ttl to replace absolute expiration time
1 parent 9197028 commit 77248da

File tree

11 files changed

+40
-42
lines changed

11 files changed

+40
-42
lines changed

‎scripts/ci/deploy/databend-query-management-mode.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ done
2323
sleep 1
2424

2525
echo 'Start databend-meta...'
26-
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=ERROR &
26+
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
2727
echo "Waiting on databend-meta 10 seconds..."
2828
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
2929

‎scripts/ci/deploy/databend-query-standalone-hive.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ done
2323
sleep 1
2424

2525
echo 'Start databend-meta...'
26-
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=ERROR &
26+
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
2727
echo "Waiting on databend-meta 10 seconds..."
2828
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
2929

‎scripts/ci/deploy/databend-query-standalone-logging.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ done
2323
sleep 1
2424

2525
echo 'Start databend-meta...'
26-
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=ERROR &
26+
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
2727
echo "Waiting on databend-meta 10 seconds..."
2828
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
2929

‎scripts/ci/deploy/databend-query-standalone-native.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ done
2323
sleep 1
2424

2525
echo 'Start databend-meta...'
26-
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=ERROR &
26+
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
2727
echo "Waiting on databend-meta 10 seconds..."
2828
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
2929

‎scripts/ci/deploy/databend-query-standalone.sh

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ done
2323
sleep 1
2424

2525
echo 'Start databend-meta...'
26-
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=ERROR &
26+
nohup target/${BUILD_PROFILE}/databend-meta --single --log-level=INFO &
2727
echo "Waiting on databend-meta 10 seconds..."
2828
python3 scripts/ci/wait_tcp.py --timeout 30 --port 9191
2929

‎src/meta/api/src/lib.rs

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,5 +76,4 @@ pub use util::txn_cond_seq;
7676
pub use util::txn_op_del;
7777
pub use util::txn_op_get;
7878
pub use util::txn_op_put;
79-
pub use util::txn_op_put_with_expire;
8079
pub use util::DEFAULT_MGET_SIZE;

‎src/meta/api/src/schema_api_impl.rs

Lines changed: 18 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -257,7 +257,6 @@ use crate::txn_cond_seq;
257257
use crate::txn_op_del;
258258
use crate::txn_op_get;
259259
use crate::txn_op_put;
260-
use crate::txn_op_put_with_expire;
261260
use crate::util::db_id_has_to_exist;
262261
use crate::util::deserialize_id_get_response;
263262
use crate::util::deserialize_struct_get_response;
@@ -3794,10 +3793,10 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
37943793
txn_cond_seq(&key, Eq, 0),
37953794
];
37963795

3797-
let if_then = vec![txn_op_put_with_expire(
3798-
&key,
3796+
let if_then = vec![TxnOp::put_with_ttl(
3797+
key.to_string_key(),
37993798
serialize_struct(&lock_meta)?,
3800-
SeqV::<()>::now_ms() / 1000 + req.expire_secs,
3799+
Some(req.expire_secs * 1000),
38013800
)];
38023801

38033802
let txn_req = TxnRequest {
@@ -3859,10 +3858,10 @@ impl<KV: kvapi::KVApi<Error = MetaError> + ?Sized> SchemaApi for KV {
38593858
txn_cond_seq(&key, Eq, lock_seq),
38603859
];
38613860

3862-
let if_then = vec![txn_op_put_with_expire(
3863-
&key,
3861+
let if_then = vec![TxnOp::put_with_ttl(
3862+
key.to_string_key(),
38643863
serialize_struct(&lock_meta)?,
3865-
SeqV::<()>::now_ms() / 1000 + req.expire_secs,
3864+
Some(req.expire_secs * 1000),
38663865
)];
38673866

38683867
let txn_req = TxnRequest {
@@ -5226,28 +5225,31 @@ fn build_upsert_table_copied_file_info_conditions(
52265225
// "fail_if_duplicated" mode, assumes files are absent
52275226
condition.push(txn_cond_seq(&key, Eq, 0));
52285227
}
5229-
set_update_expire_operation(&key, &file_info, &req.expire_at, &mut if_then)?;
5228+
set_update_expire_operation(&key, &file_info, req.ttl, &mut if_then)?;
52305229
}
52315230
Ok((condition, if_then))
52325231
}
52335232

52345233
fn build_upsert_table_deduplicated_label(deduplicated_label: String) -> TxnOp {
5235-
let expire_at = Some(SeqV::<()>::now_ms() / 1000 + 24 * 60 * 60);
5236-
TxnOp::put_with_expire(deduplicated_label, 1_i8.to_le_bytes().to_vec(), expire_at)
5234+
TxnOp::put_with_ttl(
5235+
deduplicated_label,
5236+
1_i8.to_le_bytes().to_vec(),
5237+
Some(86400 * 1000),
5238+
)
52375239
}
52385240

52395241
fn set_update_expire_operation(
52405242
key: &TableCopiedFileNameIdent,
52415243
file_info: &TableCopiedFileInfo,
5242-
expire_at_opt: &Option<u64>,
5244+
ttl: Option<std::time::Duration>,
52435245
then_branch: &mut Vec<TxnOp>,
52445246
) -> Result<(), KVAppError> {
5245-
match expire_at_opt {
5246-
Some(expire_at) => {
5247-
then_branch.push(txn_op_put_with_expire(
5248-
key,
5247+
match ttl {
5248+
Some(ttl) => {
5249+
then_branch.push(TxnOp::put_with_ttl(
5250+
key.to_string_key(),
52495251
serialize_struct(file_info)?,
5250-
*expire_at,
5252+
Some(ttl.as_millis() as u64),
52515253
));
52525254
}
52535255
None => {

‎src/meta/api/src/schema_api_test_suite.rs

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2663,7 +2663,7 @@ impl SchemaApiTestSuite {
26632663

26642664
let upsert_source_table = UpsertTableCopiedFileReq {
26652665
file_info,
2666-
expire_at: None,
2666+
ttl: None,
26672667
fail_if_duplicated: true,
26682668
};
26692669

@@ -2713,7 +2713,7 @@ impl SchemaApiTestSuite {
27132713

27142714
let upsert_source_table = UpsertTableCopiedFileReq {
27152715
file_info,
2716-
expire_at: None,
2716+
ttl: None,
27172717
fail_if_duplicated: true,
27182718
};
27192719
let req = UpdateTableMetaReq {
@@ -2762,7 +2762,7 @@ impl SchemaApiTestSuite {
27622762

27632763
let upsert_source_table = UpsertTableCopiedFileReq {
27642764
file_info,
2765-
expire_at: None,
2765+
ttl: None,
27662766
fail_if_duplicated: true,
27672767
};
27682768
let req = UpdateTableMetaReq {
@@ -3614,7 +3614,7 @@ impl SchemaApiTestSuite {
36143614

36153615
let copied_file_req = UpsertTableCopiedFileReq {
36163616
file_info: file_info.clone(),
3617-
expire_at: Some((Utc::now().timestamp() + 86400) as u64),
3617+
ttl: Some(std::time::Duration::from_secs(86400)),
36183618
fail_if_duplicated: true,
36193619
};
36203620

@@ -3778,7 +3778,7 @@ impl SchemaApiTestSuite {
37783778

37793779
let copied_file_req = UpsertTableCopiedFileReq {
37803780
file_info: file_info.clone(),
3781-
expire_at: Some((Utc::now().timestamp() + 86400) as u64),
3781+
ttl: Some(std::time::Duration::from_secs(86400)),
37823782
fail_if_duplicated: true,
37833783
};
37843784

@@ -5733,7 +5733,7 @@ impl SchemaApiTestSuite {
57335733

57345734
let copied_file_req = UpsertTableCopiedFileReq {
57355735
file_info: file_info.clone(),
5736-
expire_at: Some((Utc::now().timestamp() + 86400) as u64),
5736+
ttl: Some(std::time::Duration::from_secs(86400)),
57375737
fail_if_duplicated: true,
57385738
};
57395739

@@ -5782,7 +5782,8 @@ impl SchemaApiTestSuite {
57825782

57835783
let copied_file_req = UpsertTableCopiedFileReq {
57845784
file_info: file_info.clone(),
5785-
expire_at: Some((Utc::now().timestamp() - 86400) as u64),
5785+
// Make it expire at once.
5786+
ttl: Some(std::time::Duration::from_secs(0)),
57865787
fail_if_duplicated: true,
57875788
};
57885789

@@ -7208,7 +7209,7 @@ impl SchemaApiTestSuite {
72087209

72097210
let copied_file_req = UpsertTableCopiedFileReq {
72107211
file_info: file_info.clone(),
7211-
expire_at: Some((Utc::now().timestamp() + 86400) as u64),
7212+
ttl: Some(std::time::Duration::from_secs(86400)),
72127213
fail_if_duplicated: true,
72137214
};
72147215

@@ -7266,7 +7267,7 @@ impl SchemaApiTestSuite {
72667267

72677268
let copied_file_req = UpsertTableCopiedFileReq {
72687269
file_info: file_info.clone(),
7269-
expire_at: Some((Utc::now().timestamp() + 86400) as u64),
7270+
ttl: Some(std::time::Duration::from_secs(86400)),
72707271
fail_if_duplicated: true,
72717272
};
72727273

@@ -7321,7 +7322,7 @@ impl SchemaApiTestSuite {
73217322

73227323
let copied_file_req = UpsertTableCopiedFileReq {
73237324
file_info: file_info.clone(),
7324-
expire_at: Some((Utc::now().timestamp() + 86400) as u64),
7325+
ttl: Some(std::time::Duration::from_secs(86400)),
73257326
fail_if_duplicated: false,
73267327
};
73277328

@@ -7680,7 +7681,7 @@ where MT: SchemaApi + kvapi::AsKVApi<Error = MetaError>
76807681

76817682
let copied_file_req = UpsertTableCopiedFileReq {
76827683
file_info: file_infos.clone(),
7683-
expire_at: Some((Utc::now().timestamp() + 86400) as u64),
7684+
ttl: Some(std::time::Duration::from_secs(86400)),
76847685
fail_if_duplicated: true,
76857686
};
76867687

‎src/meta/api/src/util.rs

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -359,11 +359,6 @@ pub fn txn_op_get(key: &impl kvapi::Key) -> TxnOp {
359359
TxnOp::get(key.to_string_key())
360360
}
361361

362-
// TODO: replace it with common_meta_types::with::With
363-
pub fn txn_op_put_with_expire(key: &impl kvapi::Key, value: Vec<u8>, expire_at: u64) -> TxnOp {
364-
TxnOp::put_with_expire(key.to_string_key(), value, Some(expire_at))
365-
}
366-
367362
/// Build a txn operation that deletes a record.
368363
pub fn txn_op_del(key: &impl kvapi::Key) -> TxnOp {
369364
TxnOp::delete(key.to_string_key())

‎src/meta/app/src/schema/table.rs

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ use std::fmt::Display;
2020
use std::fmt::Formatter;
2121
use std::ops::Deref;
2222
use std::sync::Arc;
23+
use std::time::Duration;
2324

2425
use anyerror::func_name;
2526
use chrono::DateTime;
@@ -983,7 +984,8 @@ pub struct GetTableCopiedFileReply {
983984
#[derive(Clone, Debug, PartialEq, Eq)]
984985
pub struct UpsertTableCopiedFileReq {
985986
pub file_info: BTreeMap<String, TableCopiedFileInfo>,
986-
pub expire_at: Option<u64>,
987+
/// If not None, specifies the time-to-live for the keys.
988+
pub ttl: Option<Duration>,
987989
pub fail_if_duplicated: bool,
988990
}
989991

0 commit comments

Comments
 (0)