Skip to content

Commit 2783378

Browse files
b41shb41sh
authored
refactor: sync create inverted index (#15379)
* refactor: sync create inverted index * fix * fix * fix * fix * add metrics * fix prune inverted index tests * fix
1 parent aa7960a commit 2783378

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

54 files changed

+338
-692
lines changed

‎Cargo.lock

Lines changed: 0 additions & 1 deletion
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎scripts/ci/deploy/config/databend-query-node-share-1.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -142,8 +142,8 @@ table_bloom_index_meta_count = 3000
142142
table_bloom_index_filter_count = 1048576
143143

144144
### table inverted index caches ###
145-
# Max number of cached inverted index info objects. Set it to 0 to disable it.
146-
inverted_index_info_count = 3000
145+
# Max number of cached inverted index meta objects. Set it to 0 to disable it.
146+
inverted_index_meta_count = 3000
147147
# Max bytes of cached inverted index filters. Set it to 0 to disable it.
148148
inverted_index_filter_size = 2147483648
149149

‎scripts/ci/deploy/config/databend-query-node-share-2.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,8 @@ table_bloom_index_meta_count = 3000
143143
table_bloom_index_filter_count = 1048576
144144

145145
### table inverted index caches ###
146-
# Max number of cached inverted index info objects. Set it to 0 to disable it.
147-
inverted_index_info_count = 3000
146+
# Max number of cached inverted index meta objects. Set it to 0 to disable it.
147+
inverted_index_meta_count = 3000
148148
# Max bytes of cached inverted index filters. Set it to 0 to disable it.
149149
inverted_index_filter_size = 2147483648
150150

‎scripts/ci/deploy/config/databend-query-node-share-3.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -143,8 +143,8 @@ table_bloom_index_meta_count = 3000
143143
table_bloom_index_filter_count = 1048576
144144

145145
### table inverted index caches ###
146-
# Max number of cached inverted index info objects. Set it to 0 to disable it.
147-
inverted_index_info_count = 3000
146+
# Max number of cached inverted index meta objects. Set it to 0 to disable it.
147+
inverted_index_meta_count = 3000
148148
# Max bytes of cached inverted index filters. Set it to 0 to disable it.
149149
inverted_index_filter_size = 2147483648
150150

‎src/common/metrics/src/metrics/storage.rs

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,6 +144,22 @@ static BLOCK_INDEX_WRITE_MILLISECONDS: LazyLock<Histogram> =
144144
LazyLock::new(|| register_histogram_in_milliseconds("fuse_block_index_write_milliseconds"));
145145
static BLOCK_INDEX_READ_BYTES: LazyLock<Counter> =
146146
LazyLock::new(|| register_counter("fuse_block_index_read_bytes"));
147+
static BLOCK_INVERTED_INDEX_WRITE_NUMS: LazyLock<Counter> =
148+
LazyLock::new(|| register_counter("fuse_block_inverted_index_write_nums"));
149+
static BLOCK_INVERTED_INDEX_WRITE_BYTES: LazyLock<Counter> =
150+
LazyLock::new(|| register_counter("fuse_block_inverted_index_write_bytes"));
151+
static BLOCK_INVERTED_INDEX_WRITE_MILLISECONDS: LazyLock<Histogram> = LazyLock::new(|| {
152+
register_histogram_in_milliseconds("fuse_block_inverted_index_write_milliseconds")
153+
});
154+
static BLOCK_INVERTED_INDEX_GENERATE_MILLISECONDS: LazyLock<Histogram> = LazyLock::new(|| {
155+
register_histogram_in_milliseconds("fuse_block_inverted_index_generate_milliseconds")
156+
});
157+
static BLOCK_INVERTED_INDEX_READ_MILLISECONDS: LazyLock<Histogram> = LazyLock::new(|| {
158+
register_histogram_in_milliseconds("fuse_block_inverted_index_read_milliseconds")
159+
});
160+
static BLOCK_INVERTED_INDEX_SEARCH_MILLISECONDS: LazyLock<Histogram> = LazyLock::new(|| {
161+
register_histogram_in_milliseconds("fuse_block_inverted_index_search_milliseconds")
162+
});
147163
static COMPACT_BLOCK_READ_NUMS: LazyLock<Counter> =
148164
LazyLock::new(|| register_counter("fuse_compact_block_read_nums"));
149165
static COMPACT_BLOCK_READ_BYTES: LazyLock<Counter> =
@@ -496,6 +512,30 @@ pub fn metrics_inc_block_index_write_milliseconds(c: u64) {
496512
BLOCK_INDEX_WRITE_MILLISECONDS.observe(c as f64);
497513
}
498514

515+
pub fn metrics_inc_block_inverted_index_write_nums(c: u64) {
516+
BLOCK_INVERTED_INDEX_WRITE_NUMS.inc_by(c);
517+
}
518+
519+
pub fn metrics_inc_block_inverted_index_write_bytes(c: u64) {
520+
BLOCK_INVERTED_INDEX_WRITE_BYTES.inc_by(c);
521+
}
522+
523+
pub fn metrics_inc_block_inverted_index_write_milliseconds(c: u64) {
524+
BLOCK_INVERTED_INDEX_WRITE_MILLISECONDS.observe(c as f64);
525+
}
526+
527+
pub fn metrics_inc_block_inverted_index_generate_milliseconds(c: u64) {
528+
BLOCK_INVERTED_INDEX_GENERATE_MILLISECONDS.observe(c as f64);
529+
}
530+
531+
pub fn metrics_inc_block_inverted_index_read_milliseconds(c: u64) {
532+
BLOCK_INVERTED_INDEX_READ_MILLISECONDS.observe(c as f64);
533+
}
534+
535+
pub fn metrics_inc_block_inverted_index_search_milliseconds(c: u64) {
536+
BLOCK_INVERTED_INDEX_SEARCH_MILLISECONDS.observe(c as f64);
537+
}
538+
499539
pub fn metrics_inc_block_index_read_bytes(c: u64) {
500540
BLOCK_INDEX_READ_BYTES.inc_by(c);
501541
}

‎src/query/config/src/config.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2766,13 +2766,13 @@ pub struct CacheConfig {
27662766
)]
27672767
pub table_bloom_index_filter_size: u64,
27682768

2769-
/// Max number of cached inverted index info objects. Set it to 0 to disable it.
2769+
/// Max number of cached inverted index meta objects. Set it to 0 to disable it.
27702770
#[clap(
2771-
long = "cache-inverted-index-info-count",
2771+
long = "cache-inverted-index-meta-count",
27722772
value_name = "VALUE",
27732773
default_value = "3000"
27742774
)]
2775-
pub inverted_index_info_count: u64,
2775+
pub inverted_index_meta_count: u64,
27762776

27772777
/// Max bytes of cached inverted index filters used. Set it to 0 to disable it.
27782778
#[clap(
@@ -2983,7 +2983,7 @@ mod cache_config_converters {
29832983
table_bloom_index_meta_count: value.table_bloom_index_meta_count,
29842984
table_bloom_index_filter_count: value.table_bloom_index_filter_count,
29852985
table_bloom_index_filter_size: value.table_bloom_index_filter_size,
2986-
inverted_index_info_count: value.inverted_index_info_count,
2986+
inverted_index_meta_count: value.inverted_index_meta_count,
29872987
inverted_index_filter_size: value.inverted_index_filter_size,
29882988
inverted_index_filter_memory_ratio: value.inverted_index_filter_memory_ratio,
29892989
table_prune_partitions_count: value.table_prune_partitions_count,
@@ -3008,7 +3008,7 @@ mod cache_config_converters {
30083008
table_bloom_index_meta_count: value.table_bloom_index_meta_count,
30093009
table_bloom_index_filter_count: value.table_bloom_index_filter_count,
30103010
table_bloom_index_filter_size: value.table_bloom_index_filter_size,
3011-
inverted_index_info_count: value.inverted_index_info_count,
3011+
inverted_index_meta_count: value.inverted_index_meta_count,
30123012
inverted_index_filter_size: value.inverted_index_filter_size,
30133013
inverted_index_filter_memory_ratio: value.inverted_index_filter_memory_ratio,
30143014
table_prune_partitions_count: value.table_prune_partitions_count,

‎src/query/config/src/inner.rs

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -541,8 +541,8 @@ pub struct CacheConfig {
541541
// One bloom index filter per column of data block being indexed will be generated if necessary.
542542
pub table_bloom_index_filter_size: u64,
543543

544-
/// Max number of cached inverted index info objects. Set it to 0 to disable it.
545-
pub inverted_index_info_count: u64,
544+
/// Max number of cached inverted index meta objects. Set it to 0 to disable it.
545+
pub inverted_index_meta_count: u64,
546546

547547
/// Max bytes of cached inverted index filters used. Set it to 0 to disable it.
548548
pub inverted_index_filter_size: u64,
@@ -636,7 +636,7 @@ impl Default for CacheConfig {
636636
table_bloom_index_meta_count: 3000,
637637
table_bloom_index_filter_count: 0,
638638
table_bloom_index_filter_size: 2147483648,
639-
inverted_index_info_count: 3000,
639+
inverted_index_meta_count: 3000,
640640
inverted_index_filter_size: 2147483648,
641641
inverted_index_filter_memory_ratio: 0,
642642
table_prune_partitions_count: 256,

‎src/query/ee/tests/it/inverted_index/index_refresh.rs

Lines changed: 8 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,6 @@ use databend_common_exception::Result;
2121
use databend_common_meta_app::schema::CreateOption;
2222
use databend_common_meta_app::schema::CreateTableIndexReq;
2323
use databend_common_sql::plans::RefreshTableIndexPlan;
24-
use databend_common_storages_fuse::io::read::load_inverted_index_info;
2524
use databend_common_storages_fuse::io::read::InvertedIndexReader;
2625
use databend_common_storages_fuse::io::MetaReaders;
2726
use databend_common_storages_fuse::io::TableMetaLocationGenerator;
@@ -74,7 +73,7 @@ async fn test_fuse_do_refresh_inverted_index() -> Result<()> {
7473
table_id,
7574
name: index_name.clone(),
7675
column_ids: vec![0, 1],
77-
sync_creation: true,
76+
sync_creation: false,
7877
options,
7978
};
8079

@@ -100,18 +99,13 @@ async fn test_fuse_do_refresh_inverted_index() -> Result<()> {
10099
let new_snapshot = new_fuse_table.read_table_snapshot().await?;
101100
assert!(new_snapshot.is_some());
102101
let new_snapshot = new_snapshot.unwrap();
103-
assert!(new_snapshot.inverted_indexes.is_some());
104-
105-
let index_info_loc = new_snapshot
106-
.inverted_indexes
107-
.as_ref()
108-
.and_then(|i| i.get(&index_name));
109-
assert!(index_info_loc.is_some());
110-
let index_info =
111-
load_inverted_index_info(new_fuse_table.get_operator(), index_info_loc).await?;
112-
assert!(index_info.is_some());
113-
let index_info = index_info.unwrap();
114-
let index_version = index_info.index_version.clone();
102+
103+
let table_info = new_table.get_table_info();
104+
let table_indexes = &table_info.meta.indexes;
105+
let table_index = table_indexes.get(&index_name);
106+
assert!(table_index.is_some());
107+
let table_index = table_index.unwrap();
108+
let index_version = table_index.version.clone();
115109

116110
let segment_reader =
117111
MetaReaders::segment_info_reader(new_fuse_table.get_operator(), table_schema.clone());

‎src/query/ee/tests/it/inverted_index/pruning.rs

Lines changed: 7 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,6 @@ use databend_common_meta_app::schema::CreateTableIndexReq;
3838
use databend_common_sql::plans::CreateTablePlan;
3939
use databend_common_sql::plans::RefreshTableIndexPlan;
4040
use databend_common_sql::BloomIndexColumns;
41-
use databend_common_storages_fuse::io::read::load_inverted_index_info;
4241
use databend_common_storages_fuse::pruning::create_segment_location_vector;
4342
use databend_common_storages_fuse::pruning::FusePruner;
4443
use databend_common_storages_fuse::FuseTable;
@@ -506,7 +505,7 @@ async fn test_block_pruner() -> Result<()> {
506505
table_id,
507506
name: index_name.clone(),
508507
column_ids: vec![1, 2, 3],
509-
sync_creation: true,
508+
sync_creation: false,
510509
options: index_options.clone(),
511510
};
512511

@@ -537,15 +536,12 @@ async fn test_block_pruner() -> Result<()> {
537536
assert!(snapshot.is_some());
538537
let snapshot = snapshot.unwrap();
539538

540-
let index_info_loc = snapshot
541-
.inverted_indexes
542-
.as_ref()
543-
.and_then(|i| i.get(&index_name));
544-
assert!(index_info_loc.is_some());
545-
let index_info = load_inverted_index_info(fuse_table.get_operator(), index_info_loc).await?;
546-
assert!(index_info.is_some());
547-
let index_info = index_info.unwrap();
548-
let index_version = index_info.index_version.clone();
539+
let table_info = new_table.get_table_info();
540+
let table_indexes = &table_info.meta.indexes;
541+
let table_index = table_indexes.get(&index_name);
542+
assert!(table_index.is_some());
543+
let table_index = table_index.unwrap();
544+
let index_version = table_index.version.clone();
549545

550546
let index_schema = DataSchema::from(index_table_schema);
551547
let e1 = PushDownInfo {

‎src/query/service/src/interpreters/hook/refresh_hook.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -270,7 +270,7 @@ async fn generate_refresh_inverted_index_plan(
270270

271271
let table_meta = &table.get_table_info().meta;
272272
for (_, index) in table_meta.indexes.iter() {
273-
if !index.sync_creation {
273+
if index.sync_creation {
274274
continue;
275275
}
276276
let plan = RefreshTableIndexPlan {

0 commit comments

Comments
 (0)