Skip to content

Commit 533ebff

Browse files
authored
fix: add host id to parquet file paths (#25428)
1 parent eeb1aa7 commit 533ebff

File tree

3 files changed

+21
-30
lines changed

3 files changed

+21
-30
lines changed

influxdb3_write/src/paths.rs

Lines changed: 18 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -52,39 +52,23 @@ impl AsRef<ObjPath> for CatalogFilePath {
5252
pub struct ParquetFilePath(ObjPath);
5353

5454
impl ParquetFilePath {
55+
/// Generate a parquet file path using the given arguments. This will convert the provided
56+
/// `chunk_time` into a date time string with format `'YYYY-MM-DD/HH-MM'`
5557
pub fn new(
5658
host_prefix: &str,
57-
db_name: &str,
58-
db_id: u32,
59-
table_name: &str,
60-
table_id: u32,
61-
date: DateTime<Utc>,
62-
wal_file_sequence_number: WalFileSequenceNumber,
63-
) -> Self {
64-
let path = ObjPath::from(format!(
65-
"{host_prefix}/dbs/{db_name}-{db_id}/{table_name}-{table_id}/{}/{}.{}",
66-
date.format("%Y-%m-%d/%H-%M"),
67-
wal_file_sequence_number.as_u64(),
68-
PARQUET_FILE_EXTENSION
69-
));
70-
Self(path)
71-
}
72-
73-
pub fn new_with_chunk_time(
7459
db_name: &str,
7560
db_id: u32,
7661
table_name: &str,
7762
table_id: u32,
7863
chunk_time: i64,
7964
wal_file_sequence_number: WalFileSequenceNumber,
8065
) -> Self {
81-
// Convert the chunk time into a date time string for YYYY-MM-DDTHH-MM
8266
let date_time = DateTime::<Utc>::from_timestamp_nanos(chunk_time);
8367
let path = ObjPath::from(format!(
84-
"dbs/{db_name}-{db_id}/{table_name}-{table_id}/{}/{:010}.{}",
85-
date_time.format("%Y-%m-%d/%H-%M"),
86-
wal_file_sequence_number.as_u64(),
87-
PARQUET_FILE_EXTENSION
68+
"{host_prefix}/dbs/{db_name}-{db_id}/{table_name}-{table_id}/{date_string}/{wal_seq:010}.{ext}",
69+
date_string = date_time.format("%Y-%m-%d/%H-%M"),
70+
wal_seq = wal_file_sequence_number.as_u64(),
71+
ext = PARQUET_FILE_EXTENSION
8872
));
8973
Self(path)
9074
}
@@ -153,10 +137,13 @@ fn parquet_file_path_new() {
153137
0,
154138
"my_table",
155139
0,
156-
Utc.with_ymd_and_hms(2038, 1, 19, 3, 14, 7).unwrap(),
157-
WalFileSequenceNumber::new(0),
140+
Utc.with_ymd_and_hms(2038, 1, 19, 3, 14, 7)
141+
.unwrap()
142+
.timestamp_nanos_opt()
143+
.unwrap(),
144+
WalFileSequenceNumber::new(1337),
158145
),
159-
ObjPath::from("my_host/dbs/my_db-0/my_table-0/2038-01-19/03-14/0.parquet")
146+
ObjPath::from("my_host/dbs/my_db-0/my_table-0/2038-01-19/03-14/0000001337.parquet")
160147
);
161148
}
162149

@@ -169,12 +156,15 @@ fn parquet_file_percent_encoded() {
169156
0,
170157
"..",
171158
0,
172-
Utc.with_ymd_and_hms(2038, 1, 19, 3, 14, 7).unwrap(),
173-
WalFileSequenceNumber::new(0),
159+
Utc.with_ymd_and_hms(2038, 1, 19, 3, 14, 7)
160+
.unwrap()
161+
.timestamp_nanos_opt()
162+
.unwrap(),
163+
WalFileSequenceNumber::new(100),
174164
)
175165
.as_ref()
176166
.as_ref(),
177-
"%2E%2E/dbs/..-0/..-0/2038-01-19/03-14/0.parquet"
167+
"%2E%2E/dbs/..-0/..-0/2038-01-19/03-14/0000000100.parquet"
178168
);
179169
}
180170

influxdb3_write/src/persister.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -726,7 +726,7 @@ mod tests {
726726
0,
727727
"table_one",
728728
0,
729-
Utc::now(),
729+
Utc::now().timestamp_nanos_opt().unwrap(),
730730
WalFileSequenceNumber::new(1),
731731
);
732732
let (bytes_written, meta) = persister

influxdb3_write/src/write_buffer/queryable_buffer.rs

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -167,7 +167,8 @@ impl QueryableBuffer {
167167
table_id: *table_id,
168168
table_name: Arc::clone(&table_name),
169169
chunk_time: chunk.chunk_time,
170-
path: ParquetFilePath::new_with_chunk_time(
170+
path: ParquetFilePath::new(
171+
self.persister.host_identifier_prefix(),
171172
db_name.as_ref(),
172173
database_id.as_u32(),
173174
table_name.as_ref(),

0 commit comments

Comments
 (0)