refactor: optimize file reading for disk-based cache on Unix (#17638)

* refactor: Optimize file reading for disk-based cache on Unix

For Unix platforms (`#[cfg(unix)]`):
- Open files with the `O_NOATIME` flag to avoid updating access time.
- Leverage known file size to streamline reading.
- Bypass `stat` and `seek` operations used by `std::fs::File::read_to_end`.

* fix: comment typo

* fix: add licenses header

* fix: cargo fmt

Author: dantengsky

Cargo.lock

@@ -31,7 +31,16 @@ rustix = { workspace = true }
 siphasher = { workspace = true }
 
 [dev-dependencies]
+criterion = { workspace = true }
+mockall = { workspace = true }
 tempfile = { workspace = true }
 
+[[bench]]
+name = "read_cache_content"
+harness = false
+
+[target.'cfg(unix)'.dependencies]
+libc = { workspace = true }
+
 [lints]
 workspace = true

src/query/storages/common/cache/Cargo.toml

@@ -0,0 +1,114 @@
+// Copyright 2021 Datafuse Labs
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+use std::fs::File;
+use std::io::Write;
+use std::path::PathBuf;
+
+use criterion::black_box;
+use criterion::criterion_group;
+use criterion::criterion_main;
+use criterion::Criterion;
+use databend_storages_common_cache::read_cache_content;
+use tempfile::TempDir;
+
+fn create_test_file(dir: &TempDir, name: &str, size: usize) -> PathBuf {
+    let path = dir.path().join(name);
+    let mut file = File::create(&path).unwrap();
+    let content = vec![42u8; size];
+    file.write_all(&content).unwrap();
+    path
+}
+
+fn bench_read_cache_content(c: &mut Criterion) {
+    let temp_dir = TempDir::new().unwrap();
+
+    let small_file_path = create_test_file(&temp_dir, "small.bin", 4 * 1024); // 4KB
+    let medium_file_path = create_test_file(&temp_dir, "medium.bin", 64 * 1024); // 64KB
+    let large_file_path = create_test_file(&temp_dir, "large.bin", 1024 * 1024); // 1MB
+
+    // Benchmark Unix specific implementation (open with O_NOATIME, read with libc::read without stat / seek)
+    #[cfg(unix)]
+    {
+        let mut group = c.benchmark_group("read_cache_content_unix_specific_impl");
+
+        group.bench_function("small_file_4KB", |b| {
+            b.iter(|| {
+                let path = small_file_path.clone();
+                let result = read_cache_content(path, 4 * 1024);
+                black_box(result)
+            })
+        });
+
+        group.bench_function("medium_file_64KB", |b| {
+            b.iter(|| {
+                let path = medium_file_path.clone();
+                let result = read_cache_content(path, 64 * 1024);
+                black_box(result)
+            })
+        });
+
+        group.bench_function("large_file_1MB", |b| {
+            b.iter(|| {
+                let path = large_file_path.clone();
+                let result = read_cache_content(path, 1024 * 1024);
+                black_box(result)
+            })
+        });
+
+        group.finish();
+    }
+
+    // Benchmark using read_to_end from std lib
+    let mut group = c.benchmark_group("read_cache_std_lib_read_to_end");
+
+    group.bench_function("small_file_4KB", |b| {
+        b.iter(|| {
+            let path = small_file_path.clone();
+            let result = fallback_std_lib_read_to_end(path, 4 * 1024);
+            black_box(result)
+        })
+    });
+
+    group.bench_function("medium_file_64KB", |b| {
+        b.iter(|| {
+            let path = medium_file_path.clone();
+            let result = fallback_std_lib_read_to_end(path, 64 * 1024);
+            black_box(result)
+        })
+    });
+
+    group.bench_function("large_file_1MB", |b| {
+        b.iter(|| {
+            let path = large_file_path.clone();
+            let result = fallback_std_lib_read_to_end(path, 1024 * 1024);
+            black_box(result)
+        })
+    });
+
+    group.finish();
+}
+
+fn fallback_std_lib_read_to_end(path: PathBuf, size: usize) -> std::io::Result<Vec<u8>> {
+    use std::fs::File;
+    use std::io::Read;
+
+    let mut v = Vec::with_capacity(size);
+    let mut file = File::open(path)?;
+    file.read_to_end(&mut v)?;
+    Ok(v)
+}
+
+criterion_group!(benches, bench_read_cache_content);
+criterion_main!(benches);

src/query/storages/common/cache/benches/read_cache_content.rs

@@ -31,6 +31,8 @@ pub use caches::CachedObject;
 pub use caches::SegmentBlockMetasCache;
 pub use caches::SizedColumnArray;
 pub use manager::CacheManager;
+// Unfortunately, criterion benchmarks need it to be pub
+pub use providers::disk_cache::disk_cache_lru::read_cache_content;
 pub use providers::DiskCacheError;
 pub use providers::DiskCacheKey;
 pub use providers::DiskCacheResult;

src/query/storages/common/cache/src/lib.rs

@@ -321,12 +321,12 @@ impl DiskCache {
         self.cache.contains(&cache_key.0)
     }
 
-    pub fn get_cache_path(&mut self, key: &str) -> Option<PathBuf> {
+    pub fn get_cache_item_path_and_size(&mut self, key: &str) -> Option<(PathBuf, u64)> {
         let cache_key = self.cache_key(key);
         self.cache
             .get(&cache_key.0)
-            .map(|_| ()) // release the &mut self
-            .map(|_| self.abs_path_of_cache_key(&cache_key))
+            .map(|item| item.get_inner().0)
+            .map(|size| (self.abs_path_of_cache_key(&cache_key), size))
     }
 
     /// Remove the given key from the cache.