Skip to content

Commit 8f8bcfc

Browse files
borsbors
committed
Auto merge of #16335 - lnicola:salsa-lz4-file-text, r=Veykril
internal: Compress file text using LZ4 I haven't tested properly, but this roughly looks like: ``` 1246 MB 59mb 4899 FileTextQuery 1008 MB 20mb 4899 CompressedFileTextQuery 555kb 1790 FileTextQuery ``` We might want to test on something more interesting, like `bevy`.
2 parents 2f87215 + 717ba1d commit 8f8bcfc

File tree

16 files changed

+89
-36
lines changed

16 files changed

+89
-36
lines changed

‎Cargo.lock

Lines changed: 11 additions & 4 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎crates/base-db/Cargo.toml

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ rust-version.workspace = true
1212
doctest = false
1313

1414
[dependencies]
15+
lz4_flex = { version = "0.11", default-features = false }
16+
1517
la-arena.workspace = true
1618
salsa.workspace = true
1719
rustc-hash.workspace = true

‎crates/base-db/src/change.rs

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,13 +7,13 @@ use salsa::Durability;
77
use triomphe::Arc;
88
use vfs::FileId;
99

10-
use crate::{CrateGraph, SourceDatabaseExt, SourceRoot, SourceRootId};
10+
use crate::{CrateGraph, SourceDatabaseExt, SourceDatabaseExt2, SourceRoot, SourceRootId};
1111

1212
/// Encapsulate a bunch of raw `.set` calls on the database.
1313
#[derive(Default)]
1414
pub struct FileChange {
1515
pub roots: Option<Vec<SourceRoot>>,
16-
pub files_changed: Vec<(FileId, Option<Arc<str>>)>,
16+
pub files_changed: Vec<(FileId, Option<String>)>,
1717
pub crate_graph: Option<CrateGraph>,
1818
}
1919

@@ -42,7 +42,7 @@ impl FileChange {
4242
self.roots = Some(roots);
4343
}
4444

45-
pub fn change_file(&mut self, file_id: FileId, new_text: Option<Arc<str>>) {
45+
pub fn change_file(&mut self, file_id: FileId, new_text: Option<String>) {
4646
self.files_changed.push((file_id, new_text))
4747
}
4848

@@ -68,8 +68,8 @@ impl FileChange {
6868
let source_root = db.source_root(source_root_id);
6969
let durability = durability(&source_root);
7070
// XXX: can't actually remove the file, just reset the text
71-
let text = text.unwrap_or_else(|| Arc::from(""));
72-
db.set_file_text_with_durability(file_id, text, durability)
71+
let text = text.unwrap_or_default();
72+
db.set_file_text_with_durability(file_id, &text, durability)
7373
}
7474
if let Some(crate_graph) = self.crate_graph {
7575
db.set_crate_graph_with_durability(Arc::new(crate_graph), Durability::HIGH);

‎crates/base-db/src/lib.rs

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ mod input;
77

88
use std::panic;
99

10+
use salsa::Durability;
1011
use syntax::{ast, Parse, SourceFile};
1112
use triomphe::Arc;
1213

@@ -42,6 +43,7 @@ pub trait Upcast<T: ?Sized> {
4243
fn upcast(&self) -> &T;
4344
}
4445

46+
pub const DEFAULT_FILE_TEXT_LRU_CAP: usize = 16;
4547
pub const DEFAULT_PARSE_LRU_CAP: usize = 128;
4648
pub const DEFAULT_BORROWCK_LRU_CAP: usize = 1024;
4749

@@ -89,7 +91,10 @@ fn parse(db: &dyn SourceDatabase, file_id: FileId) -> Parse<ast::SourceFile> {
8991
#[salsa::query_group(SourceDatabaseExtStorage)]
9092
pub trait SourceDatabaseExt: SourceDatabase {
9193
#[salsa::input]
94+
fn compressed_file_text(&self, file_id: FileId) -> Arc<[u8]>;
95+
9296
fn file_text(&self, file_id: FileId) -> Arc<str>;
97+
9398
/// Path to a file, relative to the root of its source root.
9499
/// Source root of the file.
95100
#[salsa::input]
@@ -101,6 +106,44 @@ pub trait SourceDatabaseExt: SourceDatabase {
101106
fn source_root_crates(&self, id: SourceRootId) -> Arc<[CrateId]>;
102107
}
103108

109+
fn file_text(db: &dyn SourceDatabaseExt, file_id: FileId) -> Arc<str> {
110+
let bytes = db.compressed_file_text(file_id);
111+
let bytes =
112+
lz4_flex::decompress_size_prepended(&bytes).expect("lz4 decompression should not fail");
113+
let text = std::str::from_utf8(&bytes).expect("file contents should be valid UTF-8");
114+
Arc::from(text)
115+
}
116+
117+
pub trait SourceDatabaseExt2 {
118+
fn set_file_text(&mut self, file_id: FileId, text: &str) {
119+
self.set_file_text_with_durability(file_id, text, Durability::LOW);
120+
}
121+
122+
fn set_file_text_with_durability(
123+
&mut self,
124+
file_id: FileId,
125+
text: &str,
126+
durability: Durability,
127+
);
128+
}
129+
130+
impl<Db: ?Sized + SourceDatabaseExt> SourceDatabaseExt2 for Db {
131+
fn set_file_text_with_durability(
132+
&mut self,
133+
file_id: FileId,
134+
text: &str,
135+
durability: Durability,
136+
) {
137+
let bytes = text.as_bytes();
138+
let compressed = lz4_flex::compress_prepend_size(bytes);
139+
self.set_compressed_file_text_with_durability(
140+
file_id,
141+
Arc::from(compressed.as_slice()),
142+
durability,
143+
)
144+
}
145+
}
146+
104147
fn source_root_crates(db: &dyn SourceDatabaseExt, id: SourceRootId) -> Arc<[CrateId]> {
105148
let graph = db.crate_graph();
106149
let mut crates = graph

‎crates/hir-def/src/nameres/tests/incremental.rs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
use base_db::{SourceDatabase, SourceDatabaseExt};
1+
use base_db::{SourceDatabase, SourceDatabaseExt2 as _};
22
use test_fixture::WithFixture;
3-
use triomphe::Arc;
43

54
use crate::{db::DefDatabase, nameres::tests::TestDB, AdtId, ModuleDefId};
65

@@ -17,7 +16,7 @@ fn check_def_map_is_not_recomputed(ra_fixture_initial: &str, ra_fixture_change:
1716
});
1817
assert!(format!("{events:?}").contains("crate_def_map"), "{events:#?}")
1918
}
20-
db.set_file_text(pos.file_id, Arc::from(ra_fixture_change));
19+
db.set_file_text(pos.file_id, ra_fixture_change);
2120

2221
{
2322
let events = db.log_executed(|| {
@@ -267,7 +266,7 @@ fn quux() { 92 }
267266
m!(Y);
268267
m!(Z);
269268
"#;
270-
db.set_file_text(pos.file_id, Arc::from(new_text));
269+
db.set_file_text(pos.file_id, new_text);
271270

272271
{
273272
let events = db.log_executed(|| {

‎crates/hir-expand/src/change.rs

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ impl ChangeWithProcMacros {
4848
}
4949
}
5050

51-
pub fn change_file(&mut self, file_id: FileId, new_text: Option<Arc<str>>) {
51+
pub fn change_file(&mut self, file_id: FileId, new_text: Option<String>) {
5252
self.source_change.change_file(file_id, new_text)
5353
}
5454

‎crates/hir-ty/src/tests.rs

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ mod traits;
1212

1313
use std::env;
1414

15-
use base_db::{FileRange, SourceDatabaseExt};
15+
use base_db::{FileRange, SourceDatabaseExt2 as _};
1616
use expect_test::Expect;
1717
use hir_def::{
1818
body::{Body, BodySourceMap, SyntheticSyntax},
@@ -584,7 +584,7 @@ fn salsa_bug() {
584584
}
585585
";
586586

587-
db.set_file_text(pos.file_id, Arc::from(new_text));
587+
db.set_file_text(pos.file_id, new_text);
588588

589589
let module = db.module_for_file(pos.file_id);
590590
let crate_def_map = module.def_map(&db);

‎crates/hir-ty/src/tests/incremental.rs

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
1-
use base_db::SourceDatabaseExt;
1+
use base_db::SourceDatabaseExt2 as _;
22
use test_fixture::WithFixture;
3-
use triomphe::Arc;
43

54
use crate::{db::HirDatabase, test_db::TestDB};
65

@@ -33,7 +32,7 @@ fn foo() -> i32 {
3332
1
3433
}";
3534

36-
db.set_file_text(pos.file_id, Arc::from(new_text));
35+
db.set_file_text(pos.file_id, new_text);
3736

3837
{
3938
let events = db.log_executed(|| {
@@ -85,7 +84,7 @@ fn baz() -> i32 {
8584
}
8685
";
8786

88-
db.set_file_text(pos.file_id, Arc::from(new_text));
87+
db.set_file_text(pos.file_id, new_text);
8988

9089
{
9190
let events = db.log_executed(|| {

‎crates/ide-db/src/apply_change.rs

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -205,6 +205,7 @@ impl RootDatabase {
205205

206206
// SourceDatabaseExt
207207
base_db::FileTextQuery
208+
base_db::CompressedFileTextQuery
208209
base_db::FileSourceRootQuery
209210
base_db::SourceRootQuery
210211
base_db::SourceRootCratesQuery

‎crates/ide-db/src/lib.rs

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ use std::{fmt, mem::ManuallyDrop};
5151
use base_db::{
5252
salsa::{self, Durability},
5353
AnchoredPath, CrateId, FileId, FileLoader, FileLoaderDelegate, SourceDatabase, Upcast,
54+
DEFAULT_FILE_TEXT_LRU_CAP,
5455
};
5556
use hir::db::{DefDatabase, ExpandDatabase, HirDatabase};
5657
use triomphe::Arc;
@@ -157,6 +158,7 @@ impl RootDatabase {
157158

158159
pub fn update_base_query_lru_capacities(&mut self, lru_capacity: Option<usize>) {
159160
let lru_capacity = lru_capacity.unwrap_or(base_db::DEFAULT_PARSE_LRU_CAP);
161+
base_db::FileTextQuery.in_db_mut(self).set_lru_capacity(DEFAULT_FILE_TEXT_LRU_CAP);
160162
base_db::ParseQuery.in_db_mut(self).set_lru_capacity(lru_capacity);
161163
// macro expansions are usually rather small, so we can afford to keep more of them alive
162164
hir::db::ParseMacroExpansionQuery.in_db_mut(self).set_lru_capacity(4 * lru_capacity);
@@ -166,6 +168,7 @@ impl RootDatabase {
166168
pub fn update_lru_capacities(&mut self, lru_capacities: &FxHashMap<Box<str>, usize>) {
167169
use hir::db as hir_db;
168170

171+
base_db::FileTextQuery.in_db_mut(self).set_lru_capacity(DEFAULT_FILE_TEXT_LRU_CAP);
169172
base_db::ParseQuery.in_db_mut(self).set_lru_capacity(
170173
lru_capacities
171174
.get(stringify!(ParseQuery))
@@ -199,7 +202,7 @@ impl RootDatabase {
199202
// base_db::ProcMacrosQuery
200203

201204
// SourceDatabaseExt
202-
// base_db::FileTextQuery
205+
base_db::FileTextQuery
203206
// base_db::FileSourceRootQuery
204207
// base_db::SourceRootQuery
205208
base_db::SourceRootCratesQuery

0 commit comments

Comments
 (0)