From 8452f9e24cd2dca6a0dc05e3d1f9fed22f5987c6 Mon Sep 17 00:00:00 2001 From: Luca Ongaro Date: Wed, 2 Oct 2024 15:19:19 +0200 Subject: [PATCH 1/4] Add CubDB.writes_since_compaction function Returning the number of writes since the last successful compaction operation --- lib/cubdb.ex | 20 ++++++++++++++++++++ test/cubdb_test.exs | 3 +++ 2 files changed, 23 insertions(+) diff --git a/lib/cubdb.ex b/lib/cubdb.ex index 3e137f7..b51b974 100644 --- a/lib/cubdb.ex +++ b/lib/cubdb.ex @@ -750,6 +750,19 @@ defmodule CubDB do GenServer.call(db, :dirt_factor, :infinity) end + @spec writes_since_compaction(server) :: non_neg_integer + + @doc """ + Returns the number of writes performed since the last compaction. + + This number is reset to 0 after a compaction operation, and increases by one + on each write operation on the BTree. Deletions are considered writes, as they + modify the BTree. + """ + def writes_since_compaction(db) do + GenServer.call(db, :writes_since_compaction, :infinity) + end + @spec put(server, key, value) :: :ok @doc """ @@ -1306,6 +1319,13 @@ defmodule CubDB do {:reply, Btree.dirt_factor(btree), state} end + def handle_call(:writes_since_compaction, _, state) do + %State{btree: btree} = state + %Btree{dirt: dirt} = btree + + {:reply, dirt, state} + end + def handle_call(:start_transaction, from, state = %State{writer: nil}) do %State{btree: btree} = state diff --git a/test/cubdb_test.exs b/test/cubdb_test.exs index dd7ba20..160087d 100644 --- a/test/cubdb_test.exs +++ b/test/cubdb_test.exs @@ -984,13 +984,16 @@ defmodule CubDBTest do {:ok, db} = CubDB.start_link(tmp_dir, auto_compact: {3, 0.3}) assert CubDB.dirt_factor(db) == 0 + assert CubDB.writes_since_compaction(db) == 0 CubDB.subscribe(db) CubDB.put(db, :a, 1) + assert CubDB.writes_since_compaction(db) == 1 refute_received :compaction_started CubDB.put(db, :b, 2) + assert CubDB.writes_since_compaction(db) == 2 refute_received :compaction_started CubDB.put(db, :a, 3) From 61d68d992e2150f37b8f8a54b20504bf4752540c Mon Sep 17 00:00:00 2001 From: Luca Ongaro Date: Wed, 2 Oct 2024 15:21:21 +0200 Subject: [PATCH 2/4] Update CHANGELOG --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index e4e299f..bdd556f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,10 @@ releases](https://github.com/lucaong/cubdb/releases). Since `v1.0.0`, `CubDB` follows [semantic versioning](https://semver.org), and reports changes here. +## Unreleased + + - Add function `CubDB.writes_since_compaction/1` to get the number of writes since the last successful compaction + ## v2.0.2 (2023-01-01) Bug fixes: From ea626844a6d4041e4a0da2e7986062f6520b691b Mon Sep 17 00:00:00 2001 From: Luca Ongaro Date: Wed, 2 Oct 2024 15:36:10 +0200 Subject: [PATCH 3/4] Fix dirt calculation across restarts --- CHANGELOG.md | 4 ++++ lib/cubdb/btree.ex | 2 +- test/cubdb_test.exs | 20 ++++++++++++++++++++ 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bdd556f..f41c076 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,10 @@ reports changes here. - Add function `CubDB.writes_since_compaction/1` to get the number of writes since the last successful compaction +Bug fixes: + + - Fix dirt calculation upon restart of `CubDB` + ## v2.0.2 (2023-01-01) Bug fixes: diff --git a/lib/cubdb/btree.ex b/lib/cubdb/btree.ex index b02f4c7..403e943 100644 --- a/lib/cubdb/btree.ex +++ b/lib/cubdb/btree.ex @@ -244,7 +244,7 @@ defmodule CubDB.Btree do # updates won't be committed to the database and will be lost in case of a # restart. def commit(tree = %Btree{store: store, size: size, root_loc: root_loc, dirt: dirt}) do - Store.put_header(store, header(size: size, location: root_loc, dirt: dirt + 1)) + Store.put_header(store, header(size: size, location: root_loc, dirt: dirt)) tree end diff --git a/test/cubdb_test.exs b/test/cubdb_test.exs index 160087d..e7b4bc0 100644 --- a/test/cubdb_test.exs +++ b/test/cubdb_test.exs @@ -1003,6 +1003,26 @@ defmodule CubDBTest do refute_received :compaction_started end + test "dirt calculation is consistent after restart", %{tmp_dir: tmp_dir} do + {:ok, db} = CubDB.start_link(data_dir: tmp_dir, auto_compact: false) + + CubDB.put(db, "foo", 123) + CubDB.put(db, "bar", 234) + CubDB.delete(db, "foo") + CubDB.clear(db) + + original_dirt_factor = CubDB.dirt_factor(db) + original_writes_since_compaction = CubDB.writes_since_compaction(db) + + assert original_writes_since_compaction == 4 + + CubDB.stop(db) + + {:ok, db} = CubDB.start_link(data_dir: tmp_dir, auto_compact: false) + assert original_dirt_factor == CubDB.dirt_factor(db) + assert original_writes_since_compaction == CubDB.writes_since_compaction(db) + end + test "auto compaction is active by default", %{tmp_dir: tmp_dir} do {:ok, db} = CubDB.start_link(tmp_dir) From 168cffa61586fd67d7afeac2ae99148f2bdbb0d9 Mon Sep 17 00:00:00 2001 From: Luca Ongaro Date: Wed, 2 Oct 2024 15:50:57 +0200 Subject: [PATCH 4/4] Improve docs --- lib/cubdb.ex | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/cubdb.ex b/lib/cubdb.ex index b51b974..d769cb5 100644 --- a/lib/cubdb.ex +++ b/lib/cubdb.ex @@ -757,7 +757,15 @@ defmodule CubDB do This number is reset to 0 after a compaction operation, and increases by one on each write operation on the BTree. Deletions are considered writes, as they - modify the BTree. + modify the BTree. The way writes are computed depends on the specific BTree + implementation, and might not map one-to-one to the high level write + operations performed by the user. This metric is provided only as a hint in + case of manual compaction, where it is often a good idea to skip a wasteful + compaction if only a few write operations were performed since the last + compaction. + + The number of writes does not include uncommitted operations, therefore old + halted transactions that are never committed are not counted. """ def writes_since_compaction(db) do GenServer.call(db, :writes_since_compaction, :infinity)