diff --git a/lib/cubdb.ex b/lib/cubdb.ex index 3e137f7..12d6597 100644 --- a/lib/cubdb.ex +++ b/lib/cubdb.ex @@ -168,6 +168,7 @@ defmodule CubDB do @auto_compact_defaults {100, 0.25} @type key :: any + @type metadata_key :: atom @type value :: any @type entry :: {key, value} @type auto_compact :: {pos_integer, number} | boolean @@ -317,6 +318,24 @@ defmodule CubDB do end) end + @spec get_metadata(server, metadata_key, value) :: value | {:error, :key_not_atom} + + @doc """ + Gets the value associated to `key` from the database metadata table. + + If no value is associated with `key`, `default` is returned (which is `nil`, + unless specified otherwise). + """ + def get_metadata(db, key, default \\ nil) + + def get_metadata(db, key, default) when is_atom(key) do + with_snapshot(db, fn %Snapshot{btree: btree} -> + Reader.get_metadata(btree, key, default) + end) + end + + def get_metadata(_, _, _), do: {:error, :key_not_atom} + @spec fetch(server, key) :: {:ok, value} | :error @doc """ @@ -784,6 +803,22 @@ defmodule CubDB do end) end + @spec put_metadata(server, metadata_key, value) :: :ok | {:error, :key_not_atom} + + @doc """ + Writes an entry in the database metadata table, associating `key` to `value`, + overwriting any existing entries associated with the same key. + + Returns `{:error, :key_not_atom}` if `key` is not an atom. + """ + def put_metadata(db, key, value) when is_atom(key) do + transaction(db, fn tx -> + {:commit, Tx.put_metadata(tx, key, value), :ok} + end) + end + + def put_metadata(_, _, _), do: {:error, :key_not_atom} + @spec delete(server, key) :: :ok @doc """ @@ -797,6 +832,23 @@ defmodule CubDB do end) end + @spec delete_metadata(server, metadata_key) :: :ok | {:error, :key_not_atom} + + @doc """ + Deletes the entry associated to `key` from the database metadata table. + + If `key` was not present in the database, nothing is done. + + Returns `{:error, :key_not_atom}` if `key` is not an atom. + """ + def delete_metadata(db, key) when is_atom(key) do + transaction(db, fn tx -> + {:commit, Tx.delete_metadata(tx, key), :ok} + end) + end + + def delete_metadata(_, _), do: {:error, :key_not_atom} + @spec update(server, key, value, (value -> value)) :: :ok @doc """ diff --git a/lib/cubdb/btree.ex b/lib/cubdb/btree.ex index b02f4c7..e877895 100644 --- a/lib/cubdb/btree.ex +++ b/lib/cubdb/btree.ex @@ -30,6 +30,7 @@ defmodule CubDB.Btree do @type btree_size :: non_neg_integer @type dirt :: non_neg_integer @type location :: non_neg_integer + @type metadata :: keyword @type capacity :: pos_integer @type child_pointer :: {key, location} @type leaf_node :: record(:leaf, children: [child_pointer]) @@ -51,18 +52,19 @@ defmodule CubDB.Btree do size: btree_size, dirt: dirt, store: Store.t(), - capacity: capacity + capacity: capacity, + metadata: metadata } @default_capacity 32 - @enforce_keys [:root, :root_loc, :size, :dirt, :store, :capacity] + @enforce_keys [:root, :root_loc, :size, :dirt, :store, :capacity, :metadata] defstruct @enforce_keys - @spec header(size: btree_size, location: location, dirt: dirt) :: Macro.t() + @spec header(size: btree_size, location: location, dirt: dirt, metadata: metadata) :: Macro.t() - defmacro header(size: size, location: location, dirt: dirt) do + defmacro header(size: size, location: location, dirt: dirt, metadata: metadata) do quote do - {unquote(size), unquote(location), unquote(dirt)} + {unquote(size), unquote(location), unquote(dirt), unquote(metadata)} end end @@ -70,15 +72,33 @@ defmodule CubDB.Btree do def new(store, cap \\ @default_capacity) do case Store.get_latest_header(store) do - {_, header(size: s, location: loc, dirt: dirt)} -> + {_, header(size: s, location: loc, dirt: dirt, metadata: metadata)} -> root = Store.get_node(store, loc) - %Btree{root: root, root_loc: loc, dirt: dirt, size: s, capacity: cap, store: store} + + %Btree{ + root: root, + root_loc: loc, + dirt: dirt, + size: s, + capacity: cap, + store: store, + metadata: metadata + } nil -> root = leaf() loc = Store.put_node(store, root) - Store.put_header(store, header(size: 0, location: loc, dirt: 0)) - %Btree{root: root, root_loc: loc, dirt: 0, size: 0, capacity: cap, store: store} + Store.put_header(store, header(size: 0, location: loc, dirt: 0, metadata: [])) + + %Btree{ + root: root, + root_loc: loc, + dirt: 0, + size: 0, + capacity: cap, + store: store, + metadata: [] + } end end @@ -94,6 +114,8 @@ defmodule CubDB.Btree do unless Store.blank?(store), do: raise(ArgumentError, message: "cannot load into non-empty store") + metadata = if match?(%Btree{}, enum), do: enum.metadata, else: [] + {st, count} = Enum.reduce(enum, {[], 0}, fn {k, v}, {st, count} -> {load_node(store, k, value(val: v), st, 1, cap), count + 1} @@ -103,8 +125,17 @@ defmodule CubDB.Btree do new(store, cap) else {root, root_loc} = finalize_load(store, st, 1, cap) - Store.put_header(store, header(size: count, location: root_loc, dirt: 0)) - %Btree{root: root, root_loc: root_loc, capacity: cap, store: store, size: count, dirt: 0} + Store.put_header(store, header(size: count, location: root_loc, dirt: 0, metadata: [])) + + %Btree{ + root: root, + root_loc: root_loc, + capacity: cap, + store: store, + size: count, + dirt: 0, + metadata: metadata + } end end @@ -127,6 +158,17 @@ defmodule CubDB.Btree do end end + @spec fetch_metadata(Btree.t(), key) :: {:ok, val} | :error + + # `fetch_metadata/2` returns `{:ok, value}` if an entry with key `key` is present + # in the Btree metadata table, otherwise `:error`. + def fetch_metadata(%Btree{metadata: metadata}, key) do + case Keyword.get(metadata, key) do + nil -> :error + value -> {:ok, value} + end + end + @spec written_since?(Btree.t(), key, Btree.t()) :: boolean | {:maybe, :not_found} | {:maybe, :different_store} @@ -174,13 +216,37 @@ defmodule CubDB.Btree do insert_terminal_node(btree, key, value(val: value), false) end + @spec insert_metadata(Btree.t(), key, val) :: Btree.t() + + # `insert_metadata/3` inserts the key-value pair into the Btree metadata table, + # overwriting any existing values associated with the same key. It does not commit + # the operation, so `commit/1` must be explicitly called to commit the insertion. + def insert_metadata(btree, key, value) do + %Btree{metadata: metadata} = btree + metadata = Keyword.put(metadata, key, value) + %{btree | metadata: metadata} + end + + @spec delete_metadata(Btree.t(), key) :: Btree.t() + + # `delete_metadata/2` deletes the entry associated to `key` in the Btree metadata + # table, if it exists, otherwise it does nothing. It does not commit the operation, + # so `commit/1` must be explicitly called to commit the deletion. + def delete_metadata(btree, key) do + %Btree{metadata: metadata} = btree + metadata = Keyword.delete(metadata, key) + %{btree | metadata: metadata} + end + @spec delete(Btree.t(), key) :: Btree.t() # `delete/2` deletes the entry associated to `key` in the Btree, if existing. # It does not commit the operation, so `commit/1` must be explicitly called to # commit the deletion. def delete(btree, key) do - %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt} = btree + %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt, metadata: metadata} = + btree + {leaf = {@leaf, children}, path} = lookup_leaf(root, store, key, []) case List.keyfind(children, key, 0) do @@ -199,7 +265,8 @@ defmodule CubDB.Btree do capacity: cap, store: store, size: size, - dirt: dirt + 1 + dirt: dirt + 1, + metadata: metadata } nil -> @@ -227,11 +294,20 @@ defmodule CubDB.Btree do @spec clear(Btree.t()) :: Btree.t() def clear(btree) do - %Btree{store: store, capacity: cap, dirt: dirt} = btree + %Btree{store: store, capacity: cap, dirt: dirt, metadata: metadata} = btree root = leaf() loc = Store.put_node(store, root) - %Btree{root: root, root_loc: loc, size: 0, dirt: dirt + 1, capacity: cap, store: store} + + %Btree{ + root: root, + root_loc: loc, + size: 0, + dirt: dirt + 1, + capacity: cap, + store: store, + metadata: metadata + } end @spec commit(Btree.t()) :: Btree.t() @@ -243,8 +319,20 @@ defmodule CubDB.Btree do # If one or more updates are performed, but `commit/1` is not called, the # updates won't be committed to the database and will be lost in case of a # restart. - def commit(tree = %Btree{store: store, size: size, root_loc: root_loc, dirt: dirt}) do - Store.put_header(store, header(size: size, location: root_loc, dirt: dirt + 1)) + def commit( + tree = %Btree{ + store: store, + size: size, + root_loc: root_loc, + dirt: dirt, + metadata: metadata + } + ) do + Store.put_header( + store, + header(size: size, location: root_loc, dirt: dirt + 1, metadata: metadata) + ) + tree end @@ -293,7 +381,8 @@ defmodule CubDB.Btree do Btree.t() | {:error, :exists} defp insert_terminal_node(btree, key, terminal_node, overwrite \\ true) do - %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt} = btree + %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt, metadata: metadata} = + btree {leaf = {@leaf, children}, path} = lookup_leaf(root, store, key, []) was_set = child_is_set?(store, children, key) @@ -317,7 +406,8 @@ defmodule CubDB.Btree do capacity: cap, store: store, size: size, - dirt: dirt + 1 + dirt: dirt + 1, + metadata: metadata } end end diff --git a/lib/cubdb/compactor.ex b/lib/cubdb/compactor.ex index 9de81ed..6b87ce4 100644 --- a/lib/cubdb/compactor.ex +++ b/lib/cubdb/compactor.ex @@ -51,7 +51,7 @@ defmodule CubDB.Compactor do result = CubDB.transaction(db, fn %Tx{btree: ^original_btree} = tx -> - {:commit, %Tx{tx | btree: compacted_btree}, :done} + {:commit, %Tx{tx | btree: %{compacted_btree | metadata: tx.btree.metadata}}, :done} %Tx{btree: latest_btree} -> {:cancel, latest_btree} diff --git a/lib/cubdb/reader.ex b/lib/cubdb/reader.ex index def4d91..3173cf0 100644 --- a/lib/cubdb/reader.ex +++ b/lib/cubdb/reader.ex @@ -16,6 +16,15 @@ defmodule CubDB.Reader do end end + @spec get_metadata(Btree.t(), CubDB.key(), any) :: any + + def get_metadata(btree, key, default) do + case Btree.fetch_metadata(btree, key) do + {:ok, value} -> value + :error -> default + end + end + @spec get_multi(Btree.t(), [CubDB.key()]) :: %{CubDB.key() => CubDB.value()} def get_multi(btree, keys) do diff --git a/lib/cubdb/store/file.ex b/lib/cubdb/store/file.ex index 51a0deb..35ca249 100644 --- a/lib/cubdb/store/file.ex +++ b/lib/cubdb/store/file.ex @@ -222,6 +222,7 @@ defimpl CubDB.Store, for: CubDB.Store.File do defp read_header(file, location) do case read_term(file, location) do + {:ok, {size, loc, dirt}} -> {location, {size, loc, dirt, []}} {:ok, term} -> {location, term} {:error, _} -> get_latest_good_header(file, location - 1) end diff --git a/lib/cubdb/transaction.ex b/lib/cubdb/transaction.ex index 533d66f..8c1029b 100644 --- a/lib/cubdb/transaction.ex +++ b/lib/cubdb/transaction.ex @@ -47,6 +47,11 @@ defmodule CubDB.Tx do Reader.get(btree, key, default) end + def get_metadata(tx = %Tx{btree: btree}, key, default \\ nil) do + validate_transaction!(tx) + Reader.get_metadata(btree, key, default) + end + @spec fetch(Tx.t(), CubDB.key()) :: {:ok, CubDB.value()} | :error @doc """ @@ -252,6 +257,11 @@ defmodule CubDB.Tx do end end + def put_metadata(tx = %Tx{btree: btree}, key, value) do + validate_transaction!(tx) + %{tx | btree: Btree.insert_metadata(btree, key, value)} + end + @spec delete(Tx.t(), CubDB.key()) :: Tx.t() @doc """ @@ -272,6 +282,11 @@ defmodule CubDB.Tx do end end + def delete_metadata(tx = %Tx{btree: btree}, key) do + validate_transaction!(tx) + %{tx | btree: Btree.delete_metadata(btree, key)} + end + @spec clear(Tx.t()) :: Tx.t() @doc """ diff --git a/mix.exs b/mix.exs index 51b2f77..2e1d3eb 100644 --- a/mix.exs +++ b/mix.exs @@ -1,8 +1,9 @@ defmodule CubDB.Mixfile do use Mix.Project - @source_url "https://github.com/lucaong/cubdb" - @version "2.0.2" + @source_url "https://github.com/zteln/cubdb" + @forked_url "https://github.com/lucaong/cubdb" + @version "2.0.5" def project do [ @@ -58,13 +59,14 @@ defmodule CubDB.Mixfile do defp package() do [ + name: :cubdb_md_fork, description: "A pure-Elixir embedded key-value database", files: ["lib", "LICENSE", "mix.exs"], maintainers: ["Luca Ongaro"], licenses: ["Apache-2.0"], links: %{ - "Changelog" => "https://hexdocs.pm/cubdb/changelog.html", - "GitHub" => @source_url + "GitHub" => @source_url, + "Forked from" => @forked_url } ] end diff --git a/test/cubdb/btree_test.exs b/test/cubdb/btree_test.exs index 0850a18..15516d9 100644 --- a/test/cubdb/btree_test.exs +++ b/test/cubdb/btree_test.exs @@ -14,19 +14,46 @@ defmodule CubDB.BtreeTest do def compose_btree do {:ok, store} = Store.TestStore.create() {root_loc, root} = Utils.load(store, {:Btree, 0, Btree.leaf()}) - %Btree{root: root, root_loc: root_loc, capacity: 3, store: store, size: 0, dirt: 0} + + %Btree{ + root: root, + root_loc: root_loc, + capacity: 3, + store: store, + size: 0, + dirt: 0, + metadata: [] + } end def compose_btree(root = leaf(children: cs)) do {:ok, store} = Store.TestStore.create() {root_loc, root} = Utils.load(store, {:Btree, length(cs), root}) - %Btree{root: root, root_loc: root_loc, capacity: 3, store: store, size: length(cs), dirt: 0} + + %Btree{ + root: root, + root_loc: root_loc, + capacity: 3, + store: store, + size: length(cs), + dirt: 0, + metadata: [] + } end def compose_btree(root = branch(children: _), size \\ 0) do {:ok, store} = Store.TestStore.create() {root_loc, root} = Utils.load(store, {:Btree, size, root}) - %Btree{root: root, root_loc: root_loc, capacity: 3, store: store, size: size, dirt: 0} + + %Btree{ + root: root, + root_loc: root_loc, + capacity: 3, + store: store, + size: size, + dirt: 0, + metadata: [] + } end test "insert/3 called on non-full leaf inserts the key/value tuple" do @@ -174,6 +201,31 @@ defmodule CubDB.BtreeTest do btree |> Enum.into([]) end + test "insert_metadata/3 inserts and overwrites existing keys" do + btree = compose_btree() + btree = Btree.insert_metadata(btree, :foo, 123) + assert [foo: 123] == btree.metadata + btree = Btree.insert_metadata(btree, :foo, 456) + assert [foo: 456] == btree.metadata + end + + test "delete_metadata/3 deletes key if present, otherwise nothing" do + btree = compose_btree() + btree = Btree.insert_metadata(btree, :foo, 123) + btree = Btree.delete_metadata(btree, :foo) + assert [] = btree.metadata + btree = Btree.insert_metadata(btree, :foo, 123) + btree = Btree.delete_metadata(btree, :bar) + assert [foo: 123] = btree.metadata + end + + test "fetch_metadata/2 fetches value associated with key in metadata if present" do + btree = compose_btree() + btree = Btree.insert_metadata(btree, :foo, 123) + assert {:ok, 123} == Btree.fetch_metadata(btree, :foo) + assert :error == Btree.fetch_metadata(btree, :bar) + end + test "fetch/2 finds key and returns {:ok, value} or :error" do tiny_tree = compose_btree(leaf(children: [bar: 2, foo: 1])) assert {:ok, 1} = Btree.fetch(tiny_tree, :foo) diff --git a/test/cubdb/store/file_test.exs b/test/cubdb/store/file_test.exs index 3b17f78..3f04c20 100644 --- a/test/cubdb/store/file_test.exs +++ b/test/cubdb/store/file_test.exs @@ -29,10 +29,10 @@ defmodule CubDB.Store.FileTest do test "get_latest_header/1 skips corrupted header and locates latest good header", %{ store: store } do - good_header = header(size: 1, location: 2, dirt: 3) + good_header = header(size: 1, location: 2, dirt: 3, metadata: []) CubDB.Store.put_header(store, good_header) - CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0)) + CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0, metadata: [])) # corrupt the last header {:ok, file} = :file.open(store.file_path, [:read, :write, :raw, :binary]) @@ -45,10 +45,10 @@ defmodule CubDB.Store.FileTest do test "get_latest_header/1 skips truncated header and locates latest good header", %{ store: store } do - good_header = header(size: 1, location: 2, dirt: 3) + good_header = header(size: 1, location: 2, dirt: 3, metadata: []) CubDB.Store.put_header(store, good_header) - CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0)) + CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0, metadata: [])) # truncate the last header {:ok, file} = :file.open(store.file_path, [:read, :write, :raw, :binary]) @@ -60,7 +60,7 @@ defmodule CubDB.Store.FileTest do end test "get_latest_header/1 skips data and locates latest good header", %{store: store} do - header = header(size: 1, location: 2, dirt: 3) + header = header(size: 1, location: 2, dirt: 3, metadata: []) CubDB.Store.put_header(store, header) data_longer_than_one_block = String.duplicate("x", 1030) diff --git a/test/cubdb/transaction_test.exs b/test/cubdb/transaction_test.exs index 461450e..c2b8c37 100644 --- a/test/cubdb/transaction_test.exs +++ b/test/cubdb/transaction_test.exs @@ -44,6 +44,29 @@ defmodule CubDB.TransactionTest do assert [a: 1, c: 3] = CubDB.select(db) |> Enum.into([]) end + test "get_metadata, delete_metadata, put_metadata work as expected", %{tmp_dir: tmp_dir} do + {:ok, db} = CubDB.start_link(tmp_dir) + CubDB.put_multi(db, a: 1, c: 3) + CubDB.put_metadata(db, :foo, 123) + + CubDB.transaction(db, fn tx -> + tx = CubDB.Tx.put_metadata(tx, :foo, 456) + tx = CubDB.Tx.put_metadata(tx, :bar, 789) + + assert 456 == CubDB.Tx.get_metadata(tx, :foo) + assert 789 == CubDB.Tx.get_metadata(tx, :bar) + + tx = CubDB.Tx.delete_metadata(tx, :bar) + + assert nil == CubDB.Tx.get_metadata(tx, :bar) + + {:cancel, nil} + end) + + assert 123 == CubDB.get_metadata(db, :foo) + assert [a: 1, c: 3] = CubDB.select(db) |> Enum.into([]) + end + test "refetch/3 returns :unchanged if the entry was not written, otherwise fetches it", %{ tmp_dir: tmp_dir } do diff --git a/test/cubdb_test.exs b/test/cubdb_test.exs index dd7ba20..b8c400d 100644 --- a/test/cubdb_test.exs +++ b/test/cubdb_test.exs @@ -159,6 +159,22 @@ defmodule CubDBTest do assert [{^key, 123}] = CubDB.select(db) |> Enum.to_list() end + test "put_metadata/3, delete_metadata/2, get_metadata/2 work as expected", %{tmp_dir: tmp_dir} do + {:ok, db} = CubDB.start_link(tmp_dir) + + assert nil == CubDB.get_metadata(db, :foo) + assert :default == CubDB.get_metadata(db, :foo, :default) + assert {:error, :key_not_atom} == CubDB.get_metadata(db, "foo", :default) + + assert :ok == CubDB.put_metadata(db, :foo, 123) + assert {:error, :key_not_atom} == CubDB.put_metadata(db, "foo", 456) + assert 123 == CubDB.get_metadata(db, :foo, :default) + + assert :ok == CubDB.delete_metadata(db, :foo) + assert {:error, :key_not_atom} == CubDB.delete_metadata(db, "foo") + assert :default == CubDB.get_metadata(db, :foo, :default) + end + test "delete/2 does not error and does not write to disk when deleting an entry that was not present", %{tmp_dir: tmp_dir} do {:ok, db} = CubDB.start_link(tmp_dir) diff --git a/test/shared_examples/cubdb/store_examples.ex b/test/shared_examples/cubdb/store_examples.ex index 5b3531f..2fe6620 100644 --- a/test/shared_examples/cubdb/store_examples.ex +++ b/test/shared_examples/cubdb/store_examples.ex @@ -24,26 +24,30 @@ defmodule CubDB.StoreExamples do test "put_header/2 sets a header", %{store: store} do root_loc = CubDB.Store.put_node(store, value(val: 1)) - loc = CubDB.Store.put_header(store, header(size: 1, location: root_loc, dirt: 0)) - assert {^loc, header(size: 1, location: ^root_loc, dirt: 0)} = + loc = + CubDB.Store.put_header(store, header(size: 1, location: root_loc, dirt: 0, metadata: [])) + + assert {^loc, header(size: 1, location: ^root_loc, dirt: 0, metadata: [])} = CubDB.Store.get_latest_header(store) end test "get_latest_header/1 returns the most recently stored header", %{store: store} do CubDB.Store.put_node(store, value(val: 1)) CubDB.Store.put_node(store, value(val: 2)) - CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0)) + CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0, metadata: [])) CubDB.Store.put_node(store, value(val: 3)) - loc = CubDB.Store.put_header(store, header(size: 42, location: 0, dirt: 0)) + loc = CubDB.Store.put_header(store, header(size: 42, location: 0, dirt: 0, metadata: [])) CubDB.Store.put_node(store, value(val: 4)) - assert {^loc, header(size: 42, location: 0, dirt: 0)} = CubDB.Store.get_latest_header(store) + + assert {^loc, header(size: 42, location: 0, dirt: 0, metadata: [])} = + CubDB.Store.get_latest_header(store) end test "blank?/1 returns true if store is blank, and false otherwise", %{store: store} do assert CubDB.Store.blank?(store) == true CubDB.Store.put_node(store, value(val: 1)) - CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0)) + CubDB.Store.put_header(store, header(size: 0, location: 0, dirt: 0, metadata: [])) CubDB.Store.sync(store) assert CubDB.Store.blank?(store) == false end diff --git a/test/test_helper.exs b/test/test_helper.exs index f5ea5b3..9cdf13b 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -40,7 +40,7 @@ defmodule TestHelper do import Btree, only: [header: 1, value: 1, deleted: 0] def debug(store) do - {_, {size, root_loc, _}} = Store.get_latest_header(store) + {_, {size, root_loc, _, _}} = Store.get_latest_header(store) {:Btree, size, debug_node(store, root_loc)} end @@ -64,7 +64,7 @@ defmodule TestHelper do def load(store, {:Btree, size, root}) do {root_loc, root_node} = load_node(store, root) - Store.put_header(store, header(size: size, location: root_loc, dirt: 0)) + Store.put_header(store, header(size: size, location: root_loc, dirt: 0, metadata: [])) {root_loc, root_node} end