Add metadata support to CubDB

lib/cubdb.ex

@@ -168,6 +168,7 @@ defmodule CubDB do
   @auto_compact_defaults {100, 0.25}
 
   @type key :: any
+  @type metadata_key :: atom
   @type value :: any
   @type entry :: {key, value}
   @type auto_compact :: {pos_integer, number} | boolean
@@ -317,6 +318,24 @@ defmodule CubDB do
     end)
   end
 
+  @spec get_metadata(server, metadata_key, value) :: value | {:error, :key_not_atom}
+
+  @doc """
+  Gets the value associated to `key` from the database metadata table.
+
+  If no value is associated with `key`, `default` is returned (which is `nil`,
+  unless specified otherwise).
+  """
+  def get_metadata(db, key, default \\ nil)
+
+  def get_metadata(db, key, default) when is_atom(key) do
+    with_snapshot(db, fn %Snapshot{btree: btree} ->
+      Reader.get_metadata(btree, key, default)
+    end)
+  end
+
+  def get_metadata(_, _, _), do: {:error, :key_not_atom}
+
   @spec fetch(server, key) :: {:ok, value} | :error
 
   @doc """
@@ -784,6 +803,22 @@ defmodule CubDB do
     end)
   end
 
+  @spec put_metadata(server, metadata_key, value) :: :ok | {:error, :key_not_atom}
+
+  @doc """
+  Writes an entry in the database metadata table, associating `key` to `value`, 
+  overwriting any existing entries associated with the same key.
+
+  Returns `{:error, :key_not_atom}` if `key` is not an atom.
+  """
+  def put_metadata(db, key, value) when is_atom(key) do
+    transaction(db, fn tx ->
+      {:commit, Tx.put_metadata(tx, key, value), :ok}
+    end)
+  end
+
+  def put_metadata(_, _, _), do: {:error, :key_not_atom}
+
   @spec delete(server, key) :: :ok
 
   @doc """
@@ -797,6 +832,23 @@ defmodule CubDB do
     end)
   end
 
+  @spec delete_metadata(server, metadata_key) :: :ok | {:error, :key_not_atom}
+
+  @doc """
+  Deletes the entry associated to `key` from the database metadata table.
+
+  If `key` was not present in the database, nothing is done.
+
+  Returns `{:error, :key_not_atom}` if `key` is not an atom.
+  """
+  def delete_metadata(db, key) when is_atom(key) do
+    transaction(db, fn tx ->
+      {:commit, Tx.delete_metadata(tx, key), :ok}
+    end)
+  end
+
+  def delete_metadata(_, _), do: {:error, :key_not_atom}
+
   @spec update(server, key, value, (value -> value)) :: :ok
 
   @doc """

lib/cubdb/btree.ex

@@ -30,6 +30,7 @@ defmodule CubDB.Btree do
   @type btree_size :: non_neg_integer
   @type dirt :: non_neg_integer
   @type location :: non_neg_integer
+  @type metadata :: keyword
   @type capacity :: pos_integer
   @type child_pointer :: {key, location}
   @type leaf_node :: record(:leaf, children: [child_pointer])
@@ -51,34 +52,53 @@ defmodule CubDB.Btree do
           size: btree_size,
           dirt: dirt,
           store: Store.t(),
-          capacity: capacity
+          capacity: capacity,
+          metadata: metadata
         }
 
   @default_capacity 32
-  @enforce_keys [:root, :root_loc, :size, :dirt, :store, :capacity]
+  @enforce_keys [:root, :root_loc, :size, :dirt, :store, :capacity, :metadata]
   defstruct @enforce_keys
 
-  @spec header(size: btree_size, location: location, dirt: dirt) :: Macro.t()
+  @spec header(size: btree_size, location: location, dirt: dirt, metadata: metadata) :: Macro.t()
 
-  defmacro header(size: size, location: location, dirt: dirt) do
+  defmacro header(size: size, location: location, dirt: dirt, metadata: metadata) do
     quote do
-      {unquote(size), unquote(location), unquote(dirt)}
+      {unquote(size), unquote(location), unquote(dirt), unquote(metadata)}
     end
   end
 
   @spec new(Store.t(), capacity) :: Btree.t()
 
   def new(store, cap \\ @default_capacity) do
     case Store.get_latest_header(store) do
-      {_, header(size: s, location: loc, dirt: dirt)} ->
+      {_, header(size: s, location: loc, dirt: dirt, metadata: metadata)} ->
         root = Store.get_node(store, loc)
-        %Btree{root: root, root_loc: loc, dirt: dirt, size: s, capacity: cap, store: store}
+
+        %Btree{
+          root: root,
+          root_loc: loc,
+          dirt: dirt,
+          size: s,
+          capacity: cap,
+          store: store,
+          metadata: metadata
+        }
 
       nil ->
         root = leaf()
         loc = Store.put_node(store, root)
-        Store.put_header(store, header(size: 0, location: loc, dirt: 0))
-        %Btree{root: root, root_loc: loc, dirt: 0, size: 0, capacity: cap, store: store}
+        Store.put_header(store, header(size: 0, location: loc, dirt: 0, metadata: []))
+
+        %Btree{
+          root: root,
+          root_loc: loc,
+          dirt: 0,
+          size: 0,
+          capacity: cap,
+          store: store,
+          metadata: []
+        }
     end
   end
 
@@ -94,6 +114,8 @@ defmodule CubDB.Btree do
     unless Store.blank?(store),
       do: raise(ArgumentError, message: "cannot load into non-empty store")
 
+    metadata = if match?(%Btree{}, enum), do: enum.metadata, else: []
+
     {st, count} =
       Enum.reduce(enum, {[], 0}, fn {k, v}, {st, count} ->
         {load_node(store, k, value(val: v), st, 1, cap), count + 1}
@@ -103,8 +125,17 @@ defmodule CubDB.Btree do
       new(store, cap)
     else
       {root, root_loc} = finalize_load(store, st, 1, cap)
-      Store.put_header(store, header(size: count, location: root_loc, dirt: 0))
-      %Btree{root: root, root_loc: root_loc, capacity: cap, store: store, size: count, dirt: 0}
+      Store.put_header(store, header(size: count, location: root_loc, dirt: 0, metadata: []))
+
+      %Btree{
+        root: root,
+        root_loc: root_loc,
+        capacity: cap,
+        store: store,
+        size: count,
+        dirt: 0,
+        metadata: metadata
+      }
     end
   end
 
@@ -127,6 +158,17 @@ defmodule CubDB.Btree do
     end
   end
 
+  @spec fetch_metadata(Btree.t(), key) :: {:ok, val} | :error
+
+  # `fetch_metadata/2` returns `{:ok, value}` if an entry with key `key` is present
+  # in the Btree metadata table, otherwise `:error`.
+  def fetch_metadata(%Btree{metadata: metadata}, key) do
+    case Keyword.get(metadata, key) do
+      nil -> :error
+      value -> {:ok, value}
+    end
+  end
+
   @spec written_since?(Btree.t(), key, Btree.t()) ::
           boolean | {:maybe, :not_found} | {:maybe, :different_store}
 
@@ -174,13 +216,37 @@ defmodule CubDB.Btree do
     insert_terminal_node(btree, key, value(val: value), false)
   end
 
+  @spec insert_metadata(Btree.t(), key, val) :: Btree.t()
+
+  # `insert_metadata/3` inserts the key-value pair into the Btree metadata table,
+  # overwriting any existing values associated with the same key. It does not commit
+  # the operation, so `commit/1` must be explicitly called to commit the insertion.
+  def insert_metadata(btree, key, value) do
+    %Btree{metadata: metadata} = btree
+    metadata = Keyword.put(metadata, key, value)
+    %{btree | metadata: metadata}
+  end
+
+  @spec delete_metadata(Btree.t(), key) :: Btree.t()
+
+  # `delete_metadata/2` deletes the entry associated to `key` in the Btree metadata
+  # table, if it exists, otherwise it does nothing. It does not commit the operation,
+  # so `commit/1` must be explicitly called to commit the deletion.
+  def delete_metadata(btree, key) do
+    %Btree{metadata: metadata} = btree
+    metadata = Keyword.delete(metadata, key)
+    %{btree | metadata: metadata}
+  end
+
   @spec delete(Btree.t(), key) :: Btree.t()
 
   # `delete/2` deletes the entry associated to `key` in the Btree, if existing.
   # It does not commit the operation, so `commit/1` must be explicitly called to
   # commit the deletion.
   def delete(btree, key) do
-    %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt} = btree
+    %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt, metadata: metadata} =
+      btree
+
     {leaf = {@leaf, children}, path} = lookup_leaf(root, store, key, [])
 
     case List.keyfind(children, key, 0) do
@@ -199,7 +265,8 @@ defmodule CubDB.Btree do
           capacity: cap,
           store: store,
           size: size,
-          dirt: dirt + 1
+          dirt: dirt + 1,
+          metadata: metadata
         }
 
       nil ->
@@ -227,11 +294,20 @@ defmodule CubDB.Btree do
   @spec clear(Btree.t()) :: Btree.t()
 
   def clear(btree) do
-    %Btree{store: store, capacity: cap, dirt: dirt} = btree
+    %Btree{store: store, capacity: cap, dirt: dirt, metadata: metadata} = btree
 
     root = leaf()
     loc = Store.put_node(store, root)
-    %Btree{root: root, root_loc: loc, size: 0, dirt: dirt + 1, capacity: cap, store: store}
+
+    %Btree{
+      root: root,
+      root_loc: loc,
+      size: 0,
+      dirt: dirt + 1,
+      capacity: cap,
+      store: store,
+      metadata: metadata
+    }
   end
 
   @spec commit(Btree.t()) :: Btree.t()
@@ -243,8 +319,20 @@ defmodule CubDB.Btree do
   # If one or more updates are performed, but `commit/1` is not called, the
   # updates won't be committed to the database and will be lost in case of a
   # restart.
-  def commit(tree = %Btree{store: store, size: size, root_loc: root_loc, dirt: dirt}) do
-    Store.put_header(store, header(size: size, location: root_loc, dirt: dirt + 1))
+  def commit(
+        tree = %Btree{
+          store: store,
+          size: size,
+          root_loc: root_loc,
+          dirt: dirt,
+          metadata: metadata
+        }
+      ) do
+    Store.put_header(
+      store,
+      header(size: size, location: root_loc, dirt: dirt + 1, metadata: metadata)
+    )
+
     tree
   end
 
@@ -293,7 +381,8 @@ defmodule CubDB.Btree do
           Btree.t() | {:error, :exists}
 
   defp insert_terminal_node(btree, key, terminal_node, overwrite \\ true) do
-    %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt} = btree
+    %Btree{root: root, store: store, capacity: cap, size: s, dirt: dirt, metadata: metadata} =
+      btree
 
     {leaf = {@leaf, children}, path} = lookup_leaf(root, store, key, [])
     was_set = child_is_set?(store, children, key)
@@ -317,7 +406,8 @@ defmodule CubDB.Btree do
         capacity: cap,
         store: store,
         size: size,
-        dirt: dirt + 1
+        dirt: dirt + 1,
+        metadata: metadata
       }
     end
   end

lib/cubdb/compactor.ex

@@ -51,7 +51,7 @@ defmodule CubDB.Compactor do
     result =
       CubDB.transaction(db, fn
         %Tx{btree: ^original_btree} = tx ->
-          {:commit, %Tx{tx | btree: compacted_btree}, :done}
+          {:commit, %Tx{tx | btree: %{compacted_btree | metadata: tx.btree.metadata}}, :done}
 
         %Tx{btree: latest_btree} ->
           {:cancel, latest_btree}

lib/cubdb/reader.ex

@@ -16,6 +16,15 @@ defmodule CubDB.Reader do
     end
   end
 
+  @spec get_metadata(Btree.t(), CubDB.key(), any) :: any
+
+  def get_metadata(btree, key, default) do
+    case Btree.fetch_metadata(btree, key) do
+      {:ok, value} -> value
+      :error -> default
+    end
+  end
+
   @spec get_multi(Btree.t(), [CubDB.key()]) :: %{CubDB.key() => CubDB.value()}
 
   def get_multi(btree, keys) do

lib/cubdb/store/file.ex

@@ -222,6 +222,7 @@ defimpl CubDB.Store, for: CubDB.Store.File do
 
   defp read_header(file, location) do
     case read_term(file, location) do
+      {:ok, {size, loc, dirt}} -> {location, {size, loc, dirt, []}}
       {:ok, term} -> {location, term}
       {:error, _} -> get_latest_good_header(file, location - 1)
     end

lib/cubdb/transaction.ex

@@ -47,6 +47,11 @@ defmodule CubDB.Tx do
     Reader.get(btree, key, default)
   end
 
+  def get_metadata(tx = %Tx{btree: btree}, key, default \\ nil) do
+    validate_transaction!(tx)
+    Reader.get_metadata(btree, key, default)
+  end
+
   @spec fetch(Tx.t(), CubDB.key()) :: {:ok, CubDB.value()} | :error
 
   @doc """
@@ -252,6 +257,11 @@ defmodule CubDB.Tx do
     end
   end
 
+  def put_metadata(tx = %Tx{btree: btree}, key, value) do
+    validate_transaction!(tx)
+    %{tx | btree: Btree.insert_metadata(btree, key, value)}
+  end
+
   @spec delete(Tx.t(), CubDB.key()) :: Tx.t()
 
   @doc """
@@ -272,6 +282,11 @@ defmodule CubDB.Tx do
     end
   end
 
+  def delete_metadata(tx = %Tx{btree: btree}, key) do
+    validate_transaction!(tx)
+    %{tx | btree: Btree.delete_metadata(btree, key)}
+  end
+
   @spec clear(Tx.t()) :: Tx.t()
 
   @doc """