Add tests for std::[unordered_][multi]set

.gitignore

@@ -1,2 +1,6 @@
 *.json
 *.root
+
+# Ignore files related to dictionary generation
+*.pcm
+Nested*.cxx

Makefile

@@ -5,17 +5,23 @@ endif
 
 .PHONY: all
 all:
+	$(MAKE) dict
 	$(MAKE) write
 	$(MAKE) read
 
 # This assumes there is no whitespace in any of the paths...
+DICT_MAKEFILE_DIR := $(sort $(shell find */ -name Makefile -printf "%h\n"))
 WRITE_C := $(sort $(shell find . -name write.C))
 READ_C := $(sort $(shell find . -name read.C))
 
+.PHONY: dict
+dict:
+	@$(foreach d,$(DICT_MAKEFILE_DIR),make -C $(d) &&) true
+
 .PHONY: write
 write:
-	@$(foreach c,$(WRITE_C),$(ROOT_EXE) -q -l $(c) &&) true
+	@$(foreach c,$(WRITE_C),LD_LIBRARY_PATH=$(shell dirname $(c)) $(ROOT_EXE) -q -l $(c) &&) true
 
 .PHONY: read
 read:
-	@$(foreach c,$(READ_C),$(ROOT_EXE) -q -l $(c) &&) true
+	@$(foreach c,$(READ_C),LD_LIBRARY_PATH=$(shell dirname $(c)) $(ROOT_EXE) -q -l $(c) &&) true

README.md

@@ -7,7 +7,7 @@ To that end, each test comes with a written description of the schema and the ex
 Eventually, we will also provide a set of reference `.root` and `.json` files (see below).
 
 > [!WARNING]
-> The RNTuple Validation Suite is currently work-in-progress.
+> The RNTuple Validation Suite is currently work in progress.
 > Its structure and implementation may change during development.
 > We welcome early feedback on the current state from any interested party.
 
@@ -22,7 +22,7 @@ More tests are planned in the future, please [consult the list of issues](https:
 ## Reference Implementation
 
 This repository also contains a reference implementation with ROOT macros.
-They currently target ROOT v6.34 with the first official version of the on-disk binary format.
+They currently target ROOT v6.34 with the first official version of the RNTuple on-disk binary format.
 In this release, the API is not yet finalized and all classes are in the `ROOT::Experimental` namespace.
 We plan to rely on this version until a first version of the RNTuple Validation Suite is done.
 Then we will produce a first set of reference files that can be used to test backwards compatibility.

types/README.md

@@ -2,7 +2,11 @@
 
  * [`fundamental`](fundamental): fundamental column types
  * [`optional`](optional): `std::optional` with different element types
+ * [`multiset`](multiset): `std::multiset` with all `[Split]Index{32,64}` column types
+ * [`set`](set): `std::set` with all `[Split]Index{32,64}` column types
  * [`string`](string): `std::string` with all `[Split]Index{32,64}` column types
  * [`unique_ptr`](unique_ptr): `std::unique_ptr` with different element types
+ * [`unordered_multiset`](unordered_multiset): `std::unordered_multiset` with all `[Split]Index{32,64}` column types
+ * [`unordered_set`](unordered_set): `std::unordered_set` with all `[Split]Index{32,64}` column types
  * [`variant`](variant): `std::variant` with `Switch` column type
  * [`vector`](vector): `std::vector` with all `[Split]Index{32,64}` column types

types/multiset/README.md

@@ -0,0 +1,4 @@
+# `std::multiset`
+
+ * [`fundamental`](fundamental): `std::multiset<std::int32_t>`
+ * [`nested`](nested): `std::multiset<std::multiset<std::int32_t>>`

types/multiset/fundamental/README.md

@@ -0,0 +1,15 @@
+# `std::multiset<std::int32_t>`
+
+## Fields
+
+ * `[Split]Index{32,64}`
+
+with the corresponding column type for the offset column of the collection parent field.
+All child fields use the default column encoding `Int32`.
+
+## Entries
+
+1. Single-element sets, with ascending values
+2. Empty sets
+3. Increasing number of elements in the set:
+   one element in the first field, two elements in the second field, etc.

types/multiset/fundamental/read.C

@@ -0,0 +1,68 @@
+#include <ROOT/REntry.hxx>
+#include <ROOT/RNTupleReader.hxx>
+
+using ROOT::Experimental::REntry;
+using ROOT::Experimental::RNTupleReader;
+
+#include <cstdint>
+#include <fstream>
+#include <ostream>
+#include <set>
+#include <string>
+#include <string_view>
+
+using Multiset = std::multiset<std::int32_t>;
+
+static void PrintMultisetValue(const REntry &entry, std::string_view name,
+                               std::ostream &os, bool last = false) {
+  Multiset &value = *entry.GetPtr<Multiset>(name);
+  os << "    \"" << name << "\": [";
+  bool first = true;
+  for (auto element : value) {
+    if (first) {
+      first = false;
+    } else {
+      os << ",";
+    }
+    os << "\n      " << element;
+  }
+  if (!value.empty()) {
+    os << "\n    ";
+  }
+  os << "]";
+  if (!last) {
+    os << ",";
+  }
+  os << "\n";
+}
+
+void read(std::string_view input = "types.multiset.fundamental.root",
+          std::string_view output = "types.multiset.fundamental.json") {
+  std::ofstream os(std::string{output});
+  os << "[\n";
+
+  auto reader = RNTupleReader::Open("ntpl", input);
+  auto &entry = reader->GetModel().GetDefaultEntry();
+  bool first = true;
+  for (auto index : *reader) {
+    reader->LoadEntry(index);
+
+    if (first) {
+      first = false;
+    } else {
+      os << ",\n";
+    }
+    os << "  {\n";
+
+    PrintMultisetValue(entry, "Index32", os);
+    PrintMultisetValue(entry, "Index64", os);
+    PrintMultisetValue(entry, "SplitIndex32", os);
+    PrintMultisetValue(entry, "SplitIndex64", os, /*last=*/true);
+
+    os << "  }";
+    // Newline is intentionally missing, may need to print a comma before the
+    // next entry.
+  }
+  os << "\n";
+  os << "]\n";
+}

types/multiset/fundamental/write.C

@@ -0,0 +1,67 @@
+#include <ROOT/RField.hxx>
+#include <ROOT/RNTupleModel.hxx>
+#include <ROOT/RNTupleUtil.hxx>
+#include <ROOT/RNTupleWriteOptions.hxx>
+#include <ROOT/RNTupleWriter.hxx>
+
+using ROOT::Experimental::EColumnType;
+using ROOT::Experimental::RField;
+using ROOT::Experimental::RNTupleModel;
+using ROOT::Experimental::RNTupleWriteOptions;
+using ROOT::Experimental::RNTupleWriter;
+
+#include <cstdint>
+#include <memory>
+#include <set>
+#include <string_view>
+
+using Multiset = std::multiset<std::int32_t>;
+
+static std::shared_ptr<Multiset> MakeMultisetField(RNTupleModel &model,
+                                                   std::string_view name,
+                                                   EColumnType indexType) {
+  auto field = std::make_unique<RField<Multiset>>(name);
+  field->SetColumnRepresentatives({{indexType}});
+  model.AddField(std::move(field));
+  return model.GetDefaultEntry().GetPtr<Multiset>(name);
+}
+
+void write(std::string_view filename = "types.multiset.fundamental.root") {
+  auto model = RNTupleModel::Create();
+
+  // Non-split index encoding
+  auto Index32 = MakeMultisetField(*model, "Index32", EColumnType::kIndex32);
+  auto Index64 = MakeMultisetField(*model, "Index64", EColumnType::kIndex64);
+
+  // Split index encoding
+  auto SplitIndex32 =
+      MakeMultisetField(*model, "SplitIndex32", EColumnType::kSplitIndex32);
+  auto SplitIndex64 =
+      MakeMultisetField(*model, "SplitIndex64", EColumnType::kSplitIndex64);
+
+  RNTupleWriteOptions options;
+  options.SetCompression(0);
+  auto writer =
+      RNTupleWriter::Recreate(std::move(model), "ntpl", filename, options);
+
+  // First entry: single-element sets, with ascending values
+  *Index32 = {1};
+  *Index64 = {2};
+  *SplitIndex32 = {3};
+  *SplitIndex64 = {4};
+  writer->Fill();
+
+  // Second entry: empty sets
+  *Index32 = {};
+  *Index64 = {};
+  *SplitIndex32 = {};
+  *SplitIndex64 = {};
+  writer->Fill();
+
+  // Third entry: increasing number of elements in the set
+  *Index32 = {1};
+  *Index64 = {2, 3};
+  *SplitIndex32 = {4, 5, 6};
+  *SplitIndex64 = {7, 8, 9, 10};
+  writer->Fill();
+}

types/multiset/nested/LinkDef.h

@@ -0,0 +1,6 @@
+#include <cstdint>
+#include <set>
+
+#ifdef __CLING__
+#pragma link C++ class std::multiset<std::multiset<std::int32_t>>+;
+#endif

types/multiset/nested/Makefile

@@ -0,0 +1,20 @@
+CXX=g++
+CXXFLAGS_ROOT=$(shell root-config --cflags)
+ifeq ($(CXXFLAGS_ROOT),)
+  $(error cannot find root-config: make sure to source thisroot.sh)
+endif
+CXXFLAGS=-Wall $(CXXFLAGS_ROOT)
+LDFLAGS=$(shell root-config --libs)
+
+.PHONY: all clean
+
+all: NestedMultiset.cxx libNestedMultiset.so
+
+NestedMultiset.cxx: NestedMultiset.hxx LinkDef.h
+	rootcling -f $@ $^
+
+libNestedMultiset.so: NestedMultiset.cxx
+	$(CXX) -shared -fPIC -o $@ $^ $(CXXFLAGS) $(LDFLAGS)
+
+clean:
+	rm -f NestedMultiset.cxx NestedMultiset_rdict.pcm libNestedMultiset.so

types/multiset/nested/NestedMultiset.hxx

@@ -0,0 +1,4 @@
+#pragma once
+
+#include <cstdint>
+#include <set>

types/multiset/nested/README.md

@@ -0,0 +1,18 @@
+# `std::multiset<std::multiset<std::int32_t>>`
+
+## Fields
+
+ * `[Split]Index{32,64}`
+
+with the corresponding column type for the offset column of the two collection parent fields.
+All child fields use the default column encoding `Int32`.
+
+## Entries
+
+1. Single-element sets, with ascending values
+2. Empty sets
+3. Increasing number of elements in the outer set, with arbitrary lengths of the inner sets
+
+## Dictionaries
+
+These tests require ROOT dictionaries, which can be generated with the provided `Makefile` in this directory. This will create a `libNestedMultiset` shared object.

types/multiset/nested/read.C

@@ -0,0 +1,88 @@
+#include <ROOT/REntry.hxx>
+#include <ROOT/RNTupleReader.hxx>
+
+using ROOT::Experimental::REntry;
+using ROOT::Experimental::RNTupleReader;
+
+#include <TSystem.h>
+
+#include <cstdint>
+#include <filesystem>
+#include <fstream>
+#include <ostream>
+#include <set>
+#include <string>
+#include <string_view>
+
+using Multiset = std::multiset<std::multiset<std::int32_t>>;
+
+static void PrintNestedMultisetValue(const REntry &entry, std::string_view name,
+                                     std::ostream &os, bool last = false) {
+  Multiset &value = *entry.GetPtr<Multiset>(name);
+  os << "    \"" << name << "\": [";
+  bool outerFirst = true;
+  for (auto inner : value) {
+    if (outerFirst) {
+      outerFirst = false;
+    } else {
+      os << ",";
+    }
+    os << "\n      [";
+    bool innerFirst = true;
+    for (auto element : inner) {
+      if (innerFirst) {
+        innerFirst = false;
+      } else {
+        os << ",";
+      }
+      os << "\n        " << element;
+    }
+    if (!inner.empty()) {
+      os << "\n      ";
+    }
+    os << "]";
+  }
+  if (!value.empty()) {
+    os << "\n    ";
+  }
+  os << "]";
+  if (!last) {
+    os << ",";
+  }
+  os << "\n";
+}
+
+void read(std::string_view input = "types.multiset.nested.root",
+          std::string_view output = "types.multiset.nested.json") {
+  if (gSystem->Load("libNestedMultiset") == -1)
+    throw std::runtime_error("could not find the required ROOT dictionaries, "
+                             "please make sure to run `make` first");
+
+  std::ofstream os(std::string{output});
+  os << "[\n";
+
+  auto reader = RNTupleReader::Open("ntpl", input);
+  auto &entry = reader->GetModel().GetDefaultEntry();
+  bool first = true;
+  for (auto index : *reader) {
+    reader->LoadEntry(index);
+
+    if (first) {
+      first = false;
+    } else {
+      os << ",\n";
+    }
+    os << "  {\n";
+
+    PrintNestedMultisetValue(entry, "Index32", os);
+    PrintNestedMultisetValue(entry, "Index64", os);
+    PrintNestedMultisetValue(entry, "SplitIndex32", os);
+    PrintNestedMultisetValue(entry, "SplitIndex64", os, /*last=*/true);
+
+    os << "  }";
+    // Newline is intentionally missing, may need to print a comma before the
+    // next entry.
+  }
+  os << "\n";
+  os << "]\n";
+}