Implement paged serialization approach.

This allows to store all data in a single file instead of three separate ones.

Author: michaelwoerister

analyzeme/src/profiling_data.rs

@@ -6,7 +6,10 @@ use measureme::file_header::{
     read_file_header, write_file_header, CURRENT_FILE_FORMAT_VERSION, FILE_HEADER_SIZE,
     FILE_MAGIC_EVENT_STREAM,
 };
-use measureme::{EventId, ProfilerFiles, RawEvent, SerializationSink, StringTableBuilder};
+use measureme::{
+    EventId, PageTag, ProfilerFiles, RawEvent, SerializationSink, SerializationSinkBuilder,
+    StringTableBuilder,
+};
 use serde::{Deserialize, Deserializer};
 use std::error::Error;
 use std::fs;
@@ -46,12 +49,26 @@ impl ProfilingData {
     pub fn new(path_stem: &Path) -> Result<ProfilingData, Box<dyn Error>> {
         let paths = ProfilerFiles::new(path_stem);
 
-        let string_data = fs::read(paths.string_data_file).expect("couldn't read string_data file");
-        let index_data =
-            fs::read(paths.string_index_file).expect("couldn't read string_index file");
-        let event_data = fs::read(paths.events_file).expect("couldn't read events file");
+        let paged_path = path_stem.with_extension("mm_raw");
+
+        if paged_path.exists() {
+            let data = fs::read(paged_path).expect("couldn't read paged file");
+            let mut split_data = measureme::split_streams(&data[..]);
+
+            let string_data = split_data.remove(&PageTag::StringData).unwrap();
+            let index_data = split_data.remove(&PageTag::StringIndex).unwrap();
+            let event_data = split_data.remove(&PageTag::Events).unwrap();
 
-        ProfilingData::from_buffers(string_data, index_data, event_data)
+            ProfilingData::from_buffers(string_data, index_data, event_data)
+        } else {
+            let string_data =
+                fs::read(paths.string_data_file).expect("couldn't read string_data file");
+            let index_data =
+                fs::read(paths.string_index_file).expect("couldn't read string_index file");
+            let event_data = fs::read(paths.events_file).expect("couldn't read events file");
+
+            ProfilingData::from_buffers(string_data, index_data, event_data)
+        }
     }
 
     pub fn from_buffers(
@@ -207,9 +224,11 @@ pub struct ProfilingDataBuilder {
 
 impl ProfilingDataBuilder {
     pub fn new() -> ProfilingDataBuilder {
-        let event_sink = SerializationSink::new_in_memory();
-        let string_table_data_sink = Arc::new(SerializationSink::new_in_memory());
-        let string_table_index_sink = Arc::new(SerializationSink::new_in_memory());
+        let sink_builder = SerializationSinkBuilder::new_in_memory();
+
+        let event_sink = sink_builder.new_sink(PageTag::Events);
+        let string_table_data_sink = Arc::new(sink_builder.new_sink(PageTag::StringData));
+        let string_table_index_sink = Arc::new(sink_builder.new_sink(PageTag::StringIndex));
 
         // The first thing in every file we generate must be the file header.
         write_file_header(&event_sink, FILE_MAGIC_EVENT_STREAM);

analyzeme/src/stringtable.rs

@@ -243,13 +243,14 @@ impl StringTable {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use measureme::{SerializationSink, StringComponent, StringTableBuilder};
+    use measureme::{PageTag, SerializationSinkBuilder, StringComponent, StringTableBuilder};
     use std::sync::Arc;
 
     #[test]
     fn simple_strings() {
-        let data_sink = Arc::new(SerializationSink::new_in_memory());
-        let index_sink = Arc::new(SerializationSink::new_in_memory());
+        let sink_builder = SerializationSinkBuilder::new_in_memory();
+        let data_sink = Arc::new(sink_builder.new_sink(PageTag::StringData));
+        let index_sink = Arc::new(sink_builder.new_sink(PageTag::StringIndex));
 
         let expected_strings = &[
             "abc",
@@ -289,8 +290,9 @@ mod tests {
 
     #[test]
     fn composite_string() {
-        let data_sink = Arc::new(SerializationSink::new_in_memory());
-        let index_sink = Arc::new(SerializationSink::new_in_memory());
+        let sink_builder = SerializationSinkBuilder::new_in_memory();
+        let data_sink = Arc::new(sink_builder.new_sink(PageTag::StringData));
+        let index_sink = Arc::new(sink_builder.new_sink(PageTag::StringIndex));
 
         let expected_strings = &[
             "abc",                  // 0

analyzeme/src/testing_common.rs

@@ -96,18 +96,20 @@ fn generate_profiling_data(
         })
         .collect();
 
-    let expected_events: Vec<_> = threads
-        .into_iter()
-        .flat_map(|t| t.join().unwrap())
-        .collect();
-
     // An example of allocating the string contents of an event id that has
     // already been used
     profiler.map_virtual_to_concrete_string(
         event_id_virtual.to_string_id(),
         profiler.alloc_string("SomeQuery"),
     );
 
+    drop(profiler);
+
+    let expected_events: Vec<_> = threads
+        .into_iter()
+        .flat_map(|t| t.join().unwrap())
+        .collect();
+
     expected_events
 }
 

measureme/src/file_header.rs

@@ -5,7 +5,7 @@ use crate::SerializationSink;
 use std::convert::TryInto;
 use std::error::Error;
 
-pub const CURRENT_FILE_FORMAT_VERSION: u32 = 5;
+pub const CURRENT_FILE_FORMAT_VERSION: u32 = 6;
 pub const FILE_MAGIC_EVENT_STREAM: &[u8; 4] = b"MMES";
 pub const FILE_MAGIC_STRINGTABLE_DATA: &[u8; 4] = b"MMSD";
 pub const FILE_MAGIC_STRINGTABLE_INDEX: &[u8; 4] = b"MMSI";
@@ -53,11 +53,11 @@ pub fn strip_file_header(data: &[u8]) -> &[u8] {
 #[cfg(test)]
 mod tests {
     use super::*;
-    use crate::SerializationSink;
+    use crate::{PageTag, SerializationSinkBuilder};
 
     #[test]
     fn roundtrip() {
-        let data_sink = SerializationSink::new_in_memory();
+        let data_sink = SerializationSinkBuilder::new_in_memory().new_sink(PageTag::Events);
 
         write_file_header(&data_sink, FILE_MAGIC_EVENT_STREAM);
 
@@ -71,7 +71,7 @@ mod tests {
 
     #[test]
     fn invalid_magic() {
-        let data_sink = SerializationSink::new_in_memory();
+        let data_sink = SerializationSinkBuilder::new_in_memory().new_sink(PageTag::Events);
         write_file_header(&data_sink, FILE_MAGIC_STRINGTABLE_DATA);
         let mut data = data_sink.into_bytes();
 
@@ -82,7 +82,7 @@ mod tests {
 
     #[test]
     fn other_version() {
-        let data_sink = SerializationSink::new_in_memory();
+        let data_sink = SerializationSinkBuilder::new_in_memory().new_sink(PageTag::Events);
 
         write_file_header(&data_sink, FILE_MAGIC_STRINGTABLE_INDEX);
 

measureme/src/lib.rs

@@ -50,5 +50,7 @@ pub mod rustc;
 pub use crate::event_id::{EventId, EventIdBuilder};
 pub use crate::profiler::{Profiler, ProfilerFiles, TimingGuard};
 pub use crate::raw_event::{RawEvent, MAX_INSTANT_TIMESTAMP, MAX_INTERVAL_TIMESTAMP};
-pub use crate::serialization::{Addr, SerializationSink};
+pub use crate::serialization::{
+    split_streams, Addr, PageTag, SerializationSink, SerializationSinkBuilder,
+};
 pub use crate::stringtable::{SerializableString, StringComponent, StringId, StringTableBuilder};

measureme/src/profiler.rs

@@ -1,7 +1,7 @@
 use crate::event_id::EventId;
 use crate::file_header::{write_file_header, FILE_MAGIC_EVENT_STREAM};
 use crate::raw_event::RawEvent;
-use crate::serialization::SerializationSink;
+use crate::serialization::{PageTag, SerializationSink, SerializationSinkBuilder};
 use crate::stringtable::{SerializableString, StringId, StringTableBuilder};
 use std::error::Error;
 use std::path::{Path, PathBuf};
@@ -32,15 +32,17 @@ pub struct Profiler {
 
 impl Profiler {
     pub fn new<P: AsRef<Path>>(path_stem: P) -> Result<Profiler, Box<dyn Error + Send + Sync>> {
-        let paths = ProfilerFiles::new(path_stem.as_ref());
-        let event_sink = Arc::new(SerializationSink::from_path(&paths.events_file)?);
+        let path = path_stem.as_ref().with_extension("mm_raw");
+        let sink_builder = SerializationSinkBuilder::from_path(&path)?;
 
-        // The first thing in every file we generate must be the file header.
+        let event_sink = Arc::new(sink_builder.new_sink(PageTag::Events));
+
+        // The first thing in every stream we generate must be the stream header.
         write_file_header(&*event_sink, FILE_MAGIC_EVENT_STREAM);
 
         let string_table = StringTableBuilder::new(
-            Arc::new(SerializationSink::from_path(&paths.string_data_file)?),
-            Arc::new(SerializationSink::from_path(&paths.string_index_file)?),
+            Arc::new(sink_builder.new_sink(PageTag::StringData)),
+            Arc::new(sink_builder.new_sink(PageTag::StringIndex)),
         );
 
         let profiler = Profiler {