Merge pull request #506 from mstange/push-zmvswrxqnsry

mstange · web-flow · commit 3c8a92e0d5b3 · 2025-02-27T15:16:41.000-05:00
Reduce JSON size, emit version 53
diff --git a/fxprof-processed-profile/src/counters.rs b/fxprof-processed-profile/src/counters.rs
@@ -1,5 +1,9 @@
 use serde::ser::{Serialize, SerializeMap, Serializer};
 
+use crate::serialization_helpers::SliceWithPermutation;
+use crate::timestamp::{
+    SerializableTimestampSliceAsDeltas, SerializableTimestampSliceAsDeltasWithPermutation,
+};
 use crate::{GraphColor, ProcessHandle, Timestamp};
 
 /// A counter. Can be created with [`Profile::add_counter`](crate::Profile::add_counter).
@@ -89,6 +93,9 @@ struct CounterSamples {
     time: Vec<Timestamp>,
     number: Vec<u32>,
     count: Vec<f64>,
+
+    is_sorted_by_time: bool,
+    last_sample_timestamp: Timestamp,
 }
 
 impl CounterSamples {
@@ -97,6 +104,9 @@ impl CounterSamples {
             time: Vec::new(),
             number: Vec::new(),
             count: Vec::new(),
+
+            is_sorted_by_time: true,
+            last_sample_timestamp: Timestamp::from_nanos_since_reference(0),
         }
     }
 
@@ -109,6 +119,11 @@ impl CounterSamples {
         self.time.push(timestamp);
         self.count.push(value_delta);
         self.number.push(number_of_operations_delta);
+
+        if timestamp < self.last_sample_timestamp {
+            self.is_sorted_by_time = false;
+        }
+        self.last_sample_timestamp = timestamp;
     }
 }
 
@@ -117,9 +132,25 @@ impl Serialize for CounterSamples {
         let len = self.time.len();
         let mut map = serializer.serialize_map(None)?;
         map.serialize_entry("length", &len)?;
-        map.serialize_entry("count", &self.count)?;
-        map.serialize_entry("number", &self.number)?;
-        map.serialize_entry("time", &self.time)?;
+
+        if self.is_sorted_by_time {
+            map.serialize_entry("count", &self.count)?;
+            map.serialize_entry("number", &self.number)?;
+            map.serialize_entry(
+                "timeDeltas",
+                &SerializableTimestampSliceAsDeltas(&self.time),
+            )?;
+        } else {
+            let mut indexes: Vec<usize> = (0..self.time.len()).collect();
+            indexes.sort_unstable_by_key(|index| self.time[*index]);
+            map.serialize_entry("count", &SliceWithPermutation(&self.count, &indexes))?;
+            map.serialize_entry("number", &SliceWithPermutation(&self.number, &indexes))?;
+            map.serialize_entry(
+                "timeDeltas",
+                &SerializableTimestampSliceAsDeltasWithPermutation(&self.time, &indexes),
+            )?;
+        }
+
         map.end()
     }
 }
diff --git a/fxprof-processed-profile/src/frame_table.rs b/fxprof-processed-profile/src/frame_table.rs
@@ -100,15 +100,6 @@ impl FrameTable {
             })
     }
 
-    pub fn get_category(&self, frame_index: usize) -> CategoryPairHandle {
-        let category = self.categories[frame_index];
-        let subcategory = match self.subcategories[frame_index] {
-            Subcategory::Normal(subcategory) => Some(subcategory),
-            Subcategory::Other(_) => None,
-        };
-        CategoryPairHandle(category, subcategory)
-    }
-
     pub fn as_serializable<'a>(&'a self, categories: &'a [Category]) -> impl Serialize + 'a {
         SerializableFrameTable {
             table: self,
diff --git a/fxprof-processed-profile/src/lib.rs b/fxprof-processed-profile/src/lib.rs
@@ -78,4 +78,4 @@ pub use profile::{FrameHandle, Profile, SamplingInterval, StackHandle, StringHan
 pub use reference_timestamp::ReferenceTimestamp;
 pub use sample_table::WeightType;
 pub use thread::ProcessHandle;
-pub use timestamp::*;
+pub use timestamp::Timestamp;
diff --git a/fxprof-processed-profile/src/profile.rs b/fxprof-processed-profile/src/profile.rs
@@ -528,8 +528,7 @@ impl Profile {
             "FrameHandle from different thread passed to Profile::intern_stack"
         );
         let thread = &mut self.threads[thread.0];
-        let category_pair = thread.get_frame_category(frame_index);
-        let stack_index = thread.stack_index_for_stack(prefix, frame_index, category_pair);
+        let stack_index = thread.stack_index_for_stack(prefix, frame_index);
         StackHandle(thread_handle, stack_index)
     }
 
@@ -926,7 +925,6 @@ impl Profile {
         let process = &mut self.processes[thread.process().0];
         let mut prefix = None;
         for frame_info in frames {
-            let category_pair = frame_info.category_pair;
             let frame_index = Self::intern_frame_internal(
                 thread,
                 process,
@@ -935,7 +933,7 @@ impl Profile {
                 &mut self.kernel_libs,
                 &self.string_table,
             );
-            prefix = Some(thread.stack_index_for_stack(prefix, frame_index, category_pair));
+            prefix = Some(thread.stack_index_for_stack(prefix, frame_index));
         }
         prefix
     }
@@ -1049,7 +1047,7 @@ impl Serialize for SerializableProfileMeta<'_> {
             }),
         )?;
         map.serialize_entry("interval", &(self.0.interval.as_secs_f64() * 1000.0))?;
-        map.serialize_entry("preprocessedProfileVersion", &49)?;
+        map.serialize_entry("preprocessedProfileVersion", &53)?;
         map.serialize_entry("processType", &0)?;
         map.serialize_entry("product", &self.0.product)?;
         if let Some(os_name) = &self.0.os_name {
@@ -1066,7 +1064,7 @@ impl Serialize for SerializableProfileMeta<'_> {
         map.serialize_entry("startTime", &self.0.reference_timestamp)?;
         map.serialize_entry("symbolicated", &self.0.symbolicated)?;
         map.serialize_entry("pausedRanges", &[] as &[()])?;
-        map.serialize_entry("version", &24)?;
+        map.serialize_entry("version", &24)?; // this version is ignored, only "preprocessedProfileVersion" is used
         map.serialize_entry("usesOnlyOneStackType", &(!self.0.contains_js_function()))?;
         map.serialize_entry("doesNotUseFrameImplementation", &true)?;
         map.serialize_entry("sourceCodeIsNotOnSearchfox", &true)?;
diff --git a/fxprof-processed-profile/src/sample_table.rs b/fxprof-processed-profile/src/sample_table.rs
@@ -3,8 +3,11 @@ use std::fmt::{Display, Formatter};
 use serde::ser::{Serialize, SerializeMap, Serializer};
 
 use crate::cpu_delta::CpuDelta;
-use crate::serialization_helpers::SerializableSingleValueColumn;
-use crate::Timestamp;
+use crate::serialization_helpers::{SerializableSingleValueColumn, SliceWithPermutation};
+use crate::timestamp::{
+    SerializableTimestampSliceAsDeltas, SerializableTimestampSliceAsDeltasWithPermutation,
+    Timestamp,
+};
 
 /// The sample table contains stacks with timestamps and some extra information.
 ///
@@ -20,7 +23,7 @@ pub struct SampleTable {
     sample_stack_indexes: Vec<Option<usize>>,
     /// CPU usage delta since the previous sample for this thread, for each sample.
     sample_cpu_deltas: Vec<CpuDelta>,
-    sorted_by_time: bool,
+    is_sorted_by_time: bool,
     last_sample_timestamp: Timestamp,
 }
 
@@ -76,7 +79,7 @@ impl SampleTable {
             sample_timestamps: Vec::new(),
             sample_stack_indexes: Vec::new(),
             sample_cpu_deltas: Vec::new(),
-            sorted_by_time: true,
+            is_sorted_by_time: true,
             last_sample_timestamp: Timestamp::from_nanos_since_reference(0),
         }
     }
@@ -93,7 +96,7 @@ impl SampleTable {
         self.sample_stack_indexes.push(stack_index);
         self.sample_cpu_deltas.push(cpu_delta);
         if timestamp < self.last_sample_timestamp {
-            self.sorted_by_time = false;
+            self.is_sorted_by_time = false;
         }
         self.last_sample_timestamp = timestamp;
     }
@@ -115,9 +118,12 @@ impl Serialize for SampleTable {
         map.serialize_entry("length", &len)?;
         map.serialize_entry("weightType", &self.sample_weight_type.to_string())?;
 
-        if self.sorted_by_time {
+        if self.is_sorted_by_time {
             map.serialize_entry("stack", &self.sample_stack_indexes)?;
-            map.serialize_entry("time", &self.sample_timestamps)?;
+            map.serialize_entry(
+                "timeDeltas",
+                &SerializableTimestampSliceAsDeltas(&self.sample_timestamps),
+            )?;
             map.serialize_entry("weight", &self.sample_weights)?;
             map.serialize_entry("threadCPUDelta", &self.sample_cpu_deltas)?;
         } else {
@@ -128,8 +134,11 @@ impl Serialize for SampleTable {
                 &SliceWithPermutation(&self.sample_stack_indexes, &indexes),
             )?;
             map.serialize_entry(
-                "time",
-                &SliceWithPermutation(&self.sample_timestamps, &indexes),
+                "timeDeltas",
+                &SerializableTimestampSliceAsDeltasWithPermutation(
+                    &self.sample_timestamps,
+                    &indexes,
+                ),
             )?;
             map.serialize_entry(
                 "weight",
@@ -144,17 +153,6 @@ impl Serialize for SampleTable {
     }
 }
 
-struct SliceWithPermutation<'a, T: Serialize>(&'a [T], &'a [usize]);
-
-impl<T: Serialize> Serialize for SliceWithPermutation<'_, T> {
-    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
-    where
-        S: Serializer,
-    {
-        serializer.collect_seq(self.1.iter().map(|i| &self.0[*i]))
-    }
-}
-
 /// JS documentation of the native allocations table:
 ///
 /// ```ignore
diff --git a/fxprof-processed-profile/src/serialization_helpers.rs b/fxprof-processed-profile/src/serialization_helpers.rs
@@ -28,3 +28,14 @@ impl Serialize for SerializableOptionalTimestampColumn<'_> {
         seq.end()
     }
 }
+
+pub struct SliceWithPermutation<'a, T: Serialize>(pub &'a [T], pub &'a [usize]);
+
+impl<T: Serialize> Serialize for SliceWithPermutation<'_, T> {
+    fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
+    where
+        S: Serializer,
+    {
+        serializer.collect_seq(self.1.iter().map(|i| &self.0[*i]))
+    }
+}
diff --git a/fxprof-processed-profile/src/stack_table.rs b/fxprof-processed-profile/src/stack_table.rs
@@ -1,8 +1,5 @@
 use serde::ser::{Serialize, SerializeMap, Serializer};
 
-use crate::category::{
-    Category, CategoryHandle, CategoryPairHandle, SerializableSubcategoryColumn, Subcategory,
-};
 use crate::fast_hash_map::FastHashMap;
 
 /// The stack table stores the tree of stack nodes of a thread. The shape of the tree is encoded in
@@ -47,10 +44,6 @@ pub struct StackTable {
     stack_prefixes: Vec<Option<usize>>,
     stack_frames: Vec<usize>,
 
-    /// Imported profiles may not have categories. In this case fill the array with 0s.
-    stack_categories: Vec<CategoryHandle>,
-    stack_subcategories: Vec<Subcategory>,
-
     // (parent stack, frame_index) -> stack index
     index: FastHashMap<(Option<usize>, usize), usize>,
 }
@@ -60,60 +53,27 @@ impl StackTable {
         Default::default()
     }
 
-    pub fn index_for_stack(
-        &mut self,
-        prefix: Option<usize>,
-        frame: usize,
-        category_pair: CategoryPairHandle,
-    ) -> usize {
+    pub fn index_for_stack(&mut self, prefix: Option<usize>, frame: usize) -> usize {
         match self.index.get(&(prefix, frame)) {
             Some(stack) => *stack,
             None => {
-                let CategoryPairHandle(category, subcategory_index) = category_pair;
-                let subcategory = match subcategory_index {
-                    Some(index) => Subcategory::Normal(index),
-                    None => Subcategory::Other(category),
-                };
-
                 let stack = self.stack_prefixes.len();
                 self.stack_prefixes.push(prefix);
                 self.stack_frames.push(frame);
-                self.stack_categories.push(category);
-                self.stack_subcategories.push(subcategory);
                 self.index.insert((prefix, frame), stack);
                 stack
             }
         }
     }
-
-    pub fn serialize_with_categories<'a>(
-        &'a self,
-        categories: &'a [Category],
-    ) -> impl Serialize + 'a {
-        SerializableStackTable {
-            table: self,
-            categories,
-        }
-    }
-}
-
-struct SerializableStackTable<'a> {
-    table: &'a StackTable,
-    categories: &'a [Category],
 }
 
-impl Serialize for SerializableStackTable<'_> {
+impl Serialize for StackTable {
     fn serialize<S: Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
-        let len = self.table.stack_prefixes.len();
+        let len = self.stack_prefixes.len();
         let mut map = serializer.serialize_map(Some(3))?;
         map.serialize_entry("length", &len)?;
-        map.serialize_entry("prefix", &self.table.stack_prefixes)?;
-        map.serialize_entry("frame", &self.table.stack_frames)?;
-        map.serialize_entry("category", &self.table.stack_categories)?;
-        map.serialize_entry(
-            "subcategory",
-            &SerializableSubcategoryColumn(&self.table.stack_subcategories, self.categories),
-        )?;
+        map.serialize_entry("prefix", &self.stack_prefixes)?;
+        map.serialize_entry("frame", &self.stack_frames)?;
         map.end()
     }
 }
diff --git a/fxprof-processed-profile/src/thread.rs b/fxprof-processed-profile/src/thread.rs
@@ -3,7 +3,7 @@ use std::cmp::Ordering;
 
 use serde::ser::{SerializeMap, Serializer};
 
-use crate::category::{Category, CategoryPairHandle};
+use crate::category::Category;
 use crate::cpu_delta::CpuDelta;
 use crate::frame_table::{FrameTable, InternalFrame};
 use crate::func_table::FuncTable;
@@ -116,18 +116,8 @@ impl Thread {
         )
     }
 
-    pub fn get_frame_category(&self, frame: usize) -> CategoryPairHandle {
-        self.frame_table.get_category(frame)
-    }
-
-    pub fn stack_index_for_stack(
-        &mut self,
-        prefix: Option<usize>,
-        frame: usize,
-        category_pair: CategoryPairHandle,
-    ) -> usize {
-        self.stack_table
-            .index_for_stack(prefix, frame, category_pair)
+    pub fn stack_index_for_stack(&mut self, prefix: Option<usize>, frame: usize) -> usize {
+        self.stack_table.index_for_stack(prefix, frame)
     }
 
     pub fn add_sample(
@@ -265,10 +255,7 @@ impl Thread {
         if let Some(allocations) = &self.native_allocations {
             map.serialize_entry("nativeAllocations", &allocations)?;
         }
-        map.serialize_entry(
-            "stackTable",
-            &self.stack_table.serialize_with_categories(categories),
-        )?;
+        map.serialize_entry("stackTable", &self.stack_table)?;
         map.serialize_entry("stringArray", &self.string_table)?;
         map.serialize_entry("tid", &self.tid)?;
         map.serialize_entry("unregisterTime", &thread_unregister_time)?;
diff --git a/fxprof-processed-profile/src/timestamp.rs b/fxprof-processed-profile/src/timestamp.rs
diff --git a/fxprof-processed-profile/tests/integration_tests/main.rs b/fxprof-processed-profile/tests/integration_tests/main.rs
