Improve presymbolication (#549)

Make it respect the various `--symbol-*` command line arguments, and fix
a bug caused by incorrect caching.

Author: mstange

Cargo.lock

@@ -12,7 +12,7 @@ use nom::{Err, IResult};
 /// in [Firefox Breakpad .sym files](https://searchfox.org/mozilla-central/rev/4ebfb48f7e82251145afa4a822f970931dd06c68/toolkit/crashreporter/tools/symbolstore.py#199).
 /// This format is also used in the [Tecken symbolication API](https://tecken.readthedocs.io/en/latest/symbolication.html#id9),
 /// and [internally in the Firefox Profiler](https://github.com/firefox-devtools/profiler/blob/fb9ff03ab8b98d9e7c29e36314080fae555bbe78/src/utils/special-paths.js).
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub enum MappedPath {
     /// A path to a file in a git repository.
     Git {

samply-symbols/src/mapped_path.rs

@@ -455,7 +455,7 @@ pub trait FileContents: Send + Sync {
 
 /// The debug information (function name, file path, line number) for a single frame
 /// at the looked-up address.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct FrameDebugInfo {
     /// The function name for this frame, if known.
     pub function: Option<String>,
@@ -516,7 +516,7 @@ pub trait FileLocation: Clone + Display {
 /// refer to a file on this machine or on a different machine (i.e. the original
 /// build machine). The mapped path is something like a permalink which potentially
 /// allows obtaining the source file from a source server or a public hosted repository.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct SourceFilePath {
     /// The raw path to the source file, as written down in the debug file. This is
     /// usually an absolute path.
@@ -667,7 +667,7 @@ pub fn relative_address_base<'data>(object_file: &impl object::Object<'data>) ->
 }
 
 /// The symbol for a function.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct SymbolInfo {
     /// The function's address. This is a relative address.
     pub address: u32,
@@ -678,7 +678,7 @@ pub struct SymbolInfo {
 }
 
 /// The lookup result for an address.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct AddressInfo {
     /// Information about the symbol which contains the looked up address.
     pub symbol: SymbolInfo,
@@ -695,7 +695,7 @@ pub struct AddressInfo {
 }
 
 /// The lookup result from `lookup_sync`.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub struct SyncAddressInfo {
     /// Information about the symbol which contains the looked up address.
     pub symbol: SymbolInfo,
@@ -705,7 +705,7 @@ pub struct SyncAddressInfo {
 
 /// Contains address debug info (inlined functions, file names, line numbers) if
 /// available.
-#[derive(Debug, Clone, PartialEq, Eq)]
+#[derive(Debug, Clone, Hash, PartialEq, Eq)]
 pub enum FramesLookupResult {
     /// Debug info for this address was found in the symbol map.
     ///

samply-symbols/src/shared.rs

@@ -52,6 +52,7 @@ fs4 = "0.13"
 humantime = "2.1.0"
 shlex = "1.3.0"
 samply-quota-manager = { version = "0.1.0", path = "../samply-quota-manager" }
+indexmap = "2.9.0"
 
 [target.'cfg(any(target_os = "android", target_os = "macos", target_os = "linux"))'.dependencies]
 

samply/Cargo.toml

@@ -23,6 +23,8 @@ use std::io::BufReader;
 use std::path::Path;
 
 use fxprof_processed_profile::Profile;
+use shared::ctrl_c::CtrlC;
+use wholesym::LibraryInfo;
 
 #[cfg(any(target_os = "android", target_os = "linux"))]
 use linux::profiler;
@@ -32,9 +34,10 @@ use mac::profiler;
 use windows::profiler;
 
 use profile_json_preparse::parse_libinfo_map_from_profile_file;
-use server::start_server_main;
-use shared::prop_types::ImportProps;
+use server::{start_server, RunningServerInfo, ServerProps};
+use shared::prop_types::{ImportProps, SymbolProps};
 use shared::save_profile::save_profile_to_file;
+use symbols::create_symbol_manager_and_quota_manager;
 
 fn main() {
     env_logger::init();
@@ -64,28 +67,10 @@ fn main() {
 }
 
 fn do_load_action(load_args: cli::LoadArgs) {
-    let profile_filename = &load_args.file;
-    let input_file = match File::open(profile_filename) {
-        Ok(file) => file,
-        Err(err) => {
-            eprintln!("Could not open file {:?}: {}", profile_filename, err);
-            std::process::exit(1)
-        }
-    };
-
-    let libinfo_map = match parse_libinfo_map_from_profile_file(input_file, profile_filename) {
-        Ok(libinfo_map) => libinfo_map,
-        Err(err) => {
-            eprintln!("Could not parse the input file as JSON: {}", err);
-            eprintln!("If this is a perf.data file, please use `samply import` instead.");
-            std::process::exit(1)
-        }
-    };
-    start_server_main(
-        profile_filename,
+    run_server_serving_profile(
+        &load_args.file,
         load_args.server_props(),
         load_args.symbol_props(),
-        libinfo_map,
     );
 }
 
@@ -109,21 +94,18 @@ fn do_import_action(import_args: cli::ImportArgs) {
         crate::shared::symbol_precog::presymbolicate(
             &profile,
             &import_args.output.with_extension("syms.json"),
+            import_args.symbol_props(),
         );
     }
 
+    // Drop the profile so that it doesn't take up memory while the server is running.
+    drop(profile);
+
     if let Some(server_props) = import_args.server_props() {
-        let profile_filename = &import_args.output;
-        let libinfo_map = profile_json_preparse::parse_libinfo_map_from_profile_file(
-            File::open(profile_filename).expect("Couldn't open file we just wrote"),
-            profile_filename,
-        )
-        .expect("Couldn't parse libinfo map from profile file");
-        start_server_main(
-            profile_filename,
+        run_server_serving_profile(
+            &import_args.output,
             server_props,
             import_args.symbol_props(),
-            libinfo_map,
         );
     }
 }
@@ -155,22 +137,19 @@ fn do_record_action(record_args: cli::RecordArgs) {
         crate::shared::symbol_precog::presymbolicate(
             &profile,
             &record_args.output.with_extension("syms.json"),
+            record_args.symbol_props(),
         );
     }
 
+    // Drop the profile so that it doesn't take up memory while the server is running.
+    drop(profile);
+
     // then fire up the server for the profiler front end, if not save-only
     if let Some(server_props) = record_args.server_props() {
-        let profile_filename = &record_args.output;
-        let libinfo_map = crate::profile_json_preparse::parse_libinfo_map_from_profile_file(
-            File::open(profile_filename).expect("Couldn't open file we just wrote"),
-            profile_filename,
-        )
-        .expect("Couldn't parse libinfo map from profile file");
-        start_server_main(
-            profile_filename,
+        run_server_serving_profile(
+            &record_args.output,
             server_props,
             record_args.symbol_props(),
-            libinfo_map,
         );
     }
 
@@ -227,3 +206,85 @@ fn convert_file_to_profile(
         }
     }
 }
+
+fn run_server_serving_profile(
+    profile_path: &Path,
+    server_props: ServerProps,
+    symbol_props: SymbolProps,
+) {
+    let libinfo_map = {
+        let profile_file = match File::open(profile_path) {
+            Ok(file) => file,
+            Err(err) => {
+                eprintln!("Could not open file {:?}: {}", profile_path, err);
+                std::process::exit(1)
+            }
+        };
+
+        parse_libinfo_map_from_profile_file(profile_file, profile_path)
+            .expect("Couldn't parse libinfo map from profile file")
+    };
+
+    let runtime = tokio::runtime::Builder::new_multi_thread()
+        .enable_all()
+        .build()
+        .unwrap();
+
+    runtime.block_on(async {
+        let (mut symbol_manager, quota_manager) =
+            create_symbol_manager_and_quota_manager(symbol_props, server_props.verbose);
+        for lib_info in libinfo_map.into_values() {
+            symbol_manager.add_known_library(lib_info);
+        }
+
+        let precog_path = profile_path.with_extension("syms.json");
+        if let Some(precog_info) = shared::symbol_precog::PrecogSymbolInfo::try_load(&precog_path) {
+            for (debug_id, syms) in precog_info.into_hash_map().into_iter() {
+                let lib_info = LibraryInfo {
+                    debug_id: Some(debug_id),
+                    ..LibraryInfo::default()
+                };
+                symbol_manager.add_known_library_symbols(lib_info, syms);
+            }
+        }
+
+        let ctrl_c_receiver = CtrlC::observe_oneshot();
+
+        let open_in_browser = server_props.open_in_browser;
+
+        let RunningServerInfo {
+            server_join_handle,
+            server_origin,
+            profiler_url,
+        } = start_server(
+            Some(profile_path),
+            server_props,
+            symbol_manager,
+            ctrl_c_receiver,
+        )
+        .await;
+
+        eprintln!("Local server listening at {server_origin}");
+        if !open_in_browser {
+            if let Some(profiler_url) = &profiler_url {
+                println!("{profiler_url}");
+            }
+        }
+        eprintln!("Press Ctrl+C to stop.");
+
+        if open_in_browser {
+            if let Some(profiler_url) = &profiler_url {
+                let _ = opener::open_browser(profiler_url);
+            }
+        }
+
+        // Run this server until it stops.
+        if let Err(e) = server_join_handle.await {
+            eprintln!("server error: {e}");
+        }
+
+        if let Some(quota_manager) = quota_manager {
+            quota_manager.finish().await;
+        }
+    });
+}