feat: add memmap feature to use mmap for scanned files

Add a new feature, `memmap`, enabled by default. This adds a new
dependency and a new API to scan file using mmap to read from it.

This feature and API is used by default in boreal-cli. An option is
added to avoid using mmap.

Author: vthib

Cargo.lock

@@ -14,10 +14,12 @@ name = "boreal"
 path = "src/main.rs"
 
 [features]
-default = ["authenticode", "profiling"]
+default = ["authenticode", "memmap", "profiling"]
 
 # Enable authenticode parsing in boreal, requires OpenSSL
 authenticode = ["boreal/authenticode"]
+# Enable use of memory maps to load files to scan.
+memmap = ["boreal/memmap"]
 # Enables scan statistics. Should not impact performances
 # significantly, and very useful in a CLI tool to debug rules.
 profiling = ["boreal/profiling"]

boreal-cli/Cargo.toml

@@ -15,7 +15,7 @@ use crossbeam_channel::{bounded, Receiver, Sender};
 use walkdir::WalkDir;
 
 fn build_command() -> Command {
-    command!()
+    let mut command = command!()
         .arg(
             Arg::new("no_follow_symlinks")
                 .short('N')
@@ -89,7 +89,24 @@ fn build_command() -> Command {
                 .long("scan-stats")
                 .action(ArgAction::SetTrue)
                 .help("Display statistics on rules' evaluation"),
-        )
+        );
+
+    if cfg!(feature = "memmap") {
+        command = command.arg(
+            Arg::new("no_mmap")
+                .long("no-mmap")
+                .action(ArgAction::SetTrue)
+                .help("Disable the use of memory maps.")
+                .long_help(
+                    "Disable the use of memory maps.\n\
+                    By default, memory maps are used to load files to scan.\n\
+                    This can cause the program to abort unexpectedly \
+                    if files are simultaneous truncated.",
+                ),
+        );
+    }
+
+    command
 }
 
 fn main() -> ExitCode {
@@ -190,7 +207,7 @@ fn main() -> ExitCode {
 
         ExitCode::SUCCESS
     } else {
-        match scan_file(&scanner, input, args.get_flag("print_module_data")) {
+        match scan_file(&scanner, input, ScanOptions::new(&args)) {
             Ok(()) => ExitCode::SUCCESS,
             Err(err) => {
                 eprintln!("Cannot scan {}: {}", input.display(), err);
@@ -200,10 +217,35 @@ fn main() -> ExitCode {
     }
 }
 
-fn scan_file(scanner: &Scanner, path: &Path, print_module_data: bool) -> std::io::Result<()> {
-    let res = scanner.scan_file(path)?;
+#[derive(Copy, Clone)]
+struct ScanOptions {
+    print_module_data: bool,
+    no_mmap: bool,
+}
+
+impl ScanOptions {
+    fn new(args: &ArgMatches) -> Self {
+        Self {
+            print_module_data: args.get_flag("print_module_data"),
+            no_mmap: if cfg!(feature = "memmap") {
+                args.get_flag("no_mmap")
+            } else {
+                false
+            },
+        }
+    }
+}
+
+fn scan_file(scanner: &Scanner, path: &Path, options: ScanOptions) -> std::io::Result<()> {
+    let res = if cfg!(feature = "memmap") && !options.no_mmap {
+        // Safety: By default, we accept that this CLI tool can abort if the underlying
+        // file is truncated while the scan is ongoing.
+        unsafe { scanner.scan_file_memmap(path)? }
+    } else {
+        scanner.scan_file(path)?
+    };
 
-    if print_module_data {
+    if options.print_module_data {
         for (module_name, module_value) in res.module_values {
             // A module value must be an object. Filter out empty ones, it means the module has not
             // generated any values.
@@ -240,12 +282,11 @@ impl ThreadPool {
         };
 
         let (sender, receiver) = bounded(nb_cpus * 5);
+        let options = ScanOptions::new(args);
         (
             Self {
                 threads: (0..nb_cpus)
-                    .map(|_| {
-                        Self::worker_thread(scanner, &receiver, args.get_flag("print_module_data"))
-                    })
+                    .map(|_| Self::worker_thread(scanner, &receiver, options))
                     .collect(),
             },
             sender,
@@ -261,14 +302,14 @@ impl ThreadPool {
     fn worker_thread(
         scanner: &Scanner,
         receiver: &Receiver<PathBuf>,
-        print_module_data: bool,
+        scan_options: ScanOptions,
     ) -> JoinHandle<()> {
         let scanner = scanner.clone();
         let receiver = receiver.clone();
 
         std::thread::spawn(move || {
             while let Ok(path) = receiver.recv() {
-                if let Err(err) = scan_file(&scanner, &path, print_module_data) {
+                if let Err(err) = scan_file(&scanner, &path, scan_options) {
                     eprintln!("Cannot scan file {}: {}", path.display(), err);
                 }
             }

boreal-cli/src/main.rs

@@ -644,3 +644,34 @@ fn test_module_names() {
         // Still successful, since some other files in the directory may have been scanned
         .success();
 }
+
+#[test]
+#[cfg(feature = "memmap")]
+fn test_no_mmap() {
+    let rule_file = test_file(
+        r#"
+rule first {
+    strings:
+        $a = "abc"
+    condition:
+        any of them
+}
+rule second {
+    strings:
+        $a = "xyz"
+    condition:
+        any of them
+}"#,
+    );
+
+    let input = test_file("xyabcz");
+    // Not matching
+    cmd()
+        .arg("--no-mmap")
+        .arg(rule_file.path())
+        .arg(input.path())
+        .assert()
+        .stdout(predicate::eq(format!("first {}\n", input.path().display())))
+        .stderr("")
+        .success();
+}

boreal-cli/tests/cli.rs

@@ -12,7 +12,7 @@ edition = "2021"
 rust-version = "1.62"
 
 [features]
-default = ["hash", "object"]
+default = ["hash", "object", "memmap"]
 
 # Enables the "hash" module.
 hash = ["md-5", "sha1", "sha2", "hex", "crc32fast", "tlsh2"]
@@ -27,7 +27,10 @@ object = ["dep:object"]
 # The `object` feature must also be enabled to get access to the "pe" module.
 authenticode = ["dep:authenticode-parser"]
 
-# Enables computating of statistics during scanning.
+# Adds an API to scan files using memory maps.
+memmap = ["dep:memmap2"]
+
+# Enables computation of statistics during scanning.
 profiling = []
 
 [dependencies]
@@ -62,6 +65,9 @@ object = { version = "0.32", optional = true, default-features = false, features
 # "authenticode" feature
 authenticode-parser = { version = "0.3", optional = true }
 
+# "memmap" feature
+memmap2 = { version = "0.7", optional = true }
+
 [dev-dependencies]
 base64 = "0.21"
 glob = "0.3.1"
@@ -73,4 +79,4 @@ yara = { version = "0.19", features = ["vendored"] }
 once_cell = "1.18"
 
 [package.metadata.docs.rs]
-features = ["authenticode"]
+features = ["authenticode", "memmap"]

boreal/Cargo.toml

@@ -142,6 +142,34 @@ impl Scanner {
         Ok(self.scan_mem(&contents))
     }
 
+    /// Scan a file using memmap to read from it.
+    ///
+    /// Returns a list of rules that matched the given file.
+    ///
+    /// # Errors
+    ///
+    /// Fails if the file at the given path cannot be opened or memory mapped.
+    ///
+    /// # Safety
+    ///
+    /// See the safety documentation of [`memmap2::Mmap`]. It is unsafe to use this
+    /// method as the behavior is undefined if the underlying file is modified while the map
+    /// is still alive. For example, shrinking the underlying file can and will cause issues
+    /// in this process: on Linux, a SIGBUS can be emitted, while on Windows, a structured
+    /// exception can be raised.
+    #[cfg(feature = "memmap")]
+    pub unsafe fn scan_file_memmap<P: AsRef<std::path::Path>>(
+        &self,
+        path: P,
+    ) -> std::io::Result<ScanResult> {
+        let file = std::fs::File::open(path.as_ref())?;
+
+        // Safety: guaranteed by the safety contract of this function
+        let mmap = unsafe { memmap2::Mmap::map(&file)? };
+
+        Ok(self.scan_mem(&mmap))
+    }
+
     /// Define a value for a symbol defined and used in compiled rules.
     ///
     /// This symbol must have been defined when compiling rules using