Back off from running job on high load

Author: Mark-Simulacrum

src/runner/mod.rs

@@ -112,17 +112,15 @@ pub fn run_ex<DB: WriteResults + Sync>(
         let mut threads = Vec::new();
 
         for worker in &workers {
-            let join =
-                scope
-                    .builder()
-                    .name(worker.name().into())
-                    .spawn(move || match worker.run() {
-                        Ok(()) => Ok(()),
-                        Err(r) => {
-                            log::warn!("worker {} failed: {:?}", worker.name(), r);
-                            Err(r)
-                        }
-                    })?;
+            let join = scope.builder().name(worker.name().into()).spawn(move || {
+                match worker.run(threads_count) {
+                    Ok(()) => Ok(()),
+                    Err(r) => {
+                        log::warn!("worker {} failed: {:?}", worker.name(), r);
+                        Err(r)
+                    }
+                }
+            })?;
             threads.push(join);
         }
         let disk_watcher_thread =

src/runner/worker.rs

@@ -12,6 +12,7 @@ use std::sync::{
     Mutex,
 };
 use std::time::Duration;
+use systemstat::{Platform, System};
 
 pub(super) struct Worker<'a, DB: WriteResults + Sync> {
     name: String,
@@ -55,16 +56,32 @@ impl<'a, DB: WriteResults + Sync> Worker<'a, DB> {
         &self.name
     }
 
-    pub(super) fn run(&self) -> Fallible<()> {
+    pub(super) fn run(&self, threads_count: usize) -> Fallible<()> {
         // This uses a `loop` instead of a `while let` to avoid locking the graph too much
         let mut guard = self.graph.lock().unwrap();
+        let system = System::new();
         loop {
             self.maybe_cleanup_target_dir()?;
             let walk_result = guard.next_task(self.ex, self.db, &self.name);
             match walk_result {
                 WalkResult::Task(id, task) => {
                     drop(guard);
                     info!("running task: {:?}", task);
+
+                    // Wait for 15 seconds before running if the 1 minute load
+                    // average exceeds the thread count. This tries to back off
+                    // from spawning too many jobs on the server, hopefully
+                    // improving performance.
+                    loop {
+                        let avg = system.load_average()?;
+
+                        if avg.one > threads_count as f32 {
+                            std::thread::sleep(std::time::Duration::new(15, 0));
+                        } else {
+                            break;
+                        }
+                    }
+
                     let res = task.run(
                         self.config,
                         self.workspace,