Add metric for agent failures

Author: Mark-Simulacrum

src/server/metrics.rs

@@ -11,11 +11,13 @@ use prometheus::{
 
 const JOBS_METRIC: &str = "crater_completed_jobs_total";
 const AGENT_WORK_METRIC: &str = "crater_agent_supposed_to_work";
+const AGENT_FAILED: &str = "crater_agent_failure";
 const LAST_CRATES_UPDATE_METRIC: &str = "crater_last_crates_update";
 
 #[derive(Clone)]
 pub struct Metrics {
     crater_completed_jobs_total: IntCounterVec,
+    crater_agent_failure: IntCounterVec,
     crater_work_status: IntGaugeVec,
     crater_last_crates_update: IntGauge,
 }
@@ -25,6 +27,9 @@ impl Metrics {
         let jobs_opts = prometheus::opts!(JOBS_METRIC, "total completed jobs");
         let crater_completed_jobs_total =
             prometheus::register_int_counter_vec!(jobs_opts, &["agent", "experiment"])?;
+        let failure_opts = prometheus::opts!(AGENT_FAILED, "total completed jobs");
+        let crater_agent_failure =
+            prometheus::register_int_counter_vec!(failure_opts, &["agent", "experiment"])?;
         let agent_opts = prometheus::opts!(AGENT_WORK_METRIC, "is agent supposed to work");
         let crater_work_status = prometheus::register_int_gauge_vec!(agent_opts, &["agent"])?;
         let crates_update_opts =
@@ -33,11 +38,18 @@ impl Metrics {
 
         Ok(Metrics {
             crater_completed_jobs_total,
+            crater_agent_failure,
             crater_work_status,
             crater_last_crates_update,
         })
     }
 
+    pub fn record_error(&self, agent: &str, experiment: &str) {
+        self.crater_agent_failure
+            .with_label_values(&[agent, experiment])
+            .inc_by(1);
+    }
+
     pub fn record_completed_jobs(&self, agent: &str, experiment: &str, amount: i64) {
         self.crater_completed_jobs_total
             .with_label_values(&[agent, experiment])

src/server/routes/agent.rs

@@ -209,7 +209,7 @@ fn endpoint_error(
     let mut ex = Experiment::get(&data.db, &error.experiment_name)?
         .ok_or_else(|| err_msg("no experiment run by this agent"))?;
 
-    //also set status to failed
+    data.metrics.record_error(&auth.name, &ex.name);
     ex.handle_failure(&data.db, &Assignee::Agent(auth.name))?;
 
     Ok(ApiResponse::Success { result: true }.into_response()?)