create IndexStorage enum rather than accessing the filesystem directly

Author: pietroalbini

src/bin/offline/extract.rs

@@ -1,6 +1,7 @@
 use crate::offline;
 use crate::rla;
 
+use rla::index::IndexStorage;
 use std::fs;
 use std::io::{self, Write};
 use std::path::Path;
@@ -31,7 +32,7 @@ fn load_lines<'a>(ci: &dyn rla::ci::CiPlatform, log: &'a [u8]) -> Vec<Line<'a>>
 
 pub fn dir(
     ci: &dyn rla::ci::CiPlatform,
-    index_file: &Path,
+    index_file: &IndexStorage,
     src_dir: &Path,
     dst_dir: &Path,
 ) -> rla::Result<()> {
@@ -91,7 +92,11 @@ pub fn dir(
     Ok(())
 }
 
-pub fn one(ci: &dyn rla::ci::CiPlatform, index_file: &Path, log_file: &Path) -> rla::Result<()> {
+pub fn one(
+    ci: &dyn rla::ci::CiPlatform,
+    index_file: &IndexStorage,
+    log_file: &Path,
+) -> rla::Result<()> {
     let config = rla::extract::Config::default();
     let index = rla::Index::load(index_file)?;
 

src/bin/offline/learn.rs

@@ -1,14 +1,15 @@
 use crate::offline;
 use crate::rla;
 
-use std::path::{Path, PathBuf};
+use rla::index::IndexStorage;
+use std::path::PathBuf;
 use std::time::Duration;
 use std::time::Instant;
 use walkdir::{self, WalkDir};
 
 pub fn learn(
     ci: &dyn rla::ci::CiPlatform,
-    index_file: &Path,
+    index_file: &IndexStorage,
     inputs: &[PathBuf],
     multiplier: u32,
 ) -> rla::Result<()> {

src/bin/rla-offline.rs

@@ -12,6 +12,7 @@ extern crate rust_log_analyzer as rla;
 extern crate walkdir;
 
 use clap::Parser;
+use rla::index::IndexStorage;
 use std::path::PathBuf;
 
 mod offline;
@@ -53,7 +54,7 @@ enum Cli {
             long = "index-file",
             help = "The index file to read / write. An existing index file is updated."
         )]
-        index_file: PathBuf,
+        index_file: IndexStorage,
         #[arg(
             short = 'm',
             long = "multiplier",
@@ -79,7 +80,7 @@ enum Cli {
             long = "index-file",
             help = "The index file to read / write."
         )]
-        index_file: PathBuf,
+        index_file: IndexStorage,
         #[arg(
             short = 's',
             long = "source",
@@ -106,7 +107,7 @@ enum Cli {
             long = "index-file",
             help = "The index file to read / write."
         )]
-        index_file: PathBuf,
+        index_file: IndexStorage,
         #[arg(help = "The log file to analyze.")]
         log: PathBuf,
     },

src/bin/rla-server.rs

@@ -15,6 +15,7 @@ extern crate rust_log_analyzer as rla;
 extern crate serde_json;
 
 use clap::Parser;
+use rla::index::IndexStorage;
 use std::process;
 use std::sync::Arc;
 use std::thread;
@@ -47,7 +48,7 @@ struct Cli {
         long = "index-file",
         help = "The index file to read / write. An existing index file is updated."
     )]
-    index_file: std::path::PathBuf,
+    index_file: IndexStorage,
     #[arg(
         long = "debug-post",
         help = "Post all comments to the given issue instead of the actual PR. Format: \"user/repo#id\""

src/bin/server/worker.rs

@@ -3,14 +3,14 @@ use super::{QueueItem, QueueItemKind};
 use crate::rla;
 use crate::rla::ci::{self, BuildCommit, CiPlatform};
 use anyhow::bail;
+use rla::index::IndexStorage;
 use std::collections::{HashSet, VecDeque};
 use std::hash::Hash;
-use std::path::PathBuf;
 use std::str;
 
 pub struct Worker {
     debug_post: Option<(String, u32)>,
-    index_file: PathBuf,
+    index_file: IndexStorage,
     index: rla::Index,
     extract_config: rla::extract::Config,
     github: rla::github::Client,
@@ -26,7 +26,7 @@ pub struct Worker {
 
 impl Worker {
     pub fn new(
-        index_file: PathBuf,
+        index_file: IndexStorage,
         debug_post: Option<String>,
         queue: crossbeam::channel::Receiver<QueueItem>,
         ci: Box<dyn CiPlatform + Send>,

src/index/mod.rs

@@ -1,14 +1,11 @@
 use super::Result;
-use atomicwrites;
-use bincode;
-use fnv;
-use std;
-use std::fs;
-use std::path::Path;
 use std::slice;
 
+mod storage;
 mod table;
 
+pub use self::storage::IndexStorage;
+
 pub trait IndexData {
     fn sanitized(&self) -> &[u8];
 }
@@ -45,37 +42,50 @@ impl Index {
             .into_iter()
     }
 
-    pub fn save(&self, path: &Path) -> Result<()> {
-        debug!("Saving index to '{}'...", path.display());
-        let file = atomicwrites::AtomicFile::new(path, atomicwrites::AllowOverwrite);
-        file.write(|f| bincode::serialize_into(f, self))?;
+    pub fn save(&self, storage: &IndexStorage) -> Result<()> {
+        debug!("Saving index to '{storage}'...");
+        storage.write(self)?;
         debug!("Index saved.");
         Ok(())
     }
 
-    pub fn load(path: &Path) -> Result<Index> {
-        Index::load_or_create_internal(path, false)
+    pub fn load(storage: &IndexStorage) -> Result<Index> {
+        Index::load_or_create_internal(storage, false)
     }
 
-    pub fn load_or_create(path: &Path) -> Result<Index> {
-        Index::load_or_create_internal(path, true)
+    pub fn load_or_create(storage: &IndexStorage) -> Result<Index> {
+        Index::load_or_create_internal(storage, true)
     }
 
-    fn load_or_create_internal(path: &Path, create: bool) -> Result<Index> {
-        let index;
-
-        if path.exists() || !create {
-            info!("Loading index...");
-            index = bincode::deserialize_from(fs::File::open(path)?)?;
+    fn load_or_create_internal(storage: &IndexStorage, create: bool) -> Result<Index> {
+        info!("Loading index...");
+        let index = if let Some(index) = storage.read()? {
+            index
         } else {
-            info!("Initializing new index...");
-            index = Index::default();
+            if create {
+                info!("Index missing, initializing new index...");
+                Index::default()
+            } else {
+                anyhow::bail!("missing index, aborting");
+            }
         };
 
         info!("Index ready ({} keys).", index.internal.len());
 
         Ok(index)
     }
+
+    fn deserialize(reader: &mut dyn std::io::Read) -> Result<Self> {
+        Ok(bincode::deserialize_from(reader)?)
+    }
+
+    fn serialize(
+        &self,
+        writer: &mut dyn std::io::Write,
+    ) -> std::result::Result<(), bincode::Error> {
+        bincode::serialize_into(writer, self)?;
+        Ok(())
+    }
 }
 
 pub fn encode<I: IndexData>(data: &I) -> Vec<u8> {

src/index/storage.rs

@@ -0,0 +1,68 @@
+use crate::{Index, Result};
+use atomicwrites::{AtomicFile, OverwriteBehavior};
+use std::fs::File;
+use std::path::PathBuf;
+use std::str::FromStr;
+
+#[derive(Debug, Clone)]
+pub enum IndexStorage {
+    FileSystem(FileSystemStorage),
+}
+
+impl IndexStorage {
+    pub fn new(path: &str) -> Result<Self> {
+        Ok(IndexStorage::FileSystem(FileSystemStorage {
+            path: path.into(),
+        }))
+    }
+
+    pub(super) fn read(&self) -> Result<Option<Index>> {
+        match self {
+            IndexStorage::FileSystem(fs) => fs.read(),
+        }
+    }
+
+    pub(super) fn write(&self, index: &Index) -> Result<()> {
+        match self {
+            IndexStorage::FileSystem(fs) => fs.write(index),
+        }
+    }
+}
+
+impl FromStr for IndexStorage {
+    type Err = anyhow::Error;
+
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        IndexStorage::new(s)
+    }
+}
+
+impl std::fmt::Display for IndexStorage {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            IndexStorage::FileSystem(fs) => write!(f, "{}", fs.path.display()),
+        }
+    }
+}
+
+#[derive(Debug, Clone)]
+pub struct FileSystemStorage {
+    path: PathBuf,
+}
+
+impl FileSystemStorage {
+    fn read(&self) -> Result<Option<Index>> {
+        let mut file = match File::open(&self.path) {
+            Ok(file) => file,
+            Err(err) if err.kind() == std::io::ErrorKind::NotFound => return Ok(None),
+            Err(err) => return Err(err.into()),
+        };
+        Ok(Some(Index::deserialize(&mut file)?))
+    }
+
+    fn write(&self, index: &Index) -> Result<()> {
+        let file = AtomicFile::new(&self.path, OverwriteBehavior::AllowOverwrite);
+        file.write(|inner| index.serialize(inner))?;
+        Ok(())
+    }
+}