separate update intervals (#187)

Author: ckrause

CHANGELOG.md

@@ -2,6 +2,10 @@ To install or update LODA, please follow the [installation instructions](https:/
 
 ## [Unreleased]
 
+### Enhancements
+
+* Separate update intervals for OEIS files (7 days) and programs repository (1 day)
+
 # v22.10.13
 
 ### Features

src/include/oeis_manager.hpp

@@ -76,7 +76,8 @@ class OeisManager {
   Evaluator evaluator;
   Finder finder;
   bool finder_initialized;
-  bool update_needed;
+  bool update_oeis;
+  bool update_programs;
 
   Optimizer optimizer;
   Minimizer minimizer;

src/include/setup.hpp

@@ -53,7 +53,9 @@ class Setup {
 
   static int64_t getMaxMemory();
 
-  static int64_t getUpdateIntervalInDays();
+  static int64_t getGitHubUpdateInterval();
+
+  static int64_t getOeisUpdateInterval();
 
   static int64_t getMaxLocalProgramAgeInDays();
 
@@ -72,9 +74,10 @@ class Setup {
   static void runWizard();
 
  private:
-  static constexpr int64_t UNDEFINED_INT = -2;            // cannot use -1
-  static constexpr int64_t DEFAULT_UPDATE_INTERVAL = 1;   // 1 day default
-  static constexpr int64_t DEFAULT_MAX_PROGRAM_AGE = 14;  // 2 weeks default
+  static constexpr int64_t UNDEFINED_INT = -2;                  // cannot use -1
+  static constexpr int64_t DEFAULT_GITHUB_UPDATE_INTERVAL = 1;  // 1 day default
+  static constexpr int64_t DEFAULT_OEIS_UPDATE_INTERVAL = 7;  // 1 week default
+  static constexpr int64_t DEFAULT_MAX_PROGRAM_AGE = 14;      // 2 weeks default
   static constexpr int64_t DEFAULT_MAX_PHYSICAL_MEMORY = 1024;  // 1 GB
 
   static std::string LODA_HOME;
@@ -86,7 +89,8 @@ class Setup {
   static bool PRINTED_MEMORY_WARNING;
   static int64_t MINING_MODE;
   static int64_t MAX_MEMORY;
-  static int64_t UPDATE_INTERVAL;
+  static int64_t GITHUB_UPDATE_INTERVAL;
+  static int64_t OEIS_UPDATE_INTERVAL;
   static int64_t MAX_PROGRAM_AGE;
   static int64_t MAX_INSTANCES;
 
@@ -104,7 +108,6 @@ class Setup {
   static bool checkSubmittedBy();
   static bool checkUsageStats();
   static bool checkMaxMemory();
-  static bool checkUpdateInterval();
   static bool checkMaxLocalProgramAge();
   static bool checkMaxInstances();
 

src/oeis_manager.cpp

@@ -36,7 +36,8 @@ OeisManager::OeisManager(const Settings &settings,
       evaluator(settings),
       finder(settings, evaluator),
       finder_initialized(false),
-      update_needed(false),
+      update_oeis(false),
+      update_programs(false),
       optimizer(settings),
       minimizer(settings),
       loaded_count(0),
@@ -305,48 +306,78 @@ bool OeisManager::shouldMatch(const OeisSequence &seq) const {
 void OeisManager::update() {
   std::vector<std::string> files = {"stripped", "names"};
 
-  // check which files need to be updated
+  // check whether oeis files need to be updated
   auto it = files.begin();
-  int64_t age_in_days = -1;
+  int64_t oeis_age_in_days = -1;
   while (it != files.end()) {
     auto path = Setup::getOeisHome() + *it;
-    age_in_days = getFileAgeInDays(path);
-    if (age_in_days < 0 || age_in_days >= Setup::getUpdateIntervalInDays()) {
-      update_needed = true;
+    oeis_age_in_days = getFileAgeInDays(path);
+    if (oeis_age_in_days < 0 ||
+        oeis_age_in_days >= Setup::getOeisUpdateInterval()) {
+      update_oeis = true;
       break;
     }
     it++;
   }
-  if (update_needed) {
+
+  // check whether programs need to be updated
+  const std::string progs_dir = Setup::getProgramsHome();
+  const std::string local_dir = progs_dir + "local";
+  const std::string update_progs_file = local_dir + FILE_SEP + ".update";
+  int64_t programs_age_in_days = getFileAgeInDays(update_progs_file);
+  if (programs_age_in_days < 0) {
+    update_programs = update_oeis;  // fall-back if file is not present
+  } else if (programs_age_in_days >= Setup::getGitHubUpdateInterval()) {
+    update_programs = true;
+  }
+
+  // perform oeis update
+  if (update_oeis) {
     Setup::checkLatestedVersion();
-    if (age_in_days == -1) {
+    if (oeis_age_in_days == -1) {
       Log::get().info("Creating OEIS index at \"" + Setup::getOeisHome() +
                       "\"");
       ensureDir(Setup::getOeisHome());
     } else {
       Log::get().info("Updating OEIS index (last update " +
-                      std::to_string(age_in_days) + " days ago)");
+                      std::to_string(oeis_age_in_days) + " days ago)");
     }
     std::string cmd, path;
     for (auto &file : files) {
       path = Setup::getOeisHome() + file;
       ApiClient::getDefaultInstance().getOeisFile(file, path);
     }
-    // update programs repository using git pull
+  }
+
+  // perform programs update
+  if (update_programs) {
     auto mode = Setup::getMiningMode();
-    auto progs_dir = Setup::getProgramsHome();
     if (mode != MINING_MODE_SERVER && isDir(progs_dir + ".git")) {
-      Log::get().info("Updating programs repository");
+      std::string msg("Updating programs repository");
+      if (programs_age_in_days >= 0) {
+        msg += " (last update " + std::to_string(programs_age_in_days) +
+               " days ago)";
+      }
+      Log::get().info(msg);
+      // update programs repository using git pull
       git(progs_dir, "pull origin main -q --ff-only");
+      // touch marker file to track the age
+      ensureDir(update_progs_file);
+      std::ofstream marker(update_progs_file);
+      if (marker) {
+        marker << "1" << std::endl;
+      } else {
+        Log::get().warn("Cannot write update marker: " + update_progs_file);
+      }
     }
+
     // clean up local programs folder
     const int64_t max_age = Setup::getMaxLocalProgramAgeInDays();
-    const auto local = Setup::getProgramsHome() + "local";
-    if (max_age >= 0 && isDir(local) &&
+    if (max_age >= 0 && isDir(local_dir) &&
         Setup::getMiningMode() == MiningMode::MINING_MODE_CLIENT) {
       Log::get().info("Cleaning up local programs directory");
       int64_t num_removed = 0;
-      for (const auto &it : std::filesystem::directory_iterator(local)) {
+      for (const auto &it : std::filesystem::directory_iterator(local_dir)) {
         const auto stem = it.path().filename().stem().string();
         const auto ext = it.path().filename().extension().string();
         bool is_program;
@@ -531,9 +562,11 @@ const Stats &OeisManager::getStats() {
     stats.reset(new Stats());
 
     // check age of stats
+    auto update_interval = std::min<int64_t>(Setup::getOeisUpdateInterval(),
+                                             Setup::getGitHubUpdateInterval());
     auto age_in_days = getFileAgeInDays(stats->getMainStatsFile(stats_home));
-    if (update_needed || age_in_days < 0 ||
-        age_in_days >= Setup::getUpdateIntervalInDays()) {
+    if (update_oeis || update_programs || age_in_days < 0 ||
+        age_in_days >= update_interval) {
       generateStats(age_in_days);
 
       // generate lists in server mode

src/setup.cpp

@@ -27,7 +27,8 @@ bool Setup::LOADED_SETUP = false;
 bool Setup::PRINTED_MEMORY_WARNING = false;
 int64_t Setup::MINING_MODE = UNDEFINED_INT;
 int64_t Setup::MAX_MEMORY = UNDEFINED_INT;
-int64_t Setup::UPDATE_INTERVAL = UNDEFINED_INT;
+int64_t Setup::GITHUB_UPDATE_INTERVAL = UNDEFINED_INT;
+int64_t Setup::OEIS_UPDATE_INTERVAL = UNDEFINED_INT;
 int64_t Setup::MAX_PROGRAM_AGE = UNDEFINED_INT;
 int64_t Setup::MAX_INSTANCES = UNDEFINED_INT;
 
@@ -98,7 +99,7 @@ std::string Setup::getMinersConfig() {
   {
     FolderLock lock(getLodaHome());
     const auto age_in_days = getFileAgeInDays(default_config);
-    if (age_in_days < 0 || age_in_days >= getUpdateIntervalInDays()) {
+    if (age_in_days < 0 || age_in_days >= getGitHubUpdateInterval()) {
       const std::string url =
           "https://raw.githubusercontent.com/loda-lang/loda-cpp/main/"
           "miners.default.json";
@@ -204,12 +205,20 @@ int64_t Setup::getMaxMemory() {
   return MAX_MEMORY;
 }
 
-int64_t Setup::getUpdateIntervalInDays() {
-  if (UPDATE_INTERVAL == UNDEFINED_INT) {
-    UPDATE_INTERVAL =
-        getSetupInt("LODA_UPDATE_INTERVAL", DEFAULT_UPDATE_INTERVAL);
+int64_t Setup::getGitHubUpdateInterval() {
+  if (GITHUB_UPDATE_INTERVAL == UNDEFINED_INT) {
+    GITHUB_UPDATE_INTERVAL = getSetupInt("LODA_GITHUB_UPDATE_INTERVAL",
+                                         DEFAULT_GITHUB_UPDATE_INTERVAL);
   }
-  return UPDATE_INTERVAL;
+  return GITHUB_UPDATE_INTERVAL;
+}
+
+int64_t Setup::getOeisUpdateInterval() {
+  if (OEIS_UPDATE_INTERVAL == UNDEFINED_INT) {
+    OEIS_UPDATE_INTERVAL =
+        getSetupInt("LODA_OEIS_UPDATE_INTERVAL", DEFAULT_OEIS_UPDATE_INTERVAL);
+  }
+  return OEIS_UPDATE_INTERVAL;
 }
 
 int64_t Setup::getMaxLocalProgramAgeInDays() {
@@ -337,11 +346,6 @@ void Setup::runWizard() {
     if (!checkMaxMemory()) {
       return;
     }
-    // since we get the oeis data from the API server, we don't need to
-    // configure this anymore
-    // if (!checkUpdateInterval()) {
-    //  return;
-    // }
 #ifdef STD_FILESYSTEM
     if (getMiningMode() == MINING_MODE_CLIENT && !checkMaxLocalProgramAge()) {
       return;
@@ -699,28 +703,6 @@ bool Setup::checkMaxMemory() {
   return true;
 }
 
-bool Setup::checkUpdateInterval() {
-  std::string line;
-  std::cout << "Enter the update interval for the main OEIS files and the"
-            << std::endl
-            << "programs repository in days (default " +
-                   std::to_string(DEFAULT_UPDATE_INTERVAL)
-            << "):" << std::endl;
-  int64_t update_interval = getUpdateIntervalInDays();
-  std::cout << "[" << update_interval << "] ";
-  std::getline(std::cin, line);
-  if (!line.empty()) {
-    update_interval = std::stoll(line);
-  }
-  if (update_interval <= 0) {
-    std::cout << "Invalid value. Please restart the setup." << std::endl;
-    return false;
-  }
-  SETUP["LODA_UPDATE_INTERVAL"] = std::to_string(update_interval);
-  std::cout << std::endl;
-  return true;
-}
-
 bool Setup::checkMaxLocalProgramAge() {
   std::string line;
   std::cout << "Enter the maximum age of local programs in days (default: " +