improve static code analysis and validation

Author: ckrause

.github/workflows/full_test.yml

@@ -18,8 +18,10 @@ jobs:
         run: |
           mkdir -p $HOME/loda
           git clone https://github.com/loda-lang/loda-programs.git $HOME/loda/programs
-      - name: Test IE
-        run: ./loda test-ie
+      - name: Test IncEval
+        run: ./loda test-inceval
+      - name: Test LogEval
+        run: ./loda test-logeval
       - name: Test PARI
         run: |
           sudo apt-get update

.vscode/settings.json

@@ -76,7 +76,8 @@
         "vector": "cpp",
         "algorithm": "cpp",
         "filesystem": "cpp",
-        "queue": "cpp"
+        "queue": "cpp",
+        "functional": "cpp"
     },
     "makefile.makeDirectory": "src"
 }

CHANGELOG.md

@@ -2,6 +2,18 @@ To install or update LODA, please follow the [installation instructions](https:/
 
 ## [Unreleased]
 
+## v22.12.14
+
+### Bugfixes
+
+* Fix validation for IE programs
+* Use transitive hash values for programs with `seq`
+
+### Enhancements
+
+* Add detection of programs with logarithmic complexity
+* Add stats for incremental and logarithmic eval
+
 ## v22.12.9
 
 ### Bugfixes

src/Makefile

@@ -8,7 +8,7 @@ ifdef LODA_PLATFORM
 CXXFLAGS += -DLODA_PLATFORM=$(LODA_PLATFORM)
 endif
 
-OBJS = api_client.o benchmark.o big_number.o blocks.o boinc.o commands.o config.o distribution.o evaluator.o evaluator_inc.o expression.o expression_util.o extender.o external/jute.o file.o finder.o formula.o formula_gen.o generator.o generator_v1.o generator_v2.o generator_v3.o generator_v4.o generator_v5.o generator_v6.o generator_v7.o interpreter.o iterator.o log.o main.o matcher.o memory.o metrics.o miner.o minimizer.o mutator.o number.o oeis_list.o oeis_manager.o oeis_sequence.o optimizer.o pari.o parser.o process.o program.o program_util.o reducer.o semantics.o setup.o sequence.o stats.o test.o util.o web_client.o
+OBJS = api_client.o benchmark.o big_number.o blocks.o boinc.o commands.o config.o distribution.o evaluator.o evaluator_inc.o evaluator_log.o expression.o expression_util.o extender.o external/jute.o file.o finder.o formula.o formula_gen.o generator.o generator_v1.o generator_v2.o generator_v3.o generator_v4.o generator_v5.o generator_v6.o generator_v7.o interpreter.o iterator.o log.o main.o matcher.o memory.o metrics.o miner.o minimizer.o mutator.o number.o oeis_list.o oeis_manager.o oeis_sequence.o optimizer.o pari.o parser.o process.o program.o program_util.o reducer.o semantics.o setup.o sequence.o stats.o test.o util.o web_client.o
 
 loda: external/jute.h external/jute.cpp $(OBJS)
 	$(CXX) $(LDFLAGS) -o loda $(OBJS)

src/Makefile.windows.mk

@@ -10,7 +10,7 @@ CXXFLAGS = $(CXXFLAGS) /Z7
 CXXFLAGS = $(CXXFLAGS) /DLODA_PLATFORM=$(LODA_PLATFORM)
 !ENDIF
 
-SRCS = api_client.cpp benchmark.cpp big_number.cpp blocks.cpp boinc.cpp commands.cpp config.cpp distribution.cpp evaluator.cpp evaluator_inc.cpp expression.cpp expression_util.cpp extender.cpp external/jute.cpp file.cpp finder.cpp formula.cpp formula_gen.cpp generator.cpp generator_v1.cpp generator_v2.cpp generator_v3.cpp generator_v4.cpp generator_v5.cpp generator_v6.cpp generator_v7.cpp interpreter.cpp iterator.cpp log.cpp main.cpp matcher.cpp memory.cpp metrics.cpp miner.cpp minimizer.cpp mutator.cpp number.cpp oeis_list.cpp oeis_manager.cpp oeis_sequence.cpp optimizer.cpp pari.cpp parser.cpp process.cpp program.cpp program_util.cpp reducer.cpp semantics.cpp sequence.cpp setup.cpp stats.cpp test.cpp util.cpp web_client.cpp
+SRCS = api_client.cpp benchmark.cpp big_number.cpp blocks.cpp boinc.cpp commands.cpp config.cpp distribution.cpp evaluator.cpp evaluator_inc.cpp evaluator_log.cpp expression.cpp expression_util.cpp extender.cpp external/jute.cpp file.cpp finder.cpp formula.cpp formula_gen.cpp generator.cpp generator_v1.cpp generator_v2.cpp generator_v3.cpp generator_v4.cpp generator_v5.cpp generator_v6.cpp generator_v7.cpp interpreter.cpp iterator.cpp log.cpp main.cpp matcher.cpp memory.cpp metrics.cpp miner.cpp minimizer.cpp mutator.cpp number.cpp oeis_list.cpp oeis_manager.cpp oeis_sequence.cpp optimizer.cpp pari.cpp parser.cpp process.cpp program.cpp program_util.cpp reducer.cpp semantics.cpp sequence.cpp setup.cpp stats.cpp test.cpp util.cpp web_client.cpp
 
 loda: external/jute.h external/jute.cpp $(SRCS)
 	cl /EHsc /Feloda.exe $(CXXFLAGS) $(SRCS)

src/commands.cpp

@@ -1,11 +1,13 @@
 #include "commands.hpp"
 
+#include <fstream>
 #include <iostream>
 
 #include "benchmark.hpp"
 #include "boinc.hpp"
 #include "evaluator.hpp"
 #include "evaluator_inc.hpp"
+#include "evaluator_log.hpp"
 #include "formula_gen.hpp"
 #include "iterator.hpp"
 #include "log.hpp"
@@ -317,6 +319,32 @@ void Commands::testIncEval() {
                   std::to_string(count) + " programs");
 }
 
+void Commands::testLogEval() {
+  initLog(false);
+  Log::get().info("Testing logarithmic evaluator");
+  Parser parser;
+  OeisManager manager(settings);
+  auto& stats = manager.getStats();
+  int64_t count = 0;
+  for (size_t id = 0; id < stats.all_program_ids.size(); id++) {
+    if (!stats.all_program_ids[id]) {
+      continue;
+    }
+    OeisSequence seq(id);
+    std::ifstream in(seq.getProgramPath());
+    if (!in) {
+      continue;
+    }
+    auto program = parser.parse(in);
+    if (LogarithmicEvaluator::hasLogarithmicComplexity(program)) {
+      Log::get().info(seq.id_str() + " has logarithmic complexity");
+      count++;
+    }
+  }
+  Log::get().info(std::to_string(count) +
+                  " programs have logarithmic complexity");
+}
+
 void Commands::testPari() {
   initLog(false);
   Parser parser;

src/evaluator_log.cpp

@@ -0,0 +1,82 @@
+#include "evaluator_log.hpp"
+
+#include "program_util.hpp"
+
+bool LogarithmicEvaluator::hasLogarithmicComplexity(const Program& program) {
+  // split up the program into fragments:
+  Program pre_loop, loop_body;
+  int64_t phase = 0;
+  int64_t loop_counter_cell = 0;
+  for (auto& op : program.ops) {
+    if (op.type == Operation::Type::NOP) {
+      continue;
+    }
+    // check for forbidden operation/operand types
+    if (op.type == Operation::Type::SEQ || op.type == Operation::Type::CLR ||
+        ProgramUtil::hasIndirectOperand(op)) {
+      return false;
+    }
+    if (op.type == Operation::Type::LPB) {
+      if (phase != 0 || op.target.type != Operand::Type::DIRECT ||
+          op.source != Operand(Operand::Type::CONSTANT, 1)) {
+        return false;
+      }
+      loop_counter_cell = op.target.value.asInt();
+      phase = 1;
+      continue;
+    }
+    if (op.type == Operation::Type::LPE) {
+      if (phase != 1) {
+        return false;
+      }
+      phase = 2;
+      continue;
+    }
+    if (phase == 0) {
+      pre_loop.ops.push_back(op);
+    } else if (phase == 1) {
+      loop_body.ops.push_back(op);
+    }
+  }
+
+  // need to be in the post-loop phase here for success
+  if (phase != 2) {
+    return false;
+  }
+
+  // static code analysis of the pre-loop fragment
+  for (auto& op : pre_loop.ops) {
+    // exponentiation allowed only for constant exponents, because
+    // it could result in exponential growth of the loop counter
+    if (op.type == Operation::Type::POW &&
+        op.source.type != Operand::Type::CONSTANT) {
+      return false;
+    }
+  }
+
+  // check updates of loop counter cell in loop body
+  bool loop_counter_updated = false;
+  for (auto& op : loop_body.ops) {
+    const auto target = op.target.value.asInt();
+    if (target == loop_counter_cell) {
+      // loop counter must be updated using division
+      if (op.type == Operation::Type::DIV || op.type == Operation::Type::DIF) {
+        loop_counter_updated = true;
+      } else if (op.type != Operation::Type::SUB &&
+                 op.type != Operation::Type::TRN) {
+        // more updates using subtraction are ok, but nothing else
+        return false;
+      }
+      // all updates must be using a constant argument
+      if (op.source.type != Operand::Type::CONSTANT) {
+        return false;
+      }
+    }
+  }
+  if (!loop_counter_updated) {
+    return false;
+  }
+
+  // success: program has log complexity
+  return true;
+}

src/finder.cpp

@@ -2,9 +2,11 @@
 
 #include <fstream>
 #include <iomanip>
+#include <set>
 #include <sstream>
 
 #include "config.hpp"
+#include "evaluator_log.hpp"
 #include "file.hpp"
 #include "log.hpp"
 #include "number.hpp"
@@ -215,7 +217,7 @@ std::pair<std::string, Program> Finder::checkProgramBasic(
                                  num_default_terms);
     }
     // compare with hash of existing program
-    if (previous_hash != ProgramUtil::hash(existing)) {
+    if (previous_hash != getTransitiveProgramHash(existing)) {
       Log::get().debug("Skipping update of " + seq.id_str() +
                        " because of hash mismatch");
       return result;
@@ -265,6 +267,21 @@ bool isBetterIndirectMemory(const Program &existing, const Program &optimized) {
           !ProgramUtil::hasOp(optimized, Operation::Type::SEQ));
 }
 
+bool isBetterIncEval(const Program &existing, const Program &optimized,
+                     Evaluator &evaluator) {
+  return (ProgramUtil::hasOp(existing, Operation::Type::LPB) &&
+          (ProgramUtil::hasOp(existing, Operation::Type::SEQ) ||
+           !ProgramUtil::hasOp(optimized, Operation::Type::SEQ)) &&
+          !evaluator.supportsIncEval(existing) &&
+          evaluator.supportsIncEval(optimized));
+}
+
+bool isBetterLogEval(const Program &existing, const Program &optimized) {
+  return (ProgramUtil::hasOp(existing, Operation::Type::LPB) &&
+          !LogarithmicEvaluator::hasLogarithmicComplexity(existing) &&
+          LogarithmicEvaluator::hasLogarithmicComplexity(optimized));
+}
+
 std::string Finder::isOptimizedBetter(Program existing, Program optimized,
                                       const OeisSequence &seq) {
   static const std::string not_better;
@@ -295,17 +312,18 @@ std::string Finder::isOptimizedBetter(Program existing, Program optimized,
     return not_better;
   }
 
-  // check if the optimized program supports IE
-  if (ProgramUtil::hasOp(existing, Operation::Type::LPB) &&
-      (ProgramUtil::hasOp(existing, Operation::Type::SEQ) ||
-       !ProgramUtil::hasOp(optimized, Operation::Type::SEQ))) {
-    const bool inc_eval_existing = evaluator.supportsIncEval(existing);
-    const bool inc_eval_optimized = evaluator.supportsIncEval(optimized);
-    if (inc_eval_optimized && !inc_eval_existing) {
-      return "Faster (IE)";
-    } else if (!inc_eval_optimized && inc_eval_existing) {
-      return not_better;  // worse
-    }
+  // check if the optimized program has logarithmic complexity
+  if (isBetterLogEval(existing, optimized)) {
+    return "Faster (log)";
+  } else if (isBetterLogEval(optimized, existing)) {
+    return not_better;  // worse
+  }
+
+  // check if the optimized program supports incremental evaluation
+  if (isBetterIncEval(existing, optimized, evaluator)) {
+    return "Faster (IE)";
+  } else if (isBetterIncEval(optimized, existing, evaluator)) {
+    return not_better;  // worse
   }
 
   // check if there are loops with contant number of iterations involved
@@ -324,6 +342,8 @@ std::string Finder::isOptimizedBetter(Program existing, Program optimized,
   } else if (optimized_bad_count > existing_bad_count) {
     return not_better;  // worse
   }
+
+  // check indirect memory
   if (isBetterIndirectMemory(existing, optimized)) {
     return "Simpler";
   } else if (isBetterIndirectMemory(optimized, existing)) {
@@ -364,13 +384,13 @@ std::string Finder::isOptimizedBetter(Program existing, Program optimized,
   const auto existing_steps = evaluator.eval(existing, tmp, num_terms, false);
 
   // compare number of successfully computed terms
-  double existing_terms = existing_steps.runs;
-  double optimized_terms = optimized_steps.runs;
-  if (optimized_terms > (existing_terms * THRESHOLD_BETTER)) {
-    return "Better";
-  } else if (existing_terms > (optimized_terms * THRESHOLD_BETTER)) {
-    return not_better;  // worse
-  }
+  // double existing_terms = existing_steps.runs;
+  // double optimized_terms = optimized_steps.runs;
+  // if (optimized_terms > (existing_terms * THRESHOLD_BETTER)) {
+  //  return "Better";
+  //} else if (existing_terms > (optimized_terms * THRESHOLD_BETTER)) {
+  //  return not_better;  // worse
+  //}
 
   //  compare number of execution cycles
   double existing_total = existing_steps.total;
@@ -384,6 +404,37 @@ std::string Finder::isOptimizedBetter(Program existing, Program optimized,
   return not_better;  // not better or worse => no change
 }
 
+void collectPrograms(const Program &p, std::set<Program> &collected) {
+  if (collected.find(p) != collected.end()) {
+    return;
+  }
+  collected.insert(p);
+  for (auto &op : p.ops) {
+    if (op.type == Operation::Type::SEQ &&
+        op.source.type == Operand::Type::CONSTANT) {
+      auto id = op.source.value.asInt();
+      auto path = OeisSequence(id).getProgramPath();
+      try {
+        Parser parser;
+        auto p2 = parser.parse(path);
+        collectPrograms(p2, collected);
+      } catch (const std::exception &) {
+        Log::get().warn("Referenced program not found: " + path);
+      }
+    }
+  }
+}
+
+size_t Finder::getTransitiveProgramHash(const Program &program) {
+  std::set<Program> collected;
+  collectPrograms(program, collected);
+  size_t h = 0;
+  for (auto &p : collected) {
+    h += ProgramUtil::hash(p);
+  }
+  return h;
+}
+
 void Finder::notifyInvalidMatch(size_t id) {
   if (invalid_matches.find(id) == invalid_matches.end()) {
     invalid_matches[id] = 1;

src/include/commands.hpp

@@ -42,6 +42,8 @@ class Commands {
 
   void testIncEval();
 
+  void testLogEval();
+
   void testPari();
 
   void dot(const std::string& path);

src/include/evaluator_log.hpp

@@ -0,0 +1,14 @@
+#pragma once
+
+#include "program.hpp"
+
+// Logarithmic Evaluator is not a real evaluator, but a static code analysis
+// utility to find out whether a program consists of a loop that is executed
+// in O(log(n)) time complexity. This is done by inspecting the operations on
+// the loop counter cell and ensuring that it gets divided by a constant >1
+// in every iteration.
+//
+class LogarithmicEvaluator {
+ public:
+  static bool hasLogarithmicComplexity(const Program& program);
+};