pattern-based generator (v7) (#132)

Author: ckrause

CHANGELOG.md

@@ -5,6 +5,7 @@ To install or update LODA, please follow the [installation instructions](https:/
 ### Features
 
 * Added `profile` command for measuring program evaluation time
+* Added pattern-based generator (generator: `v7`, miner profile: `pattern`)
 
 ### Enhancements
 

miners.pattern.json

@@ -0,0 +1,26 @@
+{
+  "miners": [
+    {
+      "name": "pattern",
+      "overwrite": "none",
+      "backoff": true,
+      "generators": [
+        "v7"
+      ],
+      "matchers": [
+        "direct",
+        "linear1",
+        "linear2",
+        "binary",
+        "decimal"
+      ]
+    }
+  ],
+  "generators": [
+    {
+      "name": "v7",
+      "version": 7,
+      "mutationRate": 0.3
+    }
+  ]
+}

src/Makefile

@@ -8,7 +8,7 @@ ifdef LODA_PLATFORM
 CXXFLAGS += -DLODA_PLATFORM=$(LODA_PLATFORM)
 endif
 
-OBJS = api_client.o benchmark.o big_number.o blocks.o commands.o config.o distribution.o evaluator.o evaluator_inc.o extender.o external/jute.o file.o finder.o generator.o generator_v1.o generator_v2.o generator_v3.o generator_v4.o generator_v5.o generator_v6.o interpreter.o iterator.o main.o matcher.o memory.o metrics.o miner.o minimizer.o mutator.o number.o oeis_list.o oeis_manager.o oeis_sequence.o optimizer.o parser.o program.o program_util.o reducer.o semantics.o setup.o sequence.o stats.o test.o util.o web_client.o
+OBJS = api_client.o benchmark.o big_number.o blocks.o commands.o config.o distribution.o evaluator.o evaluator_inc.o extender.o external/jute.o file.o finder.o generator.o generator_v1.o generator_v2.o generator_v3.o generator_v4.o generator_v5.o generator_v6.o generator_v7.o interpreter.o iterator.o main.o matcher.o memory.o metrics.o miner.o minimizer.o mutator.o number.o oeis_list.o oeis_manager.o oeis_sequence.o optimizer.o parser.o program.o program_util.o reducer.o semantics.o setup.o sequence.o stats.o test.o util.o web_client.o
 
 loda: external/jute.h external/jute.cpp $(OBJS)
 	$(CXX) $(LDFLAGS) -o loda $(OBJS)

src/Makefile.windows.mk

@@ -10,7 +10,7 @@ CXXFLAGS = $(CXXFLAGS) /Z7
 CXXFLAGS = $(CXXFLAGS) /DLODA_PLATFORM=$(LODA_PLATFORM)
 !ENDIF
 
-SRCS = api_client.cpp benchmark.cpp big_number.cpp blocks.cpp commands.cpp config.cpp distribution.cpp evaluator.cpp evaluator_inc.cpp extender.cpp external/jute.cpp file.cpp finder.cpp generator.cpp generator_v1.cpp generator_v2.cpp generator_v3.cpp generator_v4.cpp generator_v5.cpp generator_v6.cpp interpreter.cpp iterator.cpp main.cpp matcher.cpp memory.cpp metrics.cpp miner.cpp minimizer.cpp mutator.cpp number.cpp oeis_list.cpp oeis_manager.cpp oeis_sequence.cpp optimizer.cpp parser.cpp program.cpp program_util.cpp reducer.cpp semantics.cpp sequence.cpp setup.cpp stats.cpp test.cpp util.cpp web_client.cpp
+SRCS = api_client.cpp benchmark.cpp big_number.cpp blocks.cpp commands.cpp config.cpp distribution.cpp evaluator.cpp evaluator_inc.cpp extender.cpp external/jute.cpp file.cpp finder.cpp generator.cpp generator_v1.cpp generator_v2.cpp generator_v3.cpp generator_v4.cpp generator_v5.cpp generator_v6.cpp generator_v7.cpp interpreter.cpp iterator.cpp main.cpp matcher.cpp memory.cpp metrics.cpp miner.cpp minimizer.cpp mutator.cpp number.cpp oeis_list.cpp oeis_manager.cpp oeis_sequence.cpp optimizer.cpp parser.cpp program.cpp program_util.cpp reducer.cpp semantics.cpp sequence.cpp setup.cpp stats.cpp test.cpp util.cpp web_client.cpp
 
 loda: external/jute.h external/jute.cpp $(SRCS)
 	cl /EHsc /Feloda.exe $(CXXFLAGS) $(SRCS)

src/generator.cpp

@@ -7,6 +7,7 @@
 #include "generator_v4.hpp"
 #include "generator_v5.hpp"
 #include "generator_v6.hpp"
+#include "generator_v7.hpp"
 #include "semantics.hpp"
 #include "util.hpp"
 
@@ -38,6 +39,10 @@ Generator::UPtr Generator::Factory::createGenerator(const Config &config,
       generator.reset(new GeneratorV6(config, stats));
       break;
     }
+    case 7: {
+      generator.reset(new GeneratorV7(config, stats));
+      break;
+    }
     default: {
       Log::get().error(
           "Unknown generator version: " + std::to_string(config.version), true);

src/generator_v7.cpp

@@ -0,0 +1,70 @@
+#include "generator_v7.hpp"
+
+#include "file.hpp"
+#include "oeis_sequence.hpp"
+#include "parser.hpp"
+#include "program_util.hpp"
+#include "setup.hpp"
+
+GeneratorV7::GeneratorV7(const Config &config, const Stats &stats)
+    : Generator(config, stats),
+      mutator(stats, config.mutation_rate, true  // mutate comments!
+      ) {
+  // load patterns
+  Parser parser;
+  Program program;
+  const std::string patterns_home = Setup::getProgramsHome() + "patterns";
+  if (isDir(patterns_home)) {
+    for (const auto &it : std::filesystem::directory_iterator(patterns_home)) {
+      if (it.path().filename().extension().string() != ".asm") {
+        continue;
+      }
+      const auto path = it.path().string();
+      Operation dummy(Operation::Type::NOP);
+      dummy.comment = "dummy";
+      try {
+        program = parser.parse(path);
+        ProgramUtil::removeOps(program, Operation::Type::NOP);
+        bool has_comment = false;
+        for (auto &op : program.ops) {
+          has_comment = has_comment || !op.comment.empty();
+        }
+        if (has_comment) {
+          // add dummy comments at begin and end of program
+          program.ops.insert(program.ops.begin(), dummy);
+          program.ops.push_back(dummy);
+          patterns.push_back(program);
+        } else {
+          Log::get().warn("Missing annotations in pattern " +
+                          it.path().filename().string());
+        }
+      } catch (std::exception &) {
+        Log::get().warn("Cannot load pattern " + path);
+      }
+    }
+  }
+  if (patterns.empty()) {
+    Log::get().error("No patterns found", true);
+  } else {
+    Log::get().info("Loaded " + std::to_string(patterns.size()) + " patterns");
+  }
+}
+
+Program GeneratorV7::generateProgram() {
+  auto program = patterns[Random::get().gen() % patterns.size()];
+
+  // std::cout << "==== BEGIN PATTERN ======" << std::endl;
+  // ProgramUtil::print(program, std::cout);
+  // std::cout << "==== END PATTERN ======" << std::endl << std::endl;
+
+  mutator.mutateRandom(program);
+  ProgramUtil::removeOps(program, Operation::Type::NOP);
+  ProgramUtil::removeComments(program);
+
+  return program;
+}
+
+std::pair<Operation, double> GeneratorV7::generateOperation() {
+  throw std::runtime_error("unsupported operation");
+  return {};
+}

src/include/generator_v7.hpp

@@ -0,0 +1,21 @@
+#pragma once
+
+#include "generator.hpp"
+#include "mutator.hpp"
+
+/*
+ * Generator that uses pattern files. A pattern is a program with annotation
+ * that indicate positions where mutations should be applied.
+ */
+class GeneratorV7 : public Generator {
+ public:
+  GeneratorV7(const Config &config, const Stats &stats);
+
+  virtual Program generateProgram() override;
+
+  virtual std::pair<Operation, double> generateOperation() override;
+
+ private:
+  std::vector<Program> patterns;
+  Mutator mutator;
+};

src/include/mutator.hpp

@@ -9,7 +9,9 @@
 
 class Mutator {
  public:
-  Mutator(const Stats &stats, double mutation_rate = 0.3);  // magic number
+  Mutator(const Stats &stats,
+          double mutation_rate = 0.3,  // magic number
+          bool mutate_comment = false);
 
   void mutateRandom(Program &program);
 
@@ -21,14 +23,16 @@ class Mutator {
   void mutateConstants(const Program &program, size_t num_results,
                        std::stack<Program> &result);
 
-  double mutation_rate;
-
   RandomProgramIds2 random_program_ids;
 
  private:
+  const double mutation_rate;
+  const bool mutate_comment;
   std::vector<Number> constants;
   std::vector<Operation::Type> operation_types;
 
   std::discrete_distribution<> constants_dist;
   std::discrete_distribution<> operation_types_dist;
+
+  std::vector<size_t> tmp_comment_positions;
 };

src/mutator.cpp

@@ -5,8 +5,10 @@
 #define CONSTANTS_START -100
 #define CONSTANTS_END 1000
 
-Mutator::Mutator(const Stats &stats, double mutation_rate)
-    : mutation_rate(mutation_rate), random_program_ids(stats) {
+Mutator::Mutator(const Stats &stats, double mutation_rate, bool mutate_comment)
+    : random_program_ids(stats),
+      mutation_rate(mutation_rate),
+      mutate_comment(mutate_comment) {
   // initialize constants distribution from stats
   constants.resize(CONSTANTS_END - CONSTANTS_START + 1);
   for (int64_t i = 0; i <= CONSTANTS_END - CONSTANTS_START; i++) {
@@ -36,31 +38,58 @@ int64_t getRandomPos(const Program &program) {
 }
 
 void Mutator::mutateRandom(Program &program) {
+  int64_t num_cells, num_mutations, pos;
+  size_t i;
+
   // get number of used memory cells
-  const int64_t num_cells =
-      ProgramUtil::getLargestDirectMemoryCell(program) + 1;
+  num_cells = ProgramUtil::getLargestDirectMemoryCell(program) + 2;
 
   // calculate the number of mutations to apply
-  int64_t num_mutations =
-      static_cast<int64_t>(program.ops.size() * mutation_rate) + 1;
+  num_mutations = static_cast<int64_t>(program.ops.size() * mutation_rate) + 1;
   num_mutations = Random::get().gen() % num_mutations;  // could be zero
   if (mutation_rate > 0.0) {
     num_mutations++;  // at least one mutation
   }
 
+  // mutate only operations with comments?
+  if (mutate_comment) {
+    tmp_comment_positions.clear();
+    for (i = 0; i < program.ops.size(); i++) {
+      if (!program.ops[i].comment.empty()) {
+        tmp_comment_positions.push_back(i);
+      }
+    }
+    if (tmp_comment_positions.empty()) {
+      throw std::runtime_error("missing program comments for mutation");
+    }
+  }
+
   // mutate existing operations or add new ones
-  int64_t pos;
-  static const Operation mov_zero(Operation::Type::MOV,
+  static const Operation add_zero(Operation::Type::ADD,
                                   Operand(Operand::Type::DIRECT, 0),
                                   Operand(Operand::Type::CONSTANT, 0));
   for (; num_mutations > 0; num_mutations--) {
+    // choose whether to add a new operation or mutate an existing one
     if (Random::get().gen() % 2 == 0 || program.ops.empty()) {
       // add new operation
-      pos = Random::get().gen() % program.ops.size();
-      program.ops.insert(program.ops.begin() + pos, mov_zero);
+      if (mutate_comment) {
+        i = Random::get().gen() % tmp_comment_positions.size();
+        pos = tmp_comment_positions[i];
+        for (; i < tmp_comment_positions.size(); i++) {
+          tmp_comment_positions[i]++;
+        }
+      } else {
+        pos = Random::get().gen() % program.ops.size();
+      }
+      program.ops.insert(program.ops.begin() + pos, add_zero);
     } else {
       // mutate existing operation
-      pos = getRandomPos(program);
+      if (mutate_comment) {
+        pos = tmp_comment_positions[Random::get().gen() %
+                                    tmp_comment_positions.size()];
+      } else {
+        pos = getRandomPos(program);
+      }
     }
     mutateOperation(program.ops[pos], num_cells);
   }
@@ -77,7 +106,15 @@ void Mutator::mutateOperation(Operation &op, int64_t num_cells) {
       op.source =
           Operand(Operand::Type::DIRECT, Random::get().gen() % num_cells);
     }
-    op.target = Operand(Operand::Type::DIRECT, Random::get().gen() % num_cells);
+    if (op.type == Operation::Type::MOV &&
+        Random::get().gen() % 4 > 0) {  // magic number
+      // avoid overwriting
+      op.target = Operand(Operand::Type::DIRECT,
+                          (Random::get().gen() % 2) + num_cells - 1);
+    } else {
+      op.target =
+          Operand(Operand::Type::DIRECT, Random::get().gen() % num_cells);
+    }
     ProgramUtil::avoidNopOrOverflow(op);
   } else if (op.type == Operation::Type::SEQ) {
     // op.comment = "mutated from " + ProgramUtil::operationToString(op);