segfault with ebnf

CMakeLists.txt

@@ -3,6 +3,8 @@ project("llama.cpp" C CXX)
 
 set(CMAKE_EXPORT_COMPILE_COMMANDS ON)
 
+set(CMAKE_BUILD_TYPE Debug CACHE STRING "Build type" FORCE)
+
 if (NOT XCODE AND NOT MSVC AND NOT CMAKE_BUILD_TYPE)
     set(CMAKE_BUILD_TYPE Release CACHE STRING "Build type" FORCE)
     set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")

common/grammar-parser.cpp

@@ -27,6 +27,7 @@ namespace grammar_parser {
     static uint32_t get_symbol_id(parse_state & state, const char * src, size_t len) {
         uint32_t next_id = static_cast<uint32_t>(state.symbol_ids.size());
         auto result = state.symbol_ids.insert(std::make_pair(std::string(src, len), next_id));
+	fprintf(stderr, "added id:%d wit string:|%s|\n",next_id,std::string(src, len).c_str());
         return result.first->second;
     }
 
@@ -41,8 +42,11 @@ namespace grammar_parser {
             uint32_t      rule_id,
             const std::vector<llama_grammar_element> & rule) {
         if (state.rules.size() <= rule_id) {
+	  fprintf(stderr, "resize id %d\n",rule_id);
             state.rules.resize(rule_id + 1);
         }
+
+	fprintf(stderr, "adding rule id %d\n",rule_id);
         state.rules[rule_id] = rule;
     }
 

grammars/ebnf.ebnf

@@ -0,0 +1,29 @@
+letter ::= "A" | "B" | "C" | "D" | "E" | "F" | "G"       | "H" | "I" | "J" | "K" | "L" | "M" | "N"       | "O" | "P" | "Q" | "R" | "S" | "T" | "U"       | "V" | "W" | "X" | "Y" | "Z" | "a" | "b"       | "c" | "d" | "e" | "f" | "g" | "h" | "i"        | "j" | "k" | "l" | "m" | "n" | "o" | "p"       | "q" | "r" | "s" | "t" | "u" | "v" | "w"       | "x" | "y" | "z" 
+
+digit ::= "0" | "1" | "2" | "3" | "4" | "5" | "6" | "7" | "8" | "9" 
+# removed "  | "\f" | "\b"
+symbol ::= "[" | "]" | "{" | "}" | "(" | ")" | "<" | ">"  | "'" |  "=" | "|" | "." | "," | ";" | "-"        | "+" | "*" | "?" | "\n" | "\t" | "\r"  
+
+character ::= letter | digit | symbol | "_" | " " 
+identifier ::= letter ( letter | digit | "_" )
+
+#| "\f" | "\b"
+S ::= ( " " | "\n" | "\t" | "\r"  )
+
+terminal ::= "'" character  "'"  ( character  "'" )  "'"
+
+terminator ::= (";" | ".")
+
+term ::= "("   S   rhs   S   ")"      | "["   S   rhs   S   "]"      | "{"   S   rhs   S   "}"      | terminal      | identifier 
+
+factor ::= term  S  "?"         | term   S   "*"        | term   S   "+"        | term   S   "-"   S   term        | term   S 
+
+concatenation ::= ( S   factor   S   "," ? ) + 
+alternation ::= ( S   concatenation   S   "|" ? ) + 
+
+rhs ::= alternation 
+lhs ::= identifier 
+
+rule ::= lhs   S   "="   S   rhs   S   terminator 
+
+root ::= ( S   rule   S ) * 

grammars/mysql.gebnf

@@ -0,0 +1,48 @@
+
+
+
+
+
+# Define a table definition
+
+
+tablename ::= [a-zA-Z_][a-zA-Z0-9_]*
+
+# Define column definitions
+columndefinitions ::= columndefinition+
+
+# Define a column definition
+columndefinition ::= columnname " " datatype
+
+# Define a column name
+columnname ::= [a-zA-Z_][a-zA-Z0-9_]*
+
+# Define a data type
+datatype ::= "INT" | "VARCHAR" | "DATE" 
+
+# Define relationship definitions
+relationshipdefinitions ::= relationshipdefinition+
+
+# Define a relationship definition
+relationshipdefinition ::= "FOREIGN KEY (" columnname ") REFERENCES " tablename " (" columnname ")"
+
+# Define ID tokens for each table
+tableidtokens ::= tableidtoken+
+
+# Define an ID token for a table
+tableidtoken ::= tablename "_id"
+
+# Define predicates for each table ID
+tableidpredicates ::= tableidpredicate+
+
+# Define predicates for a table ID
+tableidpredicate ::= tableidtoken "." columnname
+
+# Define optional whitespace between components
+ws ::= [ \t\n]+
+
+tabledefinition ::= "CREATE TABLE " tablename " (" columndefinitions ")" "\n"
+
+tabledefinitions ::= tabledefinition+
+
+root ::= tabledefinitions relationshipdefinitions