[GR-45043] Import the latest YARP (0.14.0)

PullRequest: truffleruby/4035

Author: andrykonchin

.gitignore

@@ -25,6 +25,7 @@ workingsets.xml
 # C extensions
 *.o
 *.so
+*.a
 *.bundle
 *.dylib
 *.log

spec/tags/truffle/parsing/parsing_tags.txt

@@ -197,7 +197,6 @@ fails:Parsing a Return (return operator in a method (with a value)) case is pars
 fails:Parsing a Return (return operator in a method without value) case is parsed correctly
 fails:Parsing a Return (return operator with splat operator (return *a)) case is parsed correctly
 fails:Parsing a Sequence of expressions () case is parsed correctly
-fails:Parsing a String (Frozen literal) case is parsed correctly
 fails:Parsing a String (Literal with interpolation when when expressions are % String literals) case is parsed correctly
 fails:Parsing a String (Literal with interpolation when expressions are Strings) case is parsed correctly
 fails:Parsing a Symbol (Literal with interpolation when expressions are % String literals) case is parsed correctly

src/main/c/yarp/include/prism.h

@@ -8,7 +8,6 @@
 #include "prism/pack.h"
 #include "prism/parser.h"
 #include "prism/regexp.h"
-#include "prism/unescape.h"
 #include "prism/util/pm_buffer.h"
 #include "prism/util/pm_char.h"
 #include "prism/util/pm_memchr.h"
@@ -35,7 +34,7 @@ void pm_print_node(pm_parser_t *parser, pm_node_t *node);
 void pm_parser_metadata(pm_parser_t *parser, const char *metadata);
 
 // Generate a scope node from the given node.
-void pm_scope_node_init(pm_node_t *node, pm_scope_node_t *dest);
+void pm_scope_node_init(const pm_node_t *node, pm_scope_node_t *scope, pm_scope_node_t *previous, pm_parser_t *parser);
 
 // The prism version and the serialization format.
 PRISM_EXPORTED_FUNCTION const char * pm_version(void);

src/main/c/yarp/include/prism/ast.h

@@ -358,8 +358,10 @@ typedef uint16_t pm_node_flags_t;
 
 // We store the flags enum in every node in the tree. Some flags are common to
 // all nodes (the ones listed below). Others are specific to certain node types.
-static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = 0x1;
-static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = 0x2;
+#define PM_NODE_FLAG_BITS (sizeof(pm_node_flags_t) * 8)
+static const pm_node_flags_t PM_NODE_FLAG_NEWLINE = (1 << (PM_NODE_FLAG_BITS - 1));
+static const pm_node_flags_t PM_NODE_FLAG_STATIC_LITERAL = (1 << (PM_NODE_FLAG_BITS - 2));
+static const pm_node_flags_t PM_NODE_FLAG_COMMON_MASK = (1 << (PM_NODE_FLAG_BITS - 1)) | (1 << (PM_NODE_FLAG_BITS - 2));
 
 // For easy access, we define some macros to check node type
 #define PM_NODE_TYPE(node) ((enum pm_node_type)node->type)
@@ -474,6 +476,7 @@ typedef struct pm_assoc_splat_node {
 // Type: PM_BACK_REFERENCE_READ_NODE
 typedef struct pm_back_reference_read_node {
     pm_node_t base;
+    pm_constant_id_t name;
 } pm_back_reference_read_node_t;
 
 // BeginNode
@@ -1193,11 +1196,11 @@ typedef struct pm_integer_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_interpolated_match_last_line_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1212,11 +1215,11 @@ typedef struct pm_interpolated_match_last_line_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_interpolated_regular_expression_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1369,11 +1372,11 @@ typedef struct pm_local_variable_write_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_match_last_line_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1616,11 +1619,11 @@ typedef struct pm_redo_node {
 //    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE
 //    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED
 //    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE
+//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 //    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP
 //    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT
 //    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J
 //    PM_REGULAR_EXPRESSION_FLAGS_UTF_8
-//    PM_REGULAR_EXPRESSION_FLAGS_ONCE
 typedef struct pm_regular_expression_node {
     pm_node_t base;
     pm_location_t opening_loc;
@@ -1888,43 +1891,43 @@ typedef struct pm_yield_node {
 
 // CallNodeFlags
 typedef enum pm_call_node_flags {
-    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 2,
-    PM_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 3,
+    PM_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 0,
+    PM_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 1,
 } pm_call_node_flags_t;
 
 // IntegerBaseFlags
 typedef enum pm_integer_base_flags {
-    PM_INTEGER_BASE_FLAGS_BINARY = 1 << 2,
-    PM_INTEGER_BASE_FLAGS_OCTAL = 1 << 3,
-    PM_INTEGER_BASE_FLAGS_DECIMAL = 1 << 4,
-    PM_INTEGER_BASE_FLAGS_HEXADECIMAL = 1 << 5,
+    PM_INTEGER_BASE_FLAGS_BINARY = 1 << 0,
+    PM_INTEGER_BASE_FLAGS_OCTAL = 1 << 1,
+    PM_INTEGER_BASE_FLAGS_DECIMAL = 1 << 2,
+    PM_INTEGER_BASE_FLAGS_HEXADECIMAL = 1 << 3,
 } pm_integer_base_flags_t;
 
 // LoopFlags
 typedef enum pm_loop_flags {
-    PM_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 2,
+    PM_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 0,
 } pm_loop_flags_t;
 
 // RangeFlags
 typedef enum pm_range_flags {
-    PM_RANGE_FLAGS_EXCLUDE_END = 1 << 2,
+    PM_RANGE_FLAGS_EXCLUDE_END = 1 << 0,
 } pm_range_flags_t;
 
 // RegularExpressionFlags
 typedef enum pm_regular_expression_flags {
-    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 2,
-    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 3,
-    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 4,
-    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 5,
-    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 6,
-    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 7,
-    PM_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 8,
-    PM_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 9,
+    PM_REGULAR_EXPRESSION_FLAGS_IGNORE_CASE = 1 << 0,
+    PM_REGULAR_EXPRESSION_FLAGS_EXTENDED = 1 << 1,
+    PM_REGULAR_EXPRESSION_FLAGS_MULTI_LINE = 1 << 2,
+    PM_REGULAR_EXPRESSION_FLAGS_ONCE = 1 << 3,
+    PM_REGULAR_EXPRESSION_FLAGS_EUC_JP = 1 << 4,
+    PM_REGULAR_EXPRESSION_FLAGS_ASCII_8BIT = 1 << 5,
+    PM_REGULAR_EXPRESSION_FLAGS_WINDOWS_31J = 1 << 6,
+    PM_REGULAR_EXPRESSION_FLAGS_UTF_8 = 1 << 7,
 } pm_regular_expression_flags_t;
 
 // StringFlags
 typedef enum pm_string_flags {
-    PM_STRING_FLAGS_FROZEN = 1 << 2,
+    PM_STRING_FLAGS_FROZEN = 1 << 0,
 } pm_string_flags_t;
 
 #define PRISM_SERIALIZE_ONLY_SEMANTICS_FIELDS 1

src/main/c/yarp/include/prism/node.h

@@ -33,10 +33,17 @@ PRISM_EXPORTED_FUNCTION const char * pm_node_type_to_str(pm_node_type_t node_typ
 // declare them here to avoid generating them.
 typedef struct pm_scope_node {
     pm_node_t base;
+    struct pm_scope_node *previous;
     pm_node_t *ast_node;
     struct pm_parameters_node *parameters;
     pm_node_t *body;
     pm_constant_id_list_t locals;
+    pm_parser_t *parser;
+
+    // We don't have the CRuby types ID and st_table within Prism
+    // so we use void *
+    void *constants; // ID *constants
+    void *index_lookup_table; // st_table *index_lookup_table
 } pm_scope_node_t;
 
 #endif // PRISM_NODE_H

src/main/c/yarp/include/prism/parser.h

@@ -8,6 +8,7 @@
 #include "prism/util/pm_list.h"
 #include "prism/util/pm_newline_list.h"
 #include "prism/util/pm_state_stack.h"
+#include "prism/util/pm_string.h"
 
 #include <stdbool.h>
 
@@ -172,6 +173,11 @@ typedef struct pm_lex_mode {
             // This is the pointer to the character where lexing should resume
             // once the heredoc has been completely processed.
             const uint8_t *next_start;
+
+            // This is used to track the amount of common whitespace on each
+            // line so that we know how much to dedent each line in the case of
+            // a tilde heredoc.
+            size_t common_whitespace;
         } heredoc;
     } as;
 
@@ -244,6 +250,16 @@ typedef struct pm_comment {
     pm_comment_type_t type;
 } pm_comment_t;
 
+// This is a node in the linked list of magic comments that we've found while
+// parsing.
+typedef struct {
+    pm_list_node_t node;
+    const uint8_t *key_start;
+    const uint8_t *value_start;
+    uint32_t key_length;
+    uint32_t value_length;
+} pm_magic_comment_t;
+
 // When the encoding that is being used to parse the source is changed by prism,
 // we provide the ability here to call out to a user-defined function.
 typedef void (*pm_encoding_changed_callback_t)(pm_parser_t *parser);
@@ -347,6 +363,7 @@ struct pm_parser {
     const uint8_t *heredoc_end;
 
     pm_list_t comment_list;             // the list of comments that have been found while parsing
+    pm_list_t magic_comment_list;       // the list of magic comments that have been found while parsing.
     pm_list_t warning_list;             // the list of warnings that have been found while parsing
     pm_list_t error_list;               // the list of errors that have been found while parsing
     pm_scope_t *current_scope;          // the current local scope
@@ -393,6 +410,10 @@ struct pm_parser {
     // when we find tokens that we need it for.
     pm_node_flags_t integer_base;
 
+    // This string is used to pass information from the lexer to the parser. It
+    // is particularly necessary because of escape sequences.
+    pm_string_t current_string;
+
     // Whether or not we're at the beginning of a command
     bool command_start;
 

src/main/c/yarp/include/prism/unescape.h

@@ -21,6 +21,9 @@ typedef struct {
 // Return the size of the pm_buffer_t struct.
 PRISM_EXPORTED_FUNCTION size_t pm_buffer_sizeof(void);
 
+// Initialize a pm_buffer_t with the given capacity.
+bool pm_buffer_init_capacity(pm_buffer_t *buffer, size_t capacity);
+
 // Initialize a pm_buffer_t with its default values.
 PRISM_EXPORTED_FUNCTION bool pm_buffer_init(pm_buffer_t *buffer);
 

src/main/c/yarp/include/prism/util/pm_buffer.h

@@ -1,4 +1,4 @@
 #define PRISM_VERSION_MAJOR 0
-#define PRISM_VERSION_MINOR 13
+#define PRISM_VERSION_MINOR 14
 #define PRISM_VERSION_PATCH 0
-#define PRISM_VERSION "0.13.0"
+#define PRISM_VERSION "0.14.0"

src/main/c/yarp/include/prism/version.h

@@ -265,7 +265,7 @@ pm_diagnostic_message(pm_diagnostic_id_t diag_id) {
 // Append an error to the given list of diagnostic.
 bool
 pm_diagnostic_list_append(pm_list_t *list, const uint8_t *start, const uint8_t *end, pm_diagnostic_id_t diag_id) {
-    pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) malloc(sizeof(pm_diagnostic_t));
+    pm_diagnostic_t *diagnostic = (pm_diagnostic_t *) calloc(sizeof(pm_diagnostic_t), 1);
     if (diagnostic == NULL) return false;
 
     *diagnostic = (pm_diagnostic_t) { .start = start, .end = end, .message = pm_diagnostic_message(diag_id) };