Import ruby/prism@69203cd

Author: andrykonchin

src/main/c/yarp/.gitignore

@@ -28,6 +28,7 @@ a.out
 /java/org/yarp/AbstractNodeVisitor.java
 /java/org/yarp/Loader.java
 /java/org/yarp/Nodes.java
+/java/org/yarp/ParseResult.java
 /lib/yarp/node.rb
 /lib/yarp/serialize.rb
 /src/node.c

src/main/c/yarp/Makefile.in

@@ -12,7 +12,7 @@ SOEXT := $(shell ruby -e 'puts RbConfig::CONFIG["SOEXT"]')
 
 DEFS := @DEFS@
 CPPFLAGS := @DEFS@ -Iinclude
-CFLAGS := @CFLAGS@ -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wsign-conversion -fPIC -fvisibility=hidden
+CFLAGS := @CFLAGS@ -std=c99 -Wall -Werror -Wextra -Wpedantic -Wundef -Wconversion -fPIC -fvisibility=hidden
 CC := @CC@
 
 HEADERS := $(shell find include -name '*.h')

src/main/c/yarp/config.h.in

@@ -23,6 +23,3 @@
 
 /* Define to the version of this package. */
 #undef PACKAGE_VERSION
-
-/* _XOPEN_SOURCE */
-#undef _XOPEN_SOURCE

src/main/c/yarp/configure

@@ -3196,9 +3196,6 @@ ac_link='$CC -o conftest$ac_exeext $CFLAGS $CPPFLAGS $LDFLAGS conftest.$ac_ext $
 ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
-printf "%s\n" "#define _XOPEN_SOURCE 700" >>confdefs.h
-
-
 
 ac_fn_c_check_func "$LINENO" "mmap" "ac_cv_func_mmap"
 if test "x$ac_cv_func_mmap" = xyes

src/main/c/yarp/include/yarp.h

@@ -61,7 +61,7 @@ YP_EXPORTED_FUNCTION void yp_serialize(yp_parser_t *parser, yp_node_t *node, yp_
 
 // Parse and serialize the AST represented by the given source to the given
 // buffer.
-YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer);
+YP_EXPORTED_FUNCTION void yp_parse_serialize(const char *source, size_t size, yp_buffer_t *buffer, const char *metadata);
 
 // Returns a string representation of the given token type.
 YP_EXPORTED_FUNCTION const char * yp_token_type_to_str(yp_token_type_t token_type);

src/main/c/yarp/include/yarp/ast.h

@@ -62,6 +62,9 @@ typedef enum yp_token_type {
     YP_TOKEN_EQUAL_GREATER, // =>
     YP_TOKEN_EQUAL_TILDE, // =~
     YP_TOKEN_FLOAT, // a floating point number
+    YP_TOKEN_FLOAT_IMAGINARY, // a floating pointer number with an imaginary suffix
+    YP_TOKEN_FLOAT_RATIONAL, // a floating pointer number with a rational suffix
+    YP_TOKEN_FLOAT_RATIONAL_IMAGINARY, // a floating pointer number with a rational and imaginary suffix
     YP_TOKEN_GLOBAL_VARIABLE, // a global variable
     YP_TOKEN_GREATER, // >
     YP_TOKEN_GREATER_EQUAL, // >=
@@ -71,9 +74,11 @@ typedef enum yp_token_type {
     YP_TOKEN_HEREDOC_START, // the start of a heredoc
     YP_TOKEN_IDENTIFIER, // an identifier
     YP_TOKEN_IGNORED_NEWLINE, // an ignored newline
-    YP_TOKEN_IMAGINARY_NUMBER, // an imaginary number literal
     YP_TOKEN_INSTANCE_VARIABLE, // an instance variable
     YP_TOKEN_INTEGER, // an integer (any base)
+    YP_TOKEN_INTEGER_IMAGINARY, // an integer with an imaginary suffix
+    YP_TOKEN_INTEGER_RATIONAL, // an integer with a rational suffix
+    YP_TOKEN_INTEGER_RATIONAL_IMAGINARY, // an integer with a rational and imaginary suffix
     YP_TOKEN_KEYWORD_ALIAS, // alias
     YP_TOKEN_KEYWORD_AND, // and
     YP_TOKEN_KEYWORD_BEGIN, // begin
@@ -151,7 +156,6 @@ typedef enum yp_token_type {
     YP_TOKEN_PLUS, // +
     YP_TOKEN_PLUS_EQUAL, // +=
     YP_TOKEN_QUESTION_MARK, // ?
-    YP_TOKEN_RATIONAL_NUMBER, // a rational number literal
     YP_TOKEN_REGEXP_BEGIN, // the beginning of a regular expression
     YP_TOKEN_REGEXP_END, // the end of a regular expression
     YP_TOKEN_SEMICOLON, // ;
@@ -166,6 +170,7 @@ typedef enum yp_token_type {
     YP_TOKEN_STRING_END, // the end of a string
     YP_TOKEN_SYMBOL_BEGIN, // the beginning of a symbol
     YP_TOKEN_TILDE, // ~ or ~@
+    YP_TOKEN_UAMPERSAND, // unary &
     YP_TOKEN_UCOLON_COLON, // unary ::
     YP_TOKEN_UDOT_DOT, // unary ..
     YP_TOKEN_UDOT_DOT_DOT, // unary ...
@@ -245,98 +250,111 @@ typedef enum {
     YP_NODE_CONSTANT_PATH_OPERATOR_WRITE_NODE = 34,
     YP_NODE_CONSTANT_PATH_WRITE_NODE = 35,
     YP_NODE_CONSTANT_READ_NODE = 36,
-    YP_NODE_DEF_NODE = 37,
-    YP_NODE_DEFINED_NODE = 38,
-    YP_NODE_ELSE_NODE = 39,
-    YP_NODE_EMBEDDED_STATEMENTS_NODE = 40,
-    YP_NODE_EMBEDDED_VARIABLE_NODE = 41,
-    YP_NODE_ENSURE_NODE = 42,
-    YP_NODE_FALSE_NODE = 43,
-    YP_NODE_FIND_PATTERN_NODE = 44,
-    YP_NODE_FLOAT_NODE = 45,
-    YP_NODE_FOR_NODE = 46,
-    YP_NODE_FORWARDING_ARGUMENTS_NODE = 47,
-    YP_NODE_FORWARDING_PARAMETER_NODE = 48,
-    YP_NODE_FORWARDING_SUPER_NODE = 49,
-    YP_NODE_GLOBAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 50,
-    YP_NODE_GLOBAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 51,
-    YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE = 52,
-    YP_NODE_GLOBAL_VARIABLE_READ_NODE = 53,
-    YP_NODE_GLOBAL_VARIABLE_WRITE_NODE = 54,
-    YP_NODE_HASH_NODE = 55,
-    YP_NODE_HASH_PATTERN_NODE = 56,
-    YP_NODE_IF_NODE = 57,
-    YP_NODE_IMAGINARY_NODE = 58,
-    YP_NODE_IN_NODE = 59,
-    YP_NODE_INSTANCE_VARIABLE_OPERATOR_AND_WRITE_NODE = 60,
-    YP_NODE_INSTANCE_VARIABLE_OPERATOR_OR_WRITE_NODE = 61,
-    YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE = 62,
-    YP_NODE_INSTANCE_VARIABLE_READ_NODE = 63,
-    YP_NODE_INSTANCE_VARIABLE_WRITE_NODE = 64,
-    YP_NODE_INTEGER_NODE = 65,
-    YP_NODE_INTERPOLATED_REGULAR_EXPRESSION_NODE = 66,
-    YP_NODE_INTERPOLATED_STRING_NODE = 67,
-    YP_NODE_INTERPOLATED_SYMBOL_NODE = 68,
-    YP_NODE_INTERPOLATED_X_STRING_NODE = 69,
-    YP_NODE_KEYWORD_HASH_NODE = 70,
-    YP_NODE_KEYWORD_PARAMETER_NODE = 71,
-    YP_NODE_KEYWORD_REST_PARAMETER_NODE = 72,
-    YP_NODE_LAMBDA_NODE = 73,
-    YP_NODE_LOCAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 74,
-    YP_NODE_LOCAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 75,
-    YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE = 76,
-    YP_NODE_LOCAL_VARIABLE_READ_NODE = 77,
-    YP_NODE_LOCAL_VARIABLE_WRITE_NODE = 78,
-    YP_NODE_MATCH_PREDICATE_NODE = 79,
-    YP_NODE_MATCH_REQUIRED_NODE = 80,
-    YP_NODE_MISSING_NODE = 81,
-    YP_NODE_MODULE_NODE = 82,
-    YP_NODE_MULTI_WRITE_NODE = 83,
-    YP_NODE_NEXT_NODE = 84,
-    YP_NODE_NIL_NODE = 85,
-    YP_NODE_NO_KEYWORDS_PARAMETER_NODE = 86,
-    YP_NODE_NUMBERED_REFERENCE_READ_NODE = 87,
-    YP_NODE_OPTIONAL_PARAMETER_NODE = 88,
-    YP_NODE_OR_NODE = 89,
-    YP_NODE_PARAMETERS_NODE = 90,
-    YP_NODE_PARENTHESES_NODE = 91,
-    YP_NODE_PINNED_EXPRESSION_NODE = 92,
-    YP_NODE_PINNED_VARIABLE_NODE = 93,
-    YP_NODE_POST_EXECUTION_NODE = 94,
-    YP_NODE_PRE_EXECUTION_NODE = 95,
-    YP_NODE_PROGRAM_NODE = 96,
-    YP_NODE_RANGE_NODE = 97,
-    YP_NODE_RATIONAL_NODE = 98,
-    YP_NODE_REDO_NODE = 99,
-    YP_NODE_REGULAR_EXPRESSION_NODE = 100,
-    YP_NODE_REQUIRED_DESTRUCTURED_PARAMETER_NODE = 101,
-    YP_NODE_REQUIRED_PARAMETER_NODE = 102,
-    YP_NODE_RESCUE_MODIFIER_NODE = 103,
-    YP_NODE_RESCUE_NODE = 104,
-    YP_NODE_REST_PARAMETER_NODE = 105,
-    YP_NODE_RETRY_NODE = 106,
-    YP_NODE_RETURN_NODE = 107,
-    YP_NODE_SELF_NODE = 108,
-    YP_NODE_SINGLETON_CLASS_NODE = 109,
-    YP_NODE_SOURCE_ENCODING_NODE = 110,
-    YP_NODE_SOURCE_FILE_NODE = 111,
-    YP_NODE_SOURCE_LINE_NODE = 112,
-    YP_NODE_SPLAT_NODE = 113,
-    YP_NODE_STATEMENTS_NODE = 114,
-    YP_NODE_STRING_CONCAT_NODE = 115,
-    YP_NODE_STRING_NODE = 116,
-    YP_NODE_SUPER_NODE = 117,
-    YP_NODE_SYMBOL_NODE = 118,
-    YP_NODE_TRUE_NODE = 119,
-    YP_NODE_UNDEF_NODE = 120,
-    YP_NODE_UNLESS_NODE = 121,
-    YP_NODE_UNTIL_NODE = 122,
-    YP_NODE_WHEN_NODE = 123,
-    YP_NODE_WHILE_NODE = 124,
-    YP_NODE_X_STRING_NODE = 125,
-    YP_NODE_YIELD_NODE = 126,
+    YP_NODE_CONSTANT_WRITE_NODE = 37,
+    YP_NODE_DEF_NODE = 38,
+    YP_NODE_DEFINED_NODE = 39,
+    YP_NODE_ELSE_NODE = 40,
+    YP_NODE_EMBEDDED_STATEMENTS_NODE = 41,
+    YP_NODE_EMBEDDED_VARIABLE_NODE = 42,
+    YP_NODE_ENSURE_NODE = 43,
+    YP_NODE_FALSE_NODE = 44,
+    YP_NODE_FIND_PATTERN_NODE = 45,
+    YP_NODE_FLOAT_NODE = 46,
+    YP_NODE_FOR_NODE = 47,
+    YP_NODE_FORWARDING_ARGUMENTS_NODE = 48,
+    YP_NODE_FORWARDING_PARAMETER_NODE = 49,
+    YP_NODE_FORWARDING_SUPER_NODE = 50,
+    YP_NODE_GLOBAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 51,
+    YP_NODE_GLOBAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 52,
+    YP_NODE_GLOBAL_VARIABLE_OPERATOR_WRITE_NODE = 53,
+    YP_NODE_GLOBAL_VARIABLE_READ_NODE = 54,
+    YP_NODE_GLOBAL_VARIABLE_WRITE_NODE = 55,
+    YP_NODE_HASH_NODE = 56,
+    YP_NODE_HASH_PATTERN_NODE = 57,
+    YP_NODE_IF_NODE = 58,
+    YP_NODE_IMAGINARY_NODE = 59,
+    YP_NODE_IN_NODE = 60,
+    YP_NODE_INSTANCE_VARIABLE_OPERATOR_AND_WRITE_NODE = 61,
+    YP_NODE_INSTANCE_VARIABLE_OPERATOR_OR_WRITE_NODE = 62,
+    YP_NODE_INSTANCE_VARIABLE_OPERATOR_WRITE_NODE = 63,
+    YP_NODE_INSTANCE_VARIABLE_READ_NODE = 64,
+    YP_NODE_INSTANCE_VARIABLE_WRITE_NODE = 65,
+    YP_NODE_INTEGER_NODE = 66,
+    YP_NODE_INTERPOLATED_REGULAR_EXPRESSION_NODE = 67,
+    YP_NODE_INTERPOLATED_STRING_NODE = 68,
+    YP_NODE_INTERPOLATED_SYMBOL_NODE = 69,
+    YP_NODE_INTERPOLATED_X_STRING_NODE = 70,
+    YP_NODE_KEYWORD_HASH_NODE = 71,
+    YP_NODE_KEYWORD_PARAMETER_NODE = 72,
+    YP_NODE_KEYWORD_REST_PARAMETER_NODE = 73,
+    YP_NODE_LAMBDA_NODE = 74,
+    YP_NODE_LOCAL_VARIABLE_OPERATOR_AND_WRITE_NODE = 75,
+    YP_NODE_LOCAL_VARIABLE_OPERATOR_OR_WRITE_NODE = 76,
+    YP_NODE_LOCAL_VARIABLE_OPERATOR_WRITE_NODE = 77,
+    YP_NODE_LOCAL_VARIABLE_READ_NODE = 78,
+    YP_NODE_LOCAL_VARIABLE_WRITE_NODE = 79,
+    YP_NODE_MATCH_PREDICATE_NODE = 80,
+    YP_NODE_MATCH_REQUIRED_NODE = 81,
+    YP_NODE_MISSING_NODE = 82,
+    YP_NODE_MODULE_NODE = 83,
+    YP_NODE_MULTI_WRITE_NODE = 84,
+    YP_NODE_NEXT_NODE = 85,
+    YP_NODE_NIL_NODE = 86,
+    YP_NODE_NO_KEYWORDS_PARAMETER_NODE = 87,
+    YP_NODE_NUMBERED_REFERENCE_READ_NODE = 88,
+    YP_NODE_OPTIONAL_PARAMETER_NODE = 89,
+    YP_NODE_OR_NODE = 90,
+    YP_NODE_PARAMETERS_NODE = 91,
+    YP_NODE_PARENTHESES_NODE = 92,
+    YP_NODE_PINNED_EXPRESSION_NODE = 93,
+    YP_NODE_PINNED_VARIABLE_NODE = 94,
+    YP_NODE_POST_EXECUTION_NODE = 95,
+    YP_NODE_PRE_EXECUTION_NODE = 96,
+    YP_NODE_PROGRAM_NODE = 97,
+    YP_NODE_RANGE_NODE = 98,
+    YP_NODE_RATIONAL_NODE = 99,
+    YP_NODE_REDO_NODE = 100,
+    YP_NODE_REGULAR_EXPRESSION_NODE = 101,
+    YP_NODE_REQUIRED_DESTRUCTURED_PARAMETER_NODE = 102,
+    YP_NODE_REQUIRED_PARAMETER_NODE = 103,
+    YP_NODE_RESCUE_MODIFIER_NODE = 104,
+    YP_NODE_RESCUE_NODE = 105,
+    YP_NODE_REST_PARAMETER_NODE = 106,
+    YP_NODE_RETRY_NODE = 107,
+    YP_NODE_RETURN_NODE = 108,
+    YP_NODE_SELF_NODE = 109,
+    YP_NODE_SINGLETON_CLASS_NODE = 110,
+    YP_NODE_SOURCE_ENCODING_NODE = 111,
+    YP_NODE_SOURCE_FILE_NODE = 112,
+    YP_NODE_SOURCE_LINE_NODE = 113,
+    YP_NODE_SPLAT_NODE = 114,
+    YP_NODE_STATEMENTS_NODE = 115,
+    YP_NODE_STRING_CONCAT_NODE = 116,
+    YP_NODE_STRING_NODE = 117,
+    YP_NODE_SUPER_NODE = 118,
+    YP_NODE_SYMBOL_NODE = 119,
+    YP_NODE_TRUE_NODE = 120,
+    YP_NODE_UNDEF_NODE = 121,
+    YP_NODE_UNLESS_NODE = 122,
+    YP_NODE_UNTIL_NODE = 123,
+    YP_NODE_WHEN_NODE = 124,
+    YP_NODE_WHILE_NODE = 125,
+    YP_NODE_X_STRING_NODE = 126,
+    YP_NODE_YIELD_NODE = 127,
 } yp_node_type_t;
 
+// We store the node type enum in every node in the tree. We don't have nearly
+// as many node types as we do bits in an enum, so we're going to use the top
+// half of the enum to store flags about the node.
+static const unsigned int YP_NODE_FLAGS_SHIFT = sizeof(yp_node_type_t) * 8 / 2;
+static const unsigned int YP_NODE_FLAG_NEWLINE = 1 << YP_NODE_FLAGS_SHIFT;
+
+// For easy access, we define some macros that manipulate only the bottom half
+// of the node type enum.
+static const unsigned int YP_NODE_TYPE_MASK = ((1 << (sizeof(yp_node_type_t) * 8 / 2)) - 1);
+#define YP_NODE_TYPE(node) ((node)->type & YP_NODE_TYPE_MASK)
+#define YP_NODE_TYPE_P(node, type) (YP_NODE_TYPE(node) == (type))
+
 // This is the overall tagged union representing a node in the syntax tree.
 typedef struct yp_node {
     // This represents the type of the node. It somewhat maps to the nodes that
@@ -646,6 +664,14 @@ typedef struct yp_constant_read_node {
     yp_node_t base;
 } yp_constant_read_node_t;
 
+// ConstantWriteNode
+typedef struct yp_constant_write_node {
+    yp_node_t base;
+    yp_location_t name_loc;
+    struct yp_node *value;
+    yp_location_t operator_loc;
+} yp_constant_write_node_t;
+
 // DefNode
 typedef struct yp_def_node {
     yp_node_t base;
@@ -1183,7 +1209,7 @@ typedef struct yp_rescue_node {
     yp_location_t keyword_loc;
     struct yp_node_list exceptions;
     yp_location_t operator_loc;
-    struct yp_node *exception;
+    struct yp_node *reference;
     struct yp_statements_node *statements;
     struct yp_rescue_node *consequent;
 } yp_rescue_node_t;
@@ -1315,6 +1341,7 @@ typedef struct yp_until_node {
     yp_location_t keyword_loc;
     struct yp_node *predicate;
     struct yp_statements_node *statements;
+    uint32_t flags;
 } yp_until_node_t;
 
 // WhenNode
@@ -1331,6 +1358,7 @@ typedef struct yp_while_node {
     yp_location_t keyword_loc;
     struct yp_node *predicate;
     struct yp_statements_node *statements;
+    uint32_t flags;
 } yp_while_node_t;
 
 // XStringNode
@@ -1354,8 +1382,14 @@ typedef struct yp_yield_node {
 // CallNodeFlags
 typedef enum {
     YP_CALL_NODE_FLAGS_SAFE_NAVIGATION = 1 << 0,
+    YP_CALL_NODE_FLAGS_VARIABLE_CALL = 1 << 1,
 } yp_call_node_flags_t;
 
+// LoopFlags
+typedef enum {
+    YP_LOOP_FLAGS_BEGIN_MODIFIER = 1 << 0,
+} yp_loop_flags_t;
+
 // RangeNodeFlags
 typedef enum {
     YP_RANGE_NODE_FLAGS_EXCLUDE_END = 1 << 0,

src/main/c/yarp/include/yarp/defines.h

@@ -12,18 +12,16 @@
 #include <string.h>
 
 // YP_EXPORTED_FUNCTION
-#if defined(_WIN32)
-#   define YP_EXPORTED_FUNCTION __declspec(dllexport) extern
-#elif defined(YP_EXPORT_SYMBOLS)
-#   ifndef YP_EXPORTED_FUNCTION
-#       ifndef RUBY_FUNC_EXPORTED
-#           define YP_EXPORTED_FUNCTION __attribute__((__visibility__("default"))) extern
+#ifndef YP_EXPORTED_FUNCTION
+#   ifdef YP_EXPORT_SYMBOLS
+#       ifdef _WIN32
+#          define YP_EXPORTED_FUNCTION __declspec(dllexport) extern
 #       else
-#           define YP_EXPORTED_FUNCTION RUBY_FUNC_EXPORTED
+#          define YP_EXPORTED_FUNCTION __attribute__((__visibility__("default"))) extern
 #       endif
+#   else
+#       define YP_EXPORTED_FUNCTION
 #   endif
-#else
-#   define YP_EXPORTED_FUNCTION
 #endif
 
 // YP_ATTRIBUTE_UNUSED