[GR-45043] Update to the latest YARP

PullRequest: truffleruby/4031

Author: andrykonchin

doc/contributor/prism.md

@@ -0,0 +1,21 @@
+# Prism
+
+## Update Prism
+
+* Clone `prism` as a sibling of `truffleruby-ws`.
+* Run `tool/import-prism.sh` in the truffleruby repo.
+* Commit the result with message `Import ruby/prism@COMMIT`
+
+## Print Detailed Prism AST
+
+```bash
+cd prism
+bundle exec rake
+bin/parse -e '1&.itself'
+```
+
+We can also see what the AST as Java nodes and without extra location fields looks like on TruffleRuby with:
+```bash
+cd truffleruby
+jt -q ruby -e 'puts Truffle::Debug.yarp_parse(ARGV[0])' -- '1&.itself'
+```

doc/contributor/yarp.md

@@ -138,7 +138,7 @@
 
         # ------------- Projects -------------
 
-        "org.yarp": {
+        "org.prism": {
             "dir": "src/yarp",
             "sourceDirs": ["java"],
             "jniHeaders": True,
@@ -219,7 +219,7 @@
             "ldflags": ["-pthread"],
         },
 
-        "org.yarp.libyarp": {
+        "org.prism.libyarp": {
             "class": "YARPNativeProject",
             "dir": "src/main/c/yarp",
             # "makeTarget": "all-no-debug", # Can use this to build without asserts
@@ -232,13 +232,13 @@
             "native": "shared_lib",
             "deliverable": "yarp",
             "buildDependencies": [
-                "org.yarp.libyarp", # librubyparser.a
-                "org.yarp", # for the generated JNI header file
+                "org.prism.libyarp", # librubyparser.a
+                "org.prism", # for the generated JNI header file
             ],
             "use_jdk_headers": True, # the generated JNI header includes jni.h
-            "cflags": ["-g", "-Wall", "-Werror", "-pthread", "-I<path:org.yarp.libyarp>/include"],
+            "cflags": ["-g", "-Wall", "-Werror", "-pthread", "-I<path:org.prism.libyarp>/include"],
             "ldflags": ["-pthread"],
-            "ldlibs": ["<path:org.yarp.libyarp>/build/librubyparser.a"],
+            "ldlibs": ["<path:org.prism.libyarp>/build/librubyparser.a"],
             "description": "JNI bindings for YARP"
         },
 
@@ -254,7 +254,7 @@
             ],
             "dependencies": [
                 # Projects
-                "org.yarp",
+                "org.prism",
                 # Distributions, keep in sync with TRUFFLERUBY.distDependencies
                 "truffleruby:TRUFFLERUBY-ANNOTATIONS",
                 "truffleruby:TRUFFLERUBY-SHARED",

mx.truffleruby/suite.py

@@ -35,7 +35,7 @@ build/librubyparser.a: $(STATIC_OBJECTS)
 build/shared/%.o: src/%.c Makefile $(HEADERS)
 	$(ECHO) "compiling $@"
 	$(Q) mkdir -p $(@D)
-	$(Q) $(CC) $(DEBUG_FLAGS) -DYP_EXPORT_SYMBOLS $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
+	$(Q) $(CC) $(DEBUG_FLAGS) -DPRISM_EXPORT_SYMBOLS $(CPPFLAGS) $(CFLAGS) -c -o $@ $<
 
 build/static/%.o: src/%.c Makefile $(HEADERS)
 	$(ECHO) "compiling $@"
@@ -55,20 +55,20 @@ build/fuzz.heisenbug.%: $(SOURCES) fuzz/%.c fuzz/heisenbug.c
 
 fuzz-debug:
 	$(ECHO) "entering debug shell"
-	$(Q) docker run -it --rm -e HISTFILE=/yarp/fuzz/output/.bash_history -v $(shell pwd):/yarp -v $(FUZZ_OUTPUT_DIR):/fuzz_output yarp/fuzz
+	$(Q) docker run -it --rm -e HISTFILE=/prism/fuzz/output/.bash_history -v $(shell pwd):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz
 
 fuzz-docker-build: fuzz/docker/Dockerfile
 	$(ECHO) "building docker image"
-	$(Q) docker build -t yarp/fuzz fuzz/docker/
+	$(Q) docker build -t prism/fuzz fuzz/docker/
 
 fuzz-run-%: FORCE fuzz-docker-build
 	$(ECHO) "generating templates"
 	$(Q) bundle exec rake templates
 	$(ECHO) "running $* fuzzer"
-	$(Q) docker run --rm -v $(shell pwd):/yarp yarp/fuzz /bin/bash -c "FUZZ_FLAGS=\"$(FUZZ_FLAGS)\" make build/fuzz.$*"
+	$(Q) docker run --rm -v $(shell pwd):/prism prism/fuzz /bin/bash -c "FUZZ_FLAGS=\"$(FUZZ_FLAGS)\" make build/fuzz.$*"
 	$(ECHO) "starting AFL++ run"
 	$(Q) mkdir -p $(FUZZ_OUTPUT_DIR)/$*
-	$(Q) docker run -it --rm -v $(shell pwd):/yarp -v $(FUZZ_OUTPUT_DIR):/fuzz_output yarp/fuzz /bin/bash -c "./fuzz/$*.sh /fuzz_output/$*"
+	$(Q) docker run -it --rm -v $(shell pwd):/prism -v $(FUZZ_OUTPUT_DIR):/fuzz_output prism/fuzz /bin/bash -c "./fuzz/$*.sh /fuzz_output/$*"
 FORCE:
 
 fuzz-clean:

src/main/c/yarp/.gitignore

@@ -0,0 +1,82 @@
+#ifndef PRISM_H
+#define PRISM_H
+
+#include "prism/defines.h"
+#include "prism/ast.h"
+#include "prism/diagnostic.h"
+#include "prism/node.h"
+#include "prism/pack.h"
+#include "prism/parser.h"
+#include "prism/regexp.h"
+#include "prism/unescape.h"
+#include "prism/util/pm_buffer.h"
+#include "prism/util/pm_char.h"
+#include "prism/util/pm_memchr.h"
+#include "prism/util/pm_strpbrk.h"
+#include "prism/version.h"
+
+#include <assert.h>
+#include <errno.h>
+#include <stdarg.h>
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifndef _WIN32
+#include <strings.h>
+#endif
+
+void pm_serialize_content(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
+
+void pm_print_node(pm_parser_t *parser, pm_node_t *node);
+
+void pm_parser_metadata(pm_parser_t *parser, const char *metadata);
+
+// Generate a scope node from the given node.
+void pm_scope_node_init(pm_node_t *node, pm_scope_node_t *dest);
+
+// The prism version and the serialization format.
+PRISM_EXPORTED_FUNCTION const char * pm_version(void);
+
+// Initialize a parser with the given start and end pointers.
+PRISM_EXPORTED_FUNCTION void pm_parser_init(pm_parser_t *parser, const uint8_t *source, size_t size, const char *filepath);
+
+// Register a callback that will be called whenever prism changes the encoding it
+// is using to parse based on the magic comment.
+PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_changed_callback(pm_parser_t *parser, pm_encoding_changed_callback_t callback);
+
+// Register a callback that will be called when prism encounters a magic comment
+// with an encoding referenced that it doesn't understand. The callback should
+// return NULL if it also doesn't understand the encoding or it should return a
+// pointer to a pm_encoding_t struct that contains the functions necessary to
+// parse identifiers.
+PRISM_EXPORTED_FUNCTION void pm_parser_register_encoding_decode_callback(pm_parser_t *parser, pm_encoding_decode_callback_t callback);
+
+// Free any memory associated with the given parser.
+PRISM_EXPORTED_FUNCTION void pm_parser_free(pm_parser_t *parser);
+
+// Parse the Ruby source associated with the given parser and return the tree.
+PRISM_EXPORTED_FUNCTION pm_node_t * pm_parse(pm_parser_t *parser);
+
+// Pretty-prints the AST represented by the given node to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_prettyprint(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
+
+// Serialize the AST represented by the given node to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_serialize(pm_parser_t *parser, pm_node_t *node, pm_buffer_t *buffer);
+
+// Parse the given source to the AST and serialize the AST to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_parse_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+
+// Lex the given source and serialize to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_lex_serialize(const uint8_t *source, size_t size, const char *filepath, pm_buffer_t *buffer);
+
+// Parse and serialize both the AST and the tokens represented by the given
+// source to the given buffer.
+PRISM_EXPORTED_FUNCTION void pm_parse_lex_serialize(const uint8_t *source, size_t size, pm_buffer_t *buffer, const char *metadata);
+
+// Returns a string representation of the given token type.
+PRISM_EXPORTED_FUNCTION const char * pm_token_type_to_str(pm_token_type_t token_type);
+
+#endif