Skip to content

Commit 88b2880

Browse files
eregoneregon
committed
[GR-45043] Prototype to use YARP - Yet Another Ruby Parser
1 parent 83df165 commit 88b2880

File tree

7 files changed

+255
-5
lines changed

7 files changed

+255
-5
lines changed

‎doc/contributor/yarp.md

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
# Update YARP
2+
3+
* Clone `yarp` as a sibling of `truffleruby-ws`.
4+
* Run `tool/import-yarp.sh` in the truffleruby repo.

‎mx.truffleruby/mx_truffleruby.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -243,9 +243,11 @@ def verify_ci(args):
243243
'TruffleRuby license files': ('', []),
244244
},
245245
truffle_jars=[
246+
# Distributions
246247
'truffleruby:TRUFFLERUBY',
247248
'truffleruby:TRUFFLERUBY-SHARED',
248249
'truffleruby:TRUFFLERUBY-ANNOTATIONS',
250+
# Libraries
249251
'sdk:JLINE3',
250252
'truffleruby:JCODINGS',
251253
'truffleruby:JONI',

‎mx.truffleruby/suite.py

Lines changed: 30 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -124,14 +124,23 @@
124124

125125
# ------------- Projects -------------
126126

127+
"org.yarp": {
128+
"dir": "src/yarp",
129+
"sourceDirs": ["java"],
130+
"jniHeaders": True,
131+
"jacoco": "include",
132+
"javaCompliance": "17+",
133+
"workingSets": "TruffleRuby",
134+
"license": ["MIT"],
135+
},
136+
127137
"org.truffleruby.annotations": {
128138
"dir": "src/annotations",
129139
"sourceDirs": ["java"],
130140
"jacoco": "include",
131141
"javaCompliance": "17+",
132142
"checkstyle": "org.truffleruby",
133143
"workingSets": "TruffleRuby",
134-
"checkPackagePrefix": "false",
135144
"license": ["EPL-2.0"],
136145
},
137146

@@ -194,6 +203,19 @@
194203
"ldflags": ["-pthread"],
195204
},
196205

206+
"org.yarp.libyarp": {
207+
"dir": "src/main/c/yarp",
208+
"native": "shared_lib",
209+
"deliverable": "yarp",
210+
"buildDependencies": [
211+
"org.yarp", # for the generated JNI header file
212+
],
213+
"use_jdk_headers": True, # the generated JNI header includes jni.h
214+
"cflags": ["-g", "-Wall", "-Werror", "-pthread"],
215+
"ldflags": ["-pthread"],
216+
"description": "YARP + JNI bindings as a single lib"
217+
},
218+
197219
"org.truffleruby": {
198220
"dir": "src/main",
199221
"sourceDirs": ["java"],
@@ -204,15 +226,17 @@
204226
"jdk.management",
205227
"jdk.unsupported", # sun.misc.Signal
206228
],
207-
"dependencies": [ # Keep in sync with TRUFFLERUBY distDependencies and exclude
208-
# Distributions
229+
"dependencies": [
230+
# Projects
231+
"org.yarp",
232+
# Distributions, keep in sync with TRUFFLERUBY.distDependencies
209233
"truffleruby:TRUFFLERUBY-ANNOTATIONS",
210234
"truffleruby:TRUFFLERUBY-SHARED",
211235
"truffle:TRUFFLE_API",
212236
"truffle:TRUFFLE_NFI",
213237
"regex:TREGEX",
214238
"sulong:SULONG_API",
215-
# Libraries
239+
# Libraries, keep in sync with TRUFFLERUBY.exclude and truffle_jars (in mx_truffleruby.py)
216240
"sdk:JLINE3",
217241
"truffleruby:JCODINGS",
218242
"truffleruby:JONI",
@@ -230,7 +254,7 @@
230254
"EPL-2.0", # JRuby (we're choosing EPL out of EPL,GPL,LGPL)
231255
"BSD-new", # Rubinius
232256
"BSD-simplified", # MRI
233-
"MIT", # Joni, JCodings
257+
"MIT", # Joni, JCodings, YARP
234258
],
235259
},
236260

@@ -456,6 +480,7 @@
456480
"file:lib/mri",
457481
"file:lib/patches",
458482
"file:lib/truffle",
483+
"dependency:org.yarp.libyarp",
459484
],
460485
"lib/cext/": [
461486
"file:lib/cext/*.rb",

‎src/main/c/yarp/src/yarp_bindings.c

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
#include "yarp.h"
2+
#include "org_yarp_Parser.h"
3+
4+
JNIEXPORT jbyteArray JNICALL Java_org_yarp_Parser_parseAndSerialize(JNIEnv *env, jclass clazz, jbyteArray source) {
5+
jsize size = (*env)->GetArrayLength(env, source);
6+
jbyte* bytes = (*env)->GetByteArrayElements(env, source, NULL);
7+
8+
yp_buffer_t buffer;
9+
yp_buffer_init(&buffer);
10+
11+
yp_parse_serialize((char*) bytes, size, &buffer);
12+
13+
(*env)->ReleaseByteArrayElements(env, source, bytes, JNI_ABORT);
14+
15+
jbyteArray serialized = (*env)->NewByteArray(env, buffer.length);
16+
(*env)->SetByteArrayRegion(env, serialized, 0, buffer.length, (jbyte*) buffer.value);
17+
18+
yp_buffer_free(&buffer);
19+
20+
return serialized;
21+
}

‎src/main/java/org/truffleruby/debug/TruffleDebugNodes.java

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,9 +68,11 @@
6868
import org.truffleruby.interop.ToJavaStringNode;
6969
import org.truffleruby.language.ImmutableRubyObject;
7070
import org.truffleruby.core.string.ImmutableRubyString;
71+
import org.truffleruby.language.Nil;
7172
import org.truffleruby.language.RubyDynamicObject;
7273
import org.truffleruby.language.RubyGuards;
7374
import org.truffleruby.language.RubyRootNode;
75+
import org.truffleruby.language.arguments.EmptyArgumentsDescriptor;
7476
import org.truffleruby.language.arguments.RubyArguments;
7577
import org.truffleruby.language.backtrace.BacktraceFormatter;
7678
import org.truffleruby.language.backtrace.InternalRootNode;
@@ -109,6 +111,10 @@
109111
import org.truffleruby.parser.TranslatorDriver;
110112
import org.truffleruby.parser.parser.ParserConfiguration;
111113
import org.truffleruby.parser.scope.StaticScope;
114+
import org.truffleruby.platform.Platform;
115+
import org.truffleruby.yarp.YARPTranslator;
116+
import org.yarp.Loader;
117+
import org.yarp.Parser;
112118

113119
@CoreModule("Truffle::Debug")
114120
public abstract class TruffleDebugNodes {
@@ -245,6 +251,84 @@ protected Object printBacktrace() {
245251

246252
}
247253

254+
private static byte[] yarpSerialize(RubyLanguage language, byte[] source) {
255+
// TODO: load it once during context initialization (when YARP is used as the main parser)
256+
Parser.loadLibrary(language.getRubyHome() + "/lib/libyarp" + Platform.LIB_SUFFIX);
257+
return Parser.parseAndSerialize(source);
258+
}
259+
260+
@CoreMethod(names = "yarp_serialize", onSingleton = true, required = 1)
261+
public abstract static class YARPSerializeNode extends CoreMethodArrayArgumentsNode {
262+
@TruffleBoundary
263+
@Specialization(guards = "strings.isRubyString(code)", limit = "1")
264+
protected Object serialize(Object code,
265+
@Cached RubyStringLibrary strings,
266+
@Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode,
267+
@Cached TruffleString.FromByteArrayNode fromByteArrayNode) {
268+
var tstring = strings.getTString(code);
269+
var tencoding = strings.getTEncoding(code);
270+
var source = copyToByteArrayNode.execute(tstring, tencoding);
271+
272+
byte[] serialized = yarpSerialize(getLanguage(), source);
273+
274+
return createString(fromByteArrayNode, serialized, Encodings.BINARY);
275+
}
276+
}
277+
278+
@CoreMethod(names = "yarp_parse", onSingleton = true, required = 1)
279+
public abstract static class YARPParseNode extends CoreMethodArrayArgumentsNode {
280+
@TruffleBoundary
281+
@Specialization(guards = "strings.isRubyString(code)", limit = "1")
282+
protected Object parse(Object code,
283+
@Cached RubyStringLibrary strings,
284+
@Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode,
285+
@Cached TruffleString.FromJavaStringNode fromJavaStringNode) {
286+
var tstring = strings.getTString(code);
287+
var tencoding = strings.getTEncoding(code);
288+
var source = copyToByteArrayNode.execute(tstring, tencoding);
289+
290+
byte[] serialized = yarpSerialize(getLanguage(), source);
291+
292+
var ast = Loader.load(source, serialized);
293+
294+
return createString(fromJavaStringNode, ast.toString(), Encodings.UTF_8);
295+
}
296+
}
297+
298+
@CoreMethod(names = "yarp_execute", onSingleton = true, required = 1)
299+
public abstract static class YARPExecuteNode extends CoreMethodArrayArgumentsNode {
300+
@TruffleBoundary
301+
@Specialization(guards = "strings.isRubyString(code)", limit = "1")
302+
protected Object yarpExecute(Object code,
303+
@Cached RubyStringLibrary strings,
304+
@Cached TruffleString.CopyToByteArrayNode copyToByteArrayNode) {
305+
var tstring = strings.getTString(code);
306+
var tencoding = strings.getTEncoding(code);
307+
var source = copyToByteArrayNode.execute(tstring, tencoding);
308+
309+
byte[] serialized = yarpSerialize(getLanguage(), source);
310+
311+
var ast = Loader.load(source, serialized);
312+
System.err.println("YARP AST:");
313+
System.err.println(ast);
314+
var truffleAST = new YARPTranslator(getLanguage(), source).translate(ast);
315+
316+
System.err.println("Truffle AST:");
317+
NodeUtil.printCompactTree(getContext().getEnvErrStream(), truffleAST);
318+
319+
return truffleAST.getCallTarget().call(RubyArguments.pack(
320+
null,
321+
null,
322+
null,
323+
DeclarationContext.topLevel(getContext()),
324+
null,
325+
coreLibrary().mainObject,
326+
Nil.INSTANCE,
327+
EmptyArgumentsDescriptor.INSTANCE,
328+
EMPTY_ARGUMENTS));
329+
}
330+
}
331+
248332
@CoreMethod(names = "parse_ast", onSingleton = true, required = 1)
249333
public abstract static class ParseASTNode extends CoreMethodArrayArgumentsNode {
250334
@TruffleBoundary

‎src/main/java/org/truffleruby/yarp/YARPTranslator.java

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
/*
2+
* Copyright (c) 2022 Oracle and/or its affiliates. All rights reserved. This
3+
* code is released under a tri EPL/GPL/LGPL license. You can use it,
4+
* redistribute it and/or modify it under the terms of the:
5+
*
6+
* Eclipse Public License version 2.0, or
7+
* GNU General Public License version 2, or
8+
* GNU Lesser General Public License version 2.1.
9+
*/
10+
package org.truffleruby.yarp;
11+
12+
import org.truffleruby.RubyLanguage;
13+
import org.truffleruby.annotations.Split;
14+
import org.truffleruby.core.CoreLibrary;
15+
import org.truffleruby.language.RubyNode;
16+
import org.truffleruby.language.RubyRootNode;
17+
import org.truffleruby.language.RubyTopLevelRootNode;
18+
import org.truffleruby.language.SourceIndexLength;
19+
import org.truffleruby.language.arguments.EmptyArgumentsDescriptor;
20+
import org.truffleruby.language.dispatch.RubyCallNode;
21+
import org.truffleruby.language.dispatch.RubyCallNodeParameters;
22+
import org.truffleruby.language.literal.IntegerFixnumLiteralNode;
23+
import org.truffleruby.language.methods.Arity;
24+
import org.truffleruby.language.methods.SharedMethodInfo;
25+
import org.truffleruby.language.objects.SelfNode;
26+
import org.truffleruby.parser.Translator;
27+
import org.truffleruby.parser.TranslatorEnvironment;
28+
import org.yarp.AbstractNodeVisitor;
29+
import org.yarp.Nodes;
30+
31+
import java.nio.charset.StandardCharsets;
32+
import java.util.Arrays;
33+
34+
public final class YARPTranslator extends AbstractNodeVisitor<RubyNode> {
35+
36+
private final RubyLanguage language;
37+
final byte[] source;
38+
39+
public YARPTranslator(RubyLanguage language, byte[] source) {
40+
this.language = language;
41+
this.source = source;
42+
}
43+
44+
public RubyRootNode translate(Nodes.Node node) {
45+
var body = node.accept(this);
46+
var frameDescriptor = TranslatorEnvironment.newFrameDescriptorBuilder(null, true).build();
47+
var sourceSection = CoreLibrary.JAVA_CORE_SOURCE_SECTION;
48+
var sharedMethodInfo = new SharedMethodInfo(sourceSection, null, Arity.NO_ARGUMENTS, "<main>", 0, "<main>",
49+
null, null);
50+
return new RubyTopLevelRootNode(language, sourceSection, frameDescriptor, sharedMethodInfo, body,
51+
Split.HEURISTIC, null, Arity.NO_ARGUMENTS);
52+
}
53+
54+
@Override
55+
public RubyNode visitProgramNode(Nodes.ProgramNode node) {
56+
return node.statements.accept(this);
57+
}
58+
59+
@Override
60+
public RubyNode visitStatementsNode(Nodes.StatementsNode node) {
61+
var location = new SourceIndexLength(node.startOffset, node.endOffset - node.startOffset);
62+
63+
var body = node.body;
64+
var translated = new RubyNode[body.length];
65+
for (int i = 0; i < body.length; i++) {
66+
translated[i] = body[i].accept(this);
67+
}
68+
return Translator.sequence(location, Arrays.asList(translated));
69+
}
70+
71+
@Override
72+
public RubyNode visitCallNode(Nodes.CallNode node) {
73+
var methodName = new String(node.name, StandardCharsets.UTF_8);
74+
var receiver = node.receiver == null ? new SelfNode() : node.receiver.accept(this);
75+
var argumentsNode = (Nodes.ArgumentsNode) node.arguments;
76+
var arguments = argumentsNode.arguments;
77+
var translatedArguments = new RubyNode[arguments.length];
78+
for (int i = 0; i < arguments.length; i++) {
79+
translatedArguments[i] = arguments[i].accept(this);
80+
}
81+
82+
boolean ignoreVisibility = node.receiver == null;
83+
return new RubyCallNode(new RubyCallNodeParameters(receiver, methodName, null,
84+
EmptyArgumentsDescriptor.INSTANCE, translatedArguments, false, ignoreVisibility));
85+
}
86+
87+
@Override
88+
public RubyNode visitIntegerNode(Nodes.IntegerNode node) {
89+
String string = new String(source, node.startOffset, node.endOffset - node.startOffset,
90+
StandardCharsets.US_ASCII);
91+
int value = Integer.parseInt(string);
92+
return new IntegerFixnumLiteralNode(value);
93+
}
94+
95+
@Override
96+
protected RubyNode defaultVisit(Nodes.Node node) {
97+
throw new Error("Unknown node: " + node);
98+
}
99+
}

‎tool/import-yarp.sh

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
#!/usr/bin/env bash
2+
3+
set -x
4+
set -e
5+
6+
# Create generated files
7+
pushd ../../yarp
8+
bundle exec rake
9+
popd
10+
11+
rm -rf src/main/c/yarp/src/yarp
12+
cp -R ../../yarp/src src/main/c/yarp/src/yarp
13+
14+
rm -rf src/yarp/java
15+
cp -R ../../yarp/java src/yarp/java

0 commit comments

Comments
 (0)