Skip to content

Commit 69d56d7

Browse files
eregoneregon
committed
Use YARPTranslatorDriver for ParserCache
* Handle context being null in parseToYARPAST().
1 parent 0b6adfb commit 69d56d7

File tree

6 files changed

+69
-95
lines changed

6 files changed

+69
-95
lines changed

‎src/main/java/org/truffleruby/aot/ParserCache.java

Lines changed: 21 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -10,34 +10,35 @@
1010
package org.truffleruby.aot;
1111

1212
import java.io.IOException;
13+
import java.util.Collections;
1314
import java.util.HashMap;
1415
import java.util.Map;
1516

1617
import com.oracle.truffle.api.CompilerDirectives;
1718
import com.oracle.truffle.api.source.Source;
19+
import org.graalvm.collections.Pair;
20+
import org.prism.ParseResult;
21+
import org.truffleruby.RubyLanguage;
1822
import org.truffleruby.core.CoreLibrary;
19-
import org.truffleruby.core.array.ArrayUtils;
20-
import org.truffleruby.core.string.StringUtils;
2123
import org.truffleruby.language.loader.ResourceLoader;
24+
import org.truffleruby.parser.ParseEnvironment;
25+
import org.truffleruby.parser.ParserContext;
2226
import org.truffleruby.parser.RubyDeferredWarnings;
23-
import org.truffleruby.parser.RubyDeferredWarnings.WarningMessage;
2427
import org.truffleruby.parser.RubySource;
25-
import org.truffleruby.parser.TranslatorDriver;
26-
import org.truffleruby.parser.ast.RootParseNode;
28+
import org.truffleruby.parser.YARPTranslatorDriver;
2729
import org.truffleruby.parser.parser.ParserConfiguration;
28-
import org.truffleruby.parser.scope.StaticScope;
2930
import org.truffleruby.shared.options.OptionsCatalog;
3031

3132
import com.oracle.truffle.api.TruffleOptions;
3233

3334
public final class ParserCache {
3435

35-
public static final Map<String, RootParseNode> INSTANCE;
36+
public static final Map<String, Pair<ParseResult, Source>> INSTANCE;
3637

3738
static {
3839
if (TruffleOptions.AOT) {
3940
final String defaultCoreLibraryPath = OptionsCatalog.CORE_LOAD_PATH_KEY.getDefaultValue();
40-
final Map<String, RootParseNode> cache = new HashMap<>();
41+
final Map<String, Pair<ParseResult, Source>> cache = new HashMap<>();
4142

4243
for (String coreFile : CoreLibrary.CORE_FILES) {
4344
//intern() to improve footprint
@@ -62,17 +63,18 @@ private static RubySource loadSource(String feature) {
6263
}
6364
}
6465

65-
private static RootParseNode parse(RubySource source) {
66-
final StaticScope staticScope = new StaticScope(StaticScope.Type.LOCAL, null);
67-
final ParserConfiguration parserConfiguration = new ParserConfiguration(null, false, true, false);
68-
RubyDeferredWarnings rubyWarnings = new RubyDeferredWarnings();
69-
var rootParseNode = TranslatorDriver.parseToJRubyAST(null, source, staticScope, parserConfiguration,
70-
rubyWarnings);
71-
if (!rubyWarnings.warnings.isEmpty()) {
72-
throw new RuntimeException("Core files should not emit warnings: " +
73-
StringUtils.join(ArrayUtils.map(rubyWarnings.warnings, WarningMessage::getWarningMessage), "\n"));
74-
}
75-
return rootParseNode;
66+
private static Pair<ParseResult, Source> parse(RubySource source) {
67+
var language = RubyLanguage.getCurrentLanguage();
68+
var parserConfiguration = new ParserConfiguration(null, false, true, false);
69+
var rubyWarnings = new RubyDeferredWarnings();
70+
var parseEnvironment = new ParseEnvironment(language, source,
71+
YARPTranslatorDriver.createYARPSource(source.getBytes()), ParserContext.TOP_LEVEL, null);
72+
73+
var parseResult = YARPTranslatorDriver.parseToYARPAST(null, language, source, Collections.emptyList(),
74+
parserConfiguration,
75+
rubyWarnings, parseEnvironment);
76+
77+
return Pair.create(parseResult, source.getSource());
7678
}
7779

7880
}

‎src/main/java/org/truffleruby/core/CoreLibrary.java

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,6 @@
6464
import org.truffleruby.parser.ParserContext;
6565
import org.truffleruby.parser.RubySource;
6666
import org.truffleruby.parser.TranslatorDriver;
67-
import org.truffleruby.parser.ast.RootParseNode;
6867
import org.truffleruby.platform.NativeConfiguration;
6968
import org.truffleruby.platform.NativeTypes;
7069
import org.truffleruby.shared.BuildInformationImpl;
@@ -787,8 +786,8 @@ public void loadRubyCoreLibraryAndPostBoot() {
787786
public RubySource loadCoreFileSource(String path) throws IOException {
788787
if (path.startsWith(RubyLanguage.RESOURCE_SCHEME)) {
789788
if (TruffleOptions.AOT || ParserCache.INSTANCE != null) {
790-
final RootParseNode rootParseNode = ParserCache.INSTANCE.get(path);
791-
return new RubySource(rootParseNode.getSource(), path);
789+
Source source = ParserCache.INSTANCE.get(path).getRight();
790+
return new RubySource(source, path);
792791
} else {
793792
return new RubySource(ResourceLoader.loadResource(path, language.options.CORE_AS_INTERNAL), path);
794793
}

‎src/main/java/org/truffleruby/core/encoding/Encodings.java

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,4 +114,16 @@ public static RubyEncoding getBuiltInEncoding(Encoding jcoding) {
114114
return rubyEncoding;
115115
}
116116

117+
/** Should only be used when there is no other way, because this will ignore replicated and dummy encodings */
118+
public static RubyEncoding getBuiltInEncoding(String encodingName) {
119+
byte[] encodingNameBytes = encodingName.getBytes(StandardCharsets.ISO_8859_1);
120+
var entry = EncodingDB.getEncodings().get(encodingNameBytes);
121+
if (entry != null) {
122+
var jcoding = entry.getEncoding();
123+
return getBuiltInEncoding(jcoding);
124+
} else {
125+
throw CompilerDirectives.shouldNotReachHere("Unknown encoding: " + encodingName);
126+
}
127+
}
128+
117129
}

‎src/main/java/org/truffleruby/parser/TranslatorDriver.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -179,7 +179,7 @@ public RootCallTarget parse(RubySource rubySource, ParserContext parserContext,
179179
// TOPLEVEL_BINDING.eval("self") would use the cache which is wrong.
180180
if (ParserCache.INSTANCE != null && parserContext == ParserContext.TOP_LEVEL &&
181181
ParserCache.INSTANCE.containsKey(source.getName())) {
182-
node = ParserCache.INSTANCE.get(source.getName());
182+
node = null;
183183
} else {
184184
printParseTranslateExecuteMetric("before-parsing", context, source);
185185
node = context.getMetricsProfiler().callWithMetrics(

‎src/main/java/org/truffleruby/parser/YARPLoader.java

Lines changed: 5 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -39,32 +39,29 @@
3939
import org.prism.Loader;
4040
import org.prism.Nodes;
4141
import org.prism.ParseResult;
42-
import org.truffleruby.core.encoding.EncodingManager;
42+
import org.truffleruby.core.encoding.Encodings;
4343
import org.truffleruby.core.encoding.RubyEncoding;
4444
import org.truffleruby.core.encoding.TStringUtils;
4545

4646
import java.nio.charset.Charset;
4747

4848
public final class YARPLoader extends Loader {
4949

50-
public static ParseResult load(byte[] serialized, Nodes.Source source, EncodingManager encodingManager,
51-
RubySource rubySource) {
52-
return new YARPLoader(serialized, source, encodingManager, rubySource).load();
50+
public static ParseResult load(byte[] serialized, Nodes.Source source, RubySource rubySource) {
51+
return new YARPLoader(serialized, source, rubySource).load();
5352
}
5453

55-
private final EncodingManager encodingManager;
5654
private final RubySource rubySource;
5755
private RubyEncoding encoding = null;
5856

59-
public YARPLoader(byte[] serialized, Nodes.Source source, EncodingManager encodingManager, RubySource rubySource) {
57+
public YARPLoader(byte[] serialized, Nodes.Source source, RubySource rubySource) {
6058
super(serialized, source);
61-
this.encodingManager = encodingManager;
6259
this.rubySource = rubySource;
6360
}
6461

6562
@Override
6663
public Charset getEncodingCharset(String encodingName) {
67-
encoding = encodingManager.getRubyEncoding(encodingName);
64+
encoding = Encodings.getBuiltInEncoding(encodingName);
6865
assert encoding == rubySource.getEncoding();
6966
return null; // encodingCharset is not used
7067
}

‎src/main/java/org/truffleruby/parser/YARPTranslatorDriver.java

Lines changed: 28 additions & 64 deletions
Original file line numberDiff line numberDiff line change
@@ -174,29 +174,25 @@ public RootCallTarget parse(RubySource rubySource, ParserContext parserContext,
174174
}
175175

176176
// Parse to the YARP AST
177-
178-
final org.prism.Nodes.Node node;
179-
180177
final RubyDeferredWarnings rubyWarnings = new RubyDeferredWarnings();
181178

182179
// Only use the cache while loading top-level core library files, as eval() later could use
183180
// the same Source name but should not use the cache. For instance,
184181
// TOPLEVEL_BINDING.eval("self") would use the cache which is wrong.
185-
186-
// TODO: use cache
187-
// if (ParserCache.INSTANCE != null && parserContext == ParserContext.TOP_LEVEL &&
188-
// ParserCache.INSTANCE.containsKey(source.getName())) {
189-
// node = ParserCache.INSTANCE.get(source.getName());
190-
// } else {
191-
printParseTranslateExecuteMetric("before-parsing", context, source);
192-
ParseResult parseResult = context.getMetricsProfiler().callWithMetrics(
193-
"parsing",
194-
source.getName(),
195-
() -> parseToYARPAST(context, language, rubySource, staticScope, localVariableNames,
196-
parserConfiguration, rubyWarnings, parseEnvironment));
197-
printParseTranslateExecuteMetric("after-parsing", context, source);
198-
// }
199-
node = parseResult.value;
182+
final ParseResult parseResult;
183+
if (ParserCache.INSTANCE != null && parserContext == ParserContext.TOP_LEVEL &&
184+
ParserCache.INSTANCE.containsKey(source.getName())) {
185+
parseResult = ParserCache.INSTANCE.get(source.getName()).getLeft();
186+
} else {
187+
printParseTranslateExecuteMetric("before-parsing", context, source);
188+
parseResult = context.getMetricsProfiler().callWithMetrics(
189+
"parsing",
190+
source.getName(),
191+
() -> parseToYARPAST(context, language, rubySource, localVariableNames,
192+
parserConfiguration, rubyWarnings, parseEnvironment));
193+
printParseTranslateExecuteMetric("after-parsing", context, source);
194+
}
195+
var node = parseResult.value;
200196

201197
// Needs the magic comment to be parsed
202198
parseEnvironment.allowTruffleRubyPrimitives = parserConfiguration.allowTruffleRubyPrimitives;
@@ -397,7 +393,6 @@ public static org.prism.ParseResult parseToYARPAST(RubyContext context, RubyLang
397393
ParserConfiguration configuration, RubyDeferredWarnings rubyWarnings, ParseEnvironment parseEnvironment) {
398394
TruffleSafepoint.poll(DummyNode.INSTANCE);
399395

400-
// YARP begin
401396
byte[] sourceBytes = rubySource.getBytes();
402397

403398
byte[] filepath;
@@ -442,15 +437,20 @@ public static org.prism.ParseResult parseToYARPAST(RubyContext context, RubyLang
442437
byte[] serializedBytes = Parser.parseAndSerialize(sourceBytes, parsingOptions);
443438

444439
Nodes.Source yarpSource = parseEnvironment.yarpSource;
445-
ParseResult parseResult = YARPLoader.load(serializedBytes, yarpSource, context.getEncodingManager(),
446-
rubySource);
440+
ParseResult parseResult = YARPLoader.load(serializedBytes, yarpSource, rubySource);
447441

448442
final ParseResult.Error[] errors = parseResult.errors;
449443

450444
// collect warnings generated by the parser
451445
for (ParseResult.Warning warning : parseResult.warnings) {
452446
Nodes.Location location = warning.location;
453447
SourceSection section = rubySource.getSource().createSection(location.startOffset, location.length);
448+
449+
if (context == null) {
450+
throw CompilerDirectives.shouldNotReachHere(
451+
"Warning in " + RubyLanguage.filenameLine(section) + ": " + warning.message);
452+
}
453+
454454
int lineNumber = RubySource.getStartLineAdjusted(context, section);
455455

456456
switch (warning.level) {
@@ -462,7 +462,7 @@ public static org.prism.ParseResult parseToYARPAST(RubyContext context, RubyLang
462462
if (errors.length != 0) {
463463
// print warnings immediately
464464
// if there is no syntax error they will be printed in runtime
465-
if (!rubyWarnings.warnings.isEmpty()) {
465+
if (!rubyWarnings.warnings.isEmpty() && context != null) {
466466
EmitWarningsNode.printWarnings(context, rubyWarnings);
467467
}
468468

@@ -473,8 +473,13 @@ public static org.prism.ParseResult parseToYARPAST(RubyContext context, RubyLang
473473

474474
Nodes.Location location = error.location;
475475
SourceSection section = rubySource.getSource().createSection(location.startOffset, location.length);
476-
String message = context.fileLine(section) + ": " + error.message;
477476

477+
if (context == null) {
478+
throw CompilerDirectives.shouldNotReachHere(
479+
"Parse error in " + RubyLanguage.filenameLine(section) + ": " + error.message);
480+
}
481+
482+
String message = context.fileLine(section) + ": " + error.message;
478483
throw new RaiseException(
479484
context,
480485
context.getCoreExceptions().syntaxErrorAlreadyWithFileLine(message, null, section));
@@ -495,47 +500,6 @@ public static org.prism.ParseResult parseToYARPAST(RubyContext context, RubyLang
495500
}
496501

497502
return parseResult;
498-
// YARP end
499-
500-
// RubyParser parser = new RubyParser(lexerSource, rubyWarnings);
501-
// TruffleSafepoint.poll(DummyNode.INSTANCE); // RubyParser <clinit> takes a while
502-
// RubyParserResult result;
503-
// try {
504-
// result = parser.parse(configuration);
505-
// } catch (SyntaxException e) {
506-
// if (!rubyWarnings.warnings.isEmpty()) {
507-
// EmitWarningsNode.printWarnings(context, rubyWarnings);
508-
// }
509-
// switch (e.getPid()) {
510-
// case UNKNOWN_ENCODING:
511-
// case NOT_ASCII_COMPATIBLE:
512-
// if (context != null) {
513-
// throw new RaiseException(
514-
// context,
515-
// context.getCoreExceptions().argumentError(e.getMessage(), null));
516-
// } else {
517-
// throw e;
518-
// }
519-
// default:
520-
// StringBuilder buffer = new StringBuilder(100);
521-
// buffer.append(e.getFile()).append(':');
522-
// buffer.append(e.getLine() + rubySource.getLineOffset()).append(": ");
523-
// buffer.append(e.getMessage());
524-
//
525-
// if (context != null) {
526-
// throw new RaiseException(
527-
// context,
528-
// context.getCoreExceptions().syntaxErrorAlreadyWithFileLine(
529-
// buffer.toString(),
530-
// null,
531-
// rubySource.getSource().createSection(e.getLine())));
532-
// } else {
533-
// throw new UnsupportedOperationException(buffer.toString(), e);
534-
// }
535-
// }
536-
// }
537-
538-
// return (RootParseNode) result.getAST();
539503
}
540504

541505
public static RubySource createRubySource(Object code) {

0 commit comments

Comments
 (0)