Skip to content

Commit 25ae2b3

Browse files
dfsekdfsek
committed
drastically increase script loading speed via optimised token pipeline
1 parent 44176f7 commit 25ae2b3

File tree

7 files changed

+92
-163
lines changed

7 files changed

+92
-163
lines changed

‎common/src/main/java/com/dfsek/terra/api/structures/parser/Parser.java

Lines changed: 16 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,6 @@
4444
import com.dfsek.terra.api.structures.tokenizer.Position;
4545
import com.dfsek.terra.api.structures.tokenizer.Token;
4646
import com.dfsek.terra.api.structures.tokenizer.Tokenizer;
47-
import com.dfsek.terra.api.structures.tokenizer.exceptions.TokenizerException;
4847
import com.dfsek.terra.api.util.GlueList;
4948

5049
import java.util.Collections;
@@ -79,18 +78,7 @@ public String getID() {
7978
* @throws ParseException If parsing fails.
8079
*/
8180
public Block parse() throws ParseException {
82-
Tokenizer tokenizer = new Tokenizer(data);
83-
84-
TokenHolder tokens = new TokenHolder();
85-
try {
86-
Token t = tokenizer.fetch();
87-
while(t != null) {
88-
tokens.add(t);
89-
t = tokenizer.fetch();
90-
}
91-
} catch(TokenizerException e) {
92-
throw new ParseException("Failed to tokenize input", new Position(0, 0), e);
93-
}
81+
Tokenizer tokens = new Tokenizer(data);
9482

9583
// Parse ID
9684
ParserUtil.checkType(tokens.consume(), Token.Type.ID); // First token must be ID
@@ -99,21 +87,12 @@ public Block parse() throws ParseException {
9987
ParserUtil.checkType(tokens.consume(), Token.Type.STATEMENT_END);
10088
this.id = idToken.getContent();
10189

102-
// Check for dangling brackets
103-
int blockLevel = 0;
104-
for(Token t : tokens.getTokens()) {
105-
if(t.getType().equals(Token.Type.BLOCK_BEGIN)) blockLevel++;
106-
else if(t.getType().equals(Token.Type.BLOCK_END)) blockLevel--;
107-
if(blockLevel < 0) throw new ParseException("Dangling closing brace", t.getPosition());
108-
}
109-
if(blockLevel != 0)
110-
throw new ParseException("Dangling opening brace", tokens.getTokens().get(tokens.getTokens().size() - 1).getPosition());
11190

11291
return parseBlock(tokens, new HashMap<>(), false);
11392
}
11493

11594

116-
private Keyword<?> parseLoopLike(TokenHolder tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
95+
private Keyword<?> parseLoopLike(Tokenizer tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
11796

11897
Token identifier = tokens.consume();
11998
ParserUtil.checkType(identifier, Token.Type.IF_STATEMENT, Token.Type.WHILE_LOOP, Token.Type.FOR_LOOP);
@@ -132,7 +111,7 @@ private Keyword<?> parseLoopLike(TokenHolder tokens, Map<String, Variable<?>> va
132111
}
133112
}
134113

135-
private WhileKeyword parseWhileLoop(TokenHolder tokens, Map<String, Variable<?>> variableMap, Position start) throws ParseException {
114+
private WhileKeyword parseWhileLoop(Tokenizer tokens, Map<String, Variable<?>> variableMap, Position start) throws ParseException {
136115
Returnable<?> first = parseExpression(tokens, true, variableMap);
137116
ParserUtil.checkReturnType(first, Returnable.ReturnType.BOOLEAN);
138117

@@ -141,7 +120,7 @@ private WhileKeyword parseWhileLoop(TokenHolder tokens, Map<String, Variable<?>>
141120
return new WhileKeyword(parseStatementBlock(tokens, variableMap, true), (Returnable<Boolean>) first, start); // While loop
142121
}
143122

144-
private IfKeyword parseIfStatement(TokenHolder tokens, Map<String, Variable<?>> variableMap, Position start, boolean loop) throws ParseException {
123+
private IfKeyword parseIfStatement(Tokenizer tokens, Map<String, Variable<?>> variableMap, Position start, boolean loop) throws ParseException {
145124
Returnable<?> condition = parseExpression(tokens, true, variableMap);
146125
ParserUtil.checkReturnType(condition, Returnable.ReturnType.BOOLEAN);
147126

@@ -168,7 +147,7 @@ private IfKeyword parseIfStatement(TokenHolder tokens, Map<String, Variable<?>>
168147
return new IfKeyword(statement, (Returnable<Boolean>) condition, elseIf, elseBlock, start); // If statement
169148
}
170149

171-
private Block parseStatementBlock(TokenHolder tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
150+
private Block parseStatementBlock(Tokenizer tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
172151

173152
if(tokens.get().getType().equals(Token.Type.BLOCK_BEGIN)) {
174153
ParserUtil.checkType(tokens.consume(), Token.Type.BLOCK_BEGIN);
@@ -183,7 +162,7 @@ private Block parseStatementBlock(TokenHolder tokens, Map<String, Variable<?>> v
183162
}
184163
}
185164

186-
private ForKeyword parseForLoop(TokenHolder tokens, Map<String, Variable<?>> old, Position start) throws ParseException {
165+
private ForKeyword parseForLoop(Tokenizer tokens, Map<String, Variable<?>> old, Position start) throws ParseException {
187166
Map<String, Variable<?>> variableMap = new HashMap<>(old); // New scope
188167
Token f = tokens.get();
189168
ParserUtil.checkType(f, Token.Type.NUMBER_VARIABLE, Token.Type.STRING_VARIABLE, Token.Type.BOOLEAN_VARIABLE, Token.Type.IDENTIFIER);
@@ -216,7 +195,7 @@ private ForKeyword parseForLoop(TokenHolder tokens, Map<String, Variable<?>> old
216195
return new ForKeyword(parseStatementBlock(tokens, variableMap, true), initializer, (Returnable<Boolean>) conditional, incrementer, start);
217196
}
218197

219-
private Returnable<?> parseExpression(TokenHolder tokens, boolean full, Map<String, Variable<?>> variableMap) throws ParseException {
198+
private Returnable<?> parseExpression(Tokenizer tokens, boolean full, Map<String, Variable<?>> variableMap) throws ParseException {
220199
boolean booleanInverted = false; // Check for boolean not operator
221200
boolean negate = false;
222201
if(tokens.get().getType().equals(Token.Type.BOOLEAN_NOT)) {
@@ -259,7 +238,7 @@ else if(variableMap.containsKey(id.getContent())) {
259238
return expression;
260239
}
261240

262-
private ConstantExpression<?> parseConstantExpression(TokenHolder tokens) throws ParseException {
241+
private ConstantExpression<?> parseConstantExpression(Tokenizer tokens) throws ParseException {
263242
Token constantToken = tokens.consume();
264243
Position position = constantToken.getPosition();
265244
switch(constantToken.getType()) {
@@ -275,15 +254,15 @@ private ConstantExpression<?> parseConstantExpression(TokenHolder tokens) throws
275254
}
276255
}
277256

278-
private Returnable<?> parseGroup(TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
257+
private Returnable<?> parseGroup(Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
279258
ParserUtil.checkType(tokens.consume(), Token.Type.GROUP_BEGIN);
280259
Returnable<?> expression = parseExpression(tokens, true, variableMap); // Parse inside of group as a separate expression
281260
ParserUtil.checkType(tokens.consume(), Token.Type.GROUP_END);
282261
return expression;
283262
}
284263

285264

286-
private BinaryOperation<?, ?> parseBinaryOperation(Returnable<?> left, TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
265+
private BinaryOperation<?, ?> parseBinaryOperation(Returnable<?> left, Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
287266
Token binaryOperator = tokens.consume();
288267
ParserUtil.checkBinaryOperator(binaryOperator);
289268

@@ -337,7 +316,7 @@ private Returnable<?> parseGroup(TokenHolder tokens, Map<String, Variable<?>> va
337316
}
338317
}
339318

340-
private Variable<?> parseVariableDeclaration(TokenHolder tokens, Returnable.ReturnType type) throws ParseException {
319+
private Variable<?> parseVariableDeclaration(Tokenizer tokens, Returnable.ReturnType type) throws ParseException {
341320
ParserUtil.checkVarType(tokens.get(), type); // Check for type mismatch
342321
switch(type) {
343322
case NUMBER:
@@ -350,7 +329,7 @@ private Variable<?> parseVariableDeclaration(TokenHolder tokens, Returnable.Retu
350329
throw new UnsupportedOperationException("Unsupported variable type: " + type);
351330
}
352331

353-
private Block parseBlock(TokenHolder tokens, Map<String, Variable<?>> superVars, boolean loop) throws ParseException {
332+
private Block parseBlock(Tokenizer tokens, Map<String, Variable<?>> superVars, boolean loop) throws ParseException {
354333
List<Item<?>> parsedItems = new GlueList<>();
355334

356335
Map<String, Variable<?>> parsedVariables = new HashMap<>(superVars); // New hashmap as to not mutate parent scope's declarations.
@@ -366,7 +345,7 @@ private Block parseBlock(TokenHolder tokens, Map<String, Variable<?>> superVars,
366345
return new Block(parsedItems, first.getPosition());
367346
}
368347

369-
private Item<?> parseItem(TokenHolder tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
348+
private Item<?> parseItem(Tokenizer tokens, Map<String, Variable<?>> variableMap, boolean loop) throws ParseException {
370349
Token token = tokens.get();
371350
if(loop) ParserUtil.checkType(token, Token.Type.IDENTIFIER, Token.Type.IF_STATEMENT, Token.Type.WHILE_LOOP, Token.Type.FOR_LOOP,
372351
Token.Type.NUMBER_VARIABLE, Token.Type.STRING_VARIABLE, Token.Type.BOOLEAN_VARIABLE, Token.Type.RETURN, Token.Type.BREAK, Token.Type.CONTINUE, Token.Type.FAIL);
@@ -401,7 +380,7 @@ private Item<?> parseItem(TokenHolder tokens, Map<String, Variable<?>> variableM
401380
else throw new UnsupportedOperationException("Unexpected token " + token.getType() + ": " + token.getPosition());
402381
}
403382

404-
private Assignment<?> parseAssignment(Variable<?> variable, TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
383+
private Assignment<?> parseAssignment(Variable<?> variable, Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
405384
Token name = tokens.get();
406385

407386
ParserUtil.checkType(tokens.consume(), Token.Type.IDENTIFIER);
@@ -415,7 +394,7 @@ private Assignment<?> parseAssignment(Variable<?> variable, TokenHolder tokens,
415394
return new Assignment<>((Variable<Object>) variable, (Returnable<Object>) expression, name.getPosition());
416395
}
417396

418-
private Function<?> parseFunction(TokenHolder tokens, boolean fullStatement, Map<String, Variable<?>> variableMap) throws ParseException {
397+
private Function<?> parseFunction(Tokenizer tokens, boolean fullStatement, Map<String, Variable<?>> variableMap) throws ParseException {
419398
Token identifier = tokens.consume();
420399
ParserUtil.checkType(identifier, Token.Type.IDENTIFIER); // First token must be identifier
421400

@@ -449,7 +428,7 @@ private Function<?> parseFunction(TokenHolder tokens, boolean fullStatement, Map
449428
}
450429

451430

452-
private List<Returnable<?>> getArgs(TokenHolder tokens, Map<String, Variable<?>> variableMap) throws ParseException {
431+
private List<Returnable<?>> getArgs(Tokenizer tokens, Map<String, Variable<?>> variableMap) throws ParseException {
453432
List<Returnable<?>> args = new GlueList<>();
454433

455434
while(!tokens.get().getType().equals(Token.Type.GROUP_END)) {

‎common/src/main/java/com/dfsek/terra/api/structures/parser/TokenHolder.java

Lines changed: 0 additions & 59 deletions
This file was deleted.

‎common/src/main/java/com/dfsek/terra/api/structures/tokenizer/Tokenizer.java

Lines changed: 57 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,74 @@
11
package com.dfsek.terra.api.structures.tokenizer;
22

3+
import com.dfsek.terra.api.structures.parser.exceptions.ParseException;
34
import com.dfsek.terra.api.structures.tokenizer.exceptions.EOFException;
45
import com.dfsek.terra.api.structures.tokenizer.exceptions.FormatException;
56
import com.dfsek.terra.api.structures.tokenizer.exceptions.TokenizerException;
67
import com.google.common.collect.Sets;
78

89
import java.io.StringReader;
910
import java.util.Set;
11+
import java.util.Stack;
1012

1113
public class Tokenizer {
14+
public static final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', '=', '{', '}', '+', '-', '*', '/', '>', '<', '!'); // Reserved chars
1215
private final Lookahead reader;
16+
private final Stack<Token> brackets = new Stack<>();
17+
private Token current;
1318

14-
public static final Set<Character> syntaxSignificant = Sets.newHashSet(';', '(', ')', '"', ',', '\\', '=', '{', '}', '+', '-', '*', '/', '>', '<', '!'); // Reserved chars
19+
public Tokenizer(String data) throws ParseException {
20+
reader = new Lookahead(new StringReader(data + '\0'));
21+
current = fetchCheck();
22+
}
1523

24+
/**
25+
* Get the first token.
26+
*
27+
* @return First token
28+
* @throws ParseException If token does not exist
29+
*/
30+
public Token get() throws ParseException {
31+
if(!hasNext()) throw new ParseException("Unexpected end of input", current.getPosition());
32+
return current;
33+
}
1634

17-
public Tokenizer(String data) {
18-
reader = new Lookahead(new StringReader(data + '\0'));
35+
/**
36+
* Consume (get and remove) the first token.
37+
*
38+
* @return First token
39+
* @throws ParseException If token does not exist
40+
*/
41+
public Token consume() throws ParseException {
42+
if(!hasNext()) throw new ParseException("Unexpected end of input", current.getPosition());
43+
Token temp = current;
44+
current = fetchCheck();
45+
return temp;
46+
}
47+
48+
/**
49+
* Whether this {@code Tokenizer} contains additional tokens.
50+
*
51+
* @return {@code true} if more tokens are present, otherwise {@code false}
52+
*/
53+
public boolean hasNext() {
54+
return !(current == null);
55+
}
56+
57+
private Token fetchCheck() throws ParseException {
58+
Token fetch = fetch();
59+
if(fetch != null) {
60+
if(fetch.getType().equals(Token.Type.BLOCK_BEGIN)) brackets.push(fetch); // Opening bracket
61+
else if(fetch.getType().equals(Token.Type.BLOCK_END)) {
62+
if(!brackets.isEmpty()) brackets.pop();
63+
else throw new ParseException("Dangling opening brace", new Position(0, 0));
64+
}
65+
} else if(!brackets.isEmpty()) {
66+
throw new ParseException("Dangling closing brace", brackets.peek().getPosition());
67+
}
68+
return fetch;
1969
}
2070

21-
public Token fetch() throws TokenizerException {
71+
private Token fetch() throws TokenizerException {
2272
while(!reader.current().isEOF() && reader.current().isWhitespace()) reader.consume();
2373

2474
while(reader.matches("//", true)) skipLine(); // Skip line if comment
@@ -66,7 +116,7 @@ public Token fetch() throws TokenizerException {
66116
continue;
67117
} else ignoreNext = false;
68118
if(reader.current().isEOF())
69-
throw new FormatException("No end of string literal found. " + reader.getLine() + ":" + reader.getIndex());
119+
throw new FormatException("No end of string literal found. ", new Position(reader.getLine(), reader.getIndex()));
70120
string.append(reader.consume());
71121
}
72122
reader.consume(); // Consume last quote
@@ -166,14 +216,15 @@ private void consumeWhitespace() {
166216
}
167217

168218
private void skipTo(String s) throws EOFException {
219+
Position begin = new Position(reader.getLine(), reader.getIndex());
169220
while(!reader.current().isEOF()) {
170221
if(reader.matches(s, true)) {
171222
consumeWhitespace();
172223
return;
173224
}
174225
reader.consume();
175226
}
176-
throw new EOFException("No end of expression found.");
227+
throw new EOFException("No end of expression found.", begin);
177228
}
178229

179230
public boolean isSyntaxSignificant(char c) {

‎common/src/main/java/com/dfsek/terra/api/structures/tokenizer/exceptions/EOFException.java

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,14 @@
11
package com.dfsek.terra.api.structures.tokenizer.exceptions;
22

3-
public class EOFException extends TokenizerException {
4-
5-
public EOFException(String s) {
6-
super(s);
7-
}
3+
import com.dfsek.terra.api.structures.tokenizer.Position;
84

9-
public EOFException() {
10-
super();
11-
}
5+
public class EOFException extends TokenizerException {
126

13-
public EOFException(String message, Throwable cause) {
14-
super(message, cause);
7+
public EOFException(String message, Position position) {
8+
super(message, position);
159
}
1610

17-
public EOFException(Throwable cause) {
18-
super(cause);
11+
public EOFException(String message, Position position, Throwable cause) {
12+
super(message, position, cause);
1913
}
2014
}

‎common/src/main/java/com/dfsek/terra/api/structures/tokenizer/exceptions/FormatException.java

Lines changed: 6 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,14 @@
11
package com.dfsek.terra.api.structures.tokenizer.exceptions;
22

3-
public class FormatException extends TokenizerException {
4-
5-
public FormatException(String s) {
6-
super(s);
7-
}
3+
import com.dfsek.terra.api.structures.tokenizer.Position;
84

9-
public FormatException() {
10-
super();
11-
}
5+
public class FormatException extends TokenizerException {
126

13-
public FormatException(String message, Throwable cause) {
14-
super(message, cause);
7+
public FormatException(String message, Position position) {
8+
super(message, position);
159
}
1610

17-
public FormatException(Throwable cause) {
18-
super(cause);
11+
public FormatException(String message, Position position, Throwable cause) {
12+
super(message, position, cause);
1913
}
2014
}

0 commit comments

Comments
 (0)