Skip to content

Commit 979f54d

Browse files
JJtan2002hyizhak
JJtan2002
and
hyizhak
authored
Added support for binary operators, unary operators and math library (#18)
* Changes that implement the treatment of ints as BigInts. TODO: explicit type conversions of BigInt back to Number for expressions and function calls. * Added support for binary operators, unary operators and math library * Prevent multiple assignments in the same function * Changes that implement the treatment of ints as BigInts. TODO: explicit type conversions of BigInt back to Number for expressions and function calls. * Added support for binary operators, unary operators and math library --------- Co-authored-by: yizhak <hyz0235@gmail.com>
1 parent 215754a commit 979f54d

File tree

8 files changed

+141
-41
lines changed

8 files changed

+141
-41
lines changed

‎src/ast-types.ts

Lines changed: 29 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎src/generate-ast.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,7 @@ export class AstWriter {
5151
main() {
5252
this.setup();
5353
this.defineAst("Expr", [
54+
"BigIntLiteral -> value: string",
5455
"Binary -> left: Expr, operator: Token, right: Expr",
5556
// Semantically different from Binary - for logical comparisons.
5657
"Compare -> left: Expr, operator: Token, right: Expr",

‎src/index.ts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,10 @@ export function parsePythonToEstreeAst(code: string,
150150
return translator.resolve(ast) as unknown as Program
151151
}
152152

153+
const text = `
154+
-1
155+
`
156+
console.dir(parsePythonToEstreeAst(text, 1, false));
153157
export * from './errors';
154158

155159
// import {ParserErrors, ResolverErrors, TokenizerErrors} from "./errors";

‎src/parser.ts

Lines changed: 14 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -141,9 +141,9 @@ export class Parser {
141141
const startToken = this.peek();
142142
const statements: Stmt[] = [];
143143
while (!this.isAtEnd()) {
144-
if (this.match(TokenType.NEWLINE)) {
145-
continue;
146-
}
144+
// if (this.match(TokenType.NEWLINE)) {
145+
// continue;
146+
// }
147147
statements.push(this.stmt());
148148
}
149149
const endToken = this.peek();
@@ -154,9 +154,9 @@ export class Parser {
154154
if (this.check(TokenType.DEF, TokenType.FOR, TokenType.IF, TokenType.WHILE)) {
155155
return this.compound_stmt();
156156
} else if (this.check(TokenType.NAME, ...PSEUD_NAMES, TokenType.NUMBER,
157-
TokenType.PASS, TokenType.BREAK, TokenType.CONTINUE,
157+
TokenType.PASS, TokenType.BREAK, TokenType.CONTINUE, TokenType.MINUS, TokenType.PLUS, TokenType.INDENT, TokenType.DEDENT,
158158
TokenType.RETURN, TokenType.FROM, TokenType.GLOBAL, TokenType.NONLOCAL,
159-
TokenType.ASSERT, TokenType.LPAR, TokenType.STRING, ...SPECIAL_IDENTIFIER_TOKENS)) {
159+
TokenType.ASSERT, TokenType.LPAR, TokenType.STRING, TokenType.BIGINT, ...SPECIAL_IDENTIFIER_TOKENS)) {
160160
return this.simple_stmt();
161161
}
162162
const startToken = this.peek();
@@ -239,6 +239,10 @@ export class Parser {
239239
let res = null;
240240
if (this.match(TokenType.NAME)) {
241241
res = this.assign_stmt();
242+
} else if (this.match(TokenType.INDENT)) {
243+
res = new StmtNS.Indent(startToken, startToken);
244+
} else if (this.match(TokenType.DEDENT)) {
245+
res = new StmtNS.Dedent(startToken, startToken);
242246
} else if (this.match(TokenType.PASS)) {
243247
res = new StmtNS.Pass(startToken, startToken);
244248
} else if (this.match(TokenType.BREAK)) {
@@ -255,7 +259,8 @@ export class Parser {
255259
res = new StmtNS.NonLocal(startToken, startToken, this.advance());
256260
} else if (this.match(TokenType.ASSERT)) {
257261
res = new StmtNS.Assert(startToken, startToken, this.test());
258-
} else if (this.check(TokenType.LPAR, TokenType.NUMBER, TokenType.STRING, ...SPECIAL_IDENTIFIER_TOKENS)) {
262+
} else if (this.check(TokenType.LPAR, TokenType.NUMBER, TokenType.STRING,
263+
TokenType.BIGINT, TokenType.MINUS, TokenType.PLUS, ...SPECIAL_IDENTIFIER_TOKENS)) {
259264
res = new StmtNS.SimpleExpr(startToken, startToken, this.test());
260265
} else {
261266
throw new Error("Unreachable code path");
@@ -501,6 +506,9 @@ export class Parser {
501506
if (this.match(TokenType.NUMBER)) {
502507
return new ExprNS.Literal(startToken, this.previous(), Number(this.previous().lexeme));
503508
}
509+
if (this.match(TokenType.BIGINT)) {
510+
return new ExprNS.BigIntLiteral(startToken, this.previous(), this.previous().lexeme);
511+
}
504512

505513
if (this.match(TokenType.NAME, ...PSEUD_NAMES)) {
506514
return new ExprNS.Variable(startToken, this.previous(), this.previous());

‎src/resolver.ts

Lines changed: 36 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
import {StmtNS, ExprNS} from "./ast-types";
1+
import { StmtNS, ExprNS } from "./ast-types";
22
type Expr = ExprNS.Expr;
33
type Stmt = StmtNS.Stmt;
4-
import {Token} from "./tokenizer";
5-
import {TokenType} from "./tokens";
6-
import {ResolverErrors} from "./errors";
4+
import { Token } from "./tokenizer";
5+
import { TokenType } from "./tokens";
6+
import { ResolverErrors } from "./errors";
77

88
const levenshtein = require('fast-levenshtein');
99

@@ -35,7 +35,7 @@ class Environment {
3535
const name = identifier.lexeme;
3636
let distance = 0;
3737
let curr: Environment | null = this;
38-
while(curr !== null) {
38+
while (curr !== null) {
3939
if (curr.names.has(name)) {
4040
break;
4141
}
@@ -49,7 +49,7 @@ class Environment {
4949
lookupNameCurrentEnv(identifier: Token): Token | undefined {
5050
return this.names.get(identifier.lexeme);
5151
}
52-
lookupNameCurrentEnvWithError(identifier: Token){
52+
lookupNameCurrentEnvWithError(identifier: Token) {
5353
if (this.lookupName(identifier) < 0) {
5454
throw new ResolverErrors.NameNotFoundError(identifier.line, identifier.col,
5555
this.source,
@@ -104,7 +104,7 @@ class Environment {
104104
let minDistance = Infinity;
105105
let minName = null;
106106
let curr: Environment | null = this;
107-
while(curr !== null) {
107+
while (curr !== null) {
108108
for (const declName of curr.names.keys()) {
109109
const dist = levenshtein.get(name, declName);
110110
if (dist < minDistance) {
@@ -125,12 +125,13 @@ class Environment {
125125
export class Resolver implements StmtNS.Visitor<void>, ExprNS.Visitor<void> {
126126
source: string;
127127
ast: Stmt;
128+
// change the environment to be suite scope as in python
128129
environment: Environment | null;
129130
constructor(source: string, ast: Stmt) {
130131
this.source = source;
131132
this.ast = ast;
132133
// The global environment
133-
this.environment = new Environment(source,null, new Map([
134+
this.environment = new Environment(source, null, new Map([
134135
["range", new Token(TokenType.NAME, "range", 0, 0, 0)],
135136
["display", new Token(TokenType.NAME, "display", 0, 0, 0)],
136137
["stringify", new Token(TokenType.NAME, "stringify", 0, 0, 0)],
@@ -173,21 +174,40 @@ export class Resolver implements StmtNS.Visitor<void>, ExprNS.Visitor<void> {
173174
this.environment = oldEnv;
174175
}
175176

177+
visitIndentCreation(stmt: StmtNS.Indent): void {
178+
// Create a new environment.
179+
const oldEnv = this.environment;
180+
this.environment = new Environment(this.source, this.environment, new Map());
181+
}
182+
183+
visitDedentCreation(stmt: StmtNS.Dedent): void {
184+
// Switch to the previous environment.
185+
if (this.environment?.enclosing !== undefined) {
186+
this.environment = this.environment.enclosing;
187+
}
188+
}
189+
176190
visitFunctionDefStmt(stmt: StmtNS.FunctionDef) {
177191
this.environment?.declareName(stmt.name);
178192
this.environment?.functions.add(stmt.name.lexeme);
179-
// Create a new environment.
180-
const oldEnv = this.environment;
181-
// Assign the parameters to the new environment.
182-
const newEnv = new Map(
193+
// // Create a new environment.
194+
// const oldEnv = this.environment;
195+
// // Assign the parameters to the new environment.
196+
// const newEnv = new Map(
197+
// stmt.parameters.map(param => [param.lexeme, param])
198+
// );
199+
// this.environment = new Environment(this.source, this.environment, newEnv);
200+
const params = new Map(
183201
stmt.parameters.map(param => [param.lexeme, param])
184202
);
185-
this.environment = new Environment(this.source, this.environment, newEnv);
203+
if (this.environment !== null) {
204+
this.environment.names = params;
205+
}
186206
this.resolve(stmt.body);
187207
// Grab identifiers from that new environment. That are NOT functions.
188208
// stmt.varDecls = this.varDeclNames(this.environment.names)
189209
// Restore old environment
190-
this.environment = oldEnv;
210+
// this.environment = oldEnv;
191211
}
192212

193213
visitAnnAssignStmt(stmt: StmtNS.AnnAssign): void {
@@ -319,6 +339,7 @@ export class Resolver implements StmtNS.Visitor<void>, ExprNS.Visitor<void> {
319339
}
320340
visitLiteralExpr(expr: ExprNS.Literal): void {
321341
}
322-
342+
visitBigIntLiteralExpr(expr: ExprNS.BigIntLiteral): void {
343+
}
323344

324345
}

‎src/tokenizer.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -291,6 +291,11 @@ export class Tokenizer {
291291
while (this.isDigit(this.peek())) {
292292
this.advance();
293293
}
294+
295+
if (this.peek() !== '.' && this.peek() !== 'e') {
296+
this.addToken(TokenType.BIGINT);
297+
return;
298+
}
294299
// Fractional part
295300
if (this.peek() === '.') {
296301
this.advance();

‎src/tokens.ts

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ export enum TokenType {
77
ENDMARKER,
88
NAME,
99
NUMBER,
10+
BIGINT,
1011
STRING,
1112
NEWLINE,
1213
INDENT,

0 commit comments

Comments
 (0)