Skip to content

Commit d0b1500

Browse files
committed
[python mode] Make format string tokenizing work without an extra state field
Issue codemirror#5338
1 parent 30ae1ed commit d0b1500

File tree

2 files changed

+27
-26
lines changed

2 files changed

+27
-26
lines changed

mode/python/python.js

Lines changed: 26 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -143,15 +143,15 @@
143143
// Handle Strings
144144
if (stream.match(stringPrefixes)) {
145145
var isFmtString = stream.current().toLowerCase().indexOf('f') !== -1;
146-
if (!isFmtString || state.fstr_state !== null) {
146+
if (!isFmtString) {
147147
// if this is a nested format string (e.g. f' { f"{10*10}" + "a" }' )
148148
// we do not format the nested expression and treat the nested format
149149
// string as regular string
150150
state.tokenize = tokenStringFactory(stream.current());
151151
return state.tokenize(stream, state);
152152
} else {
153153
// need to do something more sophisticated
154-
state.tokenize = formatStringFactory(stream.current());
154+
state.tokenize = formatStringFactory(stream.current(), state.tokenize);
155155
return state.tokenize(stream, state);
156156
}
157157
}
@@ -184,48 +184,49 @@
184184
return ERRORCLASS;
185185
}
186186

187-
function formatStringFactory(delimiter) {
187+
function formatStringFactory(delimiter, tokenOuter) {
188188
while ("rubf".indexOf(delimiter.charAt(0).toLowerCase()) >= 0)
189189
delimiter = delimiter.substr(1);
190190

191191
var singleline = delimiter.length == 1;
192192
var OUTCLASS = "string";
193193

194-
function tokenString(stream, state) {
195-
if (state.fstr_state) {
196-
// inside f-str Expression
197-
if (stream.match(delimiter)) {
198-
// expression ends pre-maturally, but very common in editing
199-
// Could show error to remind users to close brace here
200-
state.fstr_state = null;
201-
return OUTCLASS;
202-
} else if (stream.match('{')) {
203-
// starting brace, if not eaten below
204-
return "punctuation";
205-
} else if (stream.match('}')) {
206-
// return to regular inside string state
207-
state.fstr_state = null;
208-
return "punctuation";
209-
} else {
210-
// use tokenBaseInner to parse the expression
211-
return tokenBaseInner(stream, state.fstr_state);
212-
}
194+
function tokenFString(stream, state) {
195+
// inside f-str Expression
196+
if (stream.match(delimiter)) {
197+
// expression ends pre-maturally, but very common in editing
198+
// Could show error to remind users to close brace here
199+
state.tokenize = tokenString
200+
return OUTCLASS;
201+
} else if (stream.match('{')) {
202+
// starting brace, if not eaten below
203+
return "punctuation";
204+
} else if (stream.match('}')) {
205+
// return to regular inside string state
206+
state.tokenize = tokenString
207+
return "punctuation";
208+
} else {
209+
// use tokenBaseInner to parse the expression
210+
return tokenBaseInner(stream, state);
213211
}
212+
}
213+
214+
function tokenString(stream, state) {
214215
while (!stream.eol()) {
215216
stream.eatWhile(/[^'"\{\}\\]/);
216217
if (stream.eat("\\")) {
217218
stream.next();
218219
if (singleline && stream.eol())
219220
return OUTCLASS;
220221
} else if (stream.match(delimiter)) {
221-
state.tokenize = tokenBase;
222+
state.tokenize = tokenOuter;
222223
return OUTCLASS;
223224
} else if (stream.match('{{')) {
224225
// ignore {{ in f-str
225226
return OUTCLASS;
226227
} else if (stream.match('{', false)) {
227228
// switch to nested mode
228-
state.fstr_state = {};
229+
state.tokenize = tokenFString
229230
if (stream.current()) {
230231
return OUTCLASS;
231232
} else {
@@ -246,7 +247,7 @@
246247
if (parserConf.singleLineStringErrors)
247248
return ERRORCLASS;
248249
else
249-
state.tokenize = tokenBase;
250+
state.tokenize = tokenOuter;
250251
}
251252
return OUTCLASS;
252253
}
@@ -358,7 +359,6 @@
358359
return {
359360
tokenize: tokenBase,
360361
scopes: [{offset: basecolumn || 0, type: "py", align: null}],
361-
fstr_state: null,
362362
indent: basecolumn || 0,
363363
lastToken: null,
364364
lambda: false,

mode/python/test.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,5 +33,6 @@
3333
MT("fValidStringPrefix", "[string f'this is a]{[variable formatted]}[string string']");
3434
MT("fValidExpressioninFString", "[string f'expression ]{[number 100][operator *][number 5]}[string string']");
3535
MT("fInvalidFString", "[error f'this is wrong}]");
36+
MT("fNestedFString", "[string f'expression ]{[number 100] [operator +] [string f'inner]{[number 5]}[string ']}[string string']");
3637
MT("uValidStringPrefix", "[string u'this is an unicode string']");
3738
})();

0 commit comments

Comments
 (0)