Refactoring: special characters and escape sequences

Removed escape sequences for hardly ever used special characters (\a, \b, \f) and for custom character codes (\xhh). Added escape sequence for null character (\0). Improved displaying special characters in nested structures. Improved error handling for invalid escape sequences.

Author: adamsol

docs/.vuepress/utils/pyxell-syntax.js

@@ -5,5 +5,5 @@ module.exports = {
     variable_name: /\b[a-z_][\w']*/,  // so that apostrophes in variable names are handled properly
     type_name: /\b[A-Z][\w']*/,
     number: /(?:0b[01_]+|0o[0-7_]+|0x[\da-fA-F_]+|\d[\d_]*(?:r|(?:\.[\d_]+)?(?:[eE][-+]?[\d_]+|f)?)?)\b/,
-    string: /"(?:[^\\"]|\\(?:["\\abfnrt]|x[0-9a-fA-F]+))*(?:"|$)|'(?:[^\\']|\\(?:['\\abfnrt]|x[0-9a-fA-F]+))?(?:'|$)/,
+    string: /"(?:[^\\"]|\\.)*(?:"|$)|'(?:[^\\']|\\.)?(?:'|$)/,
 };

docs/specification.md

@@ -46,7 +46,7 @@ This section describes all data types available in Pyxell, together with their p
 | `Rat`      | arbitrary-precision rational number         | `1.5`         | `0r`          |
 | `Float`    | double-precision floating-point number      | `3.14f`       | `0f`          |
 | `Bool`     | boolean value                               | `true`        | `false`       |
-| `Char`     | single-byte character                       | `'A'`         | `'\x0'`       |
+| `Char`     | single-byte character                       | `'A'`         | `'\0'`        |
 | `String`   | arbitrary-length string of characters       | `"example"`   | `""`          |
 
 ### Compound types

lib/base.hpp

@@ -1054,14 +1054,44 @@ String toString(Bool x)
     return make_string(x ? "true" : "false");
 }
 
+std::string transform_character_to_display(char c)
+{
+    switch (c) {
+        case '\\': return "\\\\";
+        case '\n': return "\\n";
+        case '\r': return "\\r";
+        case '\t': return "\\t";
+        case '\0': return "\\0";
+    }
+    return std::string(1, c);
+}
+
 String toString(Char x)
 {
-    return make_string("'" + std::string(1, x) + "'");
+    auto r = make_string();
+    r->append("'");
+    if (x == '\'') {
+        r->append("\\'");
+    } else {
+        r->append(transform_character_to_display(x));
+    }
+    r->append("'");
+    return r;
 }
 
 String toString(const String& x)
 {
-    return make_string('"' + *x + '"');
+    auto r = make_string();
+    r->append("\"");
+    for (auto c: *x) {
+        if (c == '"') {
+            r->append("\\\"");
+        } else {
+            r->append(transform_character_to_display(c));
+        }
+    }
+    r->append("\"");
+    return r;
 }
 
 template <typename T>

src/errors.py

@@ -1,6 +1,7 @@
 
 class PyxellError(Exception):
 
+    InvalidEscapeSequence = lambda s: f"Invalid escape sequence `{s}`"
     InvalidSyntax = lambda: f"Syntax error"
 
     AbstractClass = lambda t: f"Cannot instantiate an abstract class `{t.show()}`"

src/lexer.py

@@ -12,9 +12,8 @@
 
 ID_REGEX = r'''[a-zA-Z_][\w']*'''
 NUMBER_REGEX = r'''0b[01_]+|0o[0-7_]+|0x[\da-fA-F_]+|\d[\d_]*(?:r|(?:\.[\d_]+)?(?:[eE][-+]?[\d_]+|f)?)?'''
-ESCAPE_SEQ_REGEX = r'''[\\abfnrt]|x[0-9a-fA-F]+'''
-CHAR_REGEX = rf'''\'(?:[^\\']|\\(?:'|{ESCAPE_SEQ_REGEX}))\''''
-STRING_REGEX = rf'''\"(?:[^\\"]|\\(?:"|{ESCAPE_SEQ_REGEX}))*\"'''
+CHAR_REGEX = rf'''\'(?:[^\\']|\\.)\''''
+STRING_REGEX = rf'''\"(?:[^\\"]|\\.)*\"'''
 
 
 class Token:

src/parser.py

@@ -412,9 +412,17 @@ def parse_expr_prefix_op(self, token):
                 'value': token.text == 'true',
             }
         if token.type in {Token.CHAR, Token.STRING}:
+            value = token.text[1:-1]
+            i = 0
+            while i < len(value):
+                if value[i] == '\\':
+                    i += 1
+                    if value[i] not in {'\\', '\'', '"', 'n', 'r', 't', '0'}:
+                        raise err(self.filepath, (token.position[0], token.position[1] + i), err.InvalidEscapeSequence(value[i-1:i+1]))
+                i += 1
             return {
                 **self.expr_node(f'Atom{token.type.capitalize()}', token),
-                'value': token.text[1:-1],
+                'value': value,
             }
         if token.text in {'null', 'super', 'this'}:
             return {

test/bad/strings/escape01.err

@@ -0,0 +1 @@
+:2:2: Invalid escape sequence `\x`.

test/bad/strings/char06.px renamed to test/bad/strings/escape01.px

@@ -0,0 +1 @@
+:2:8: Invalid escape sequence `\T`.

test/bad/strings/escape02.err

@@ -0,0 +1,2 @@
+
+print "\T"

test/bad/strings/escape02.px

@@ -0,0 +1 @@
+:2:4: Invalid escape sequence `\.`.

test/bad/strings/escape03.err

@@ -0,0 +1,2 @@
+
+"\"\.\\"

test/bad/strings/escape03.px

@@ -0,0 +1,2 @@
+
+'\'

test/bad/strings/char06.err renamed to test/bad/strings/escape04.err

@@ -0,0 +1 @@
+:2:1: Syntax error.

test/bad/strings/escape04.px

@@ -0,0 +1,2 @@
+
+"\"\\\"

test/bad/strings/escape05.err

@@ -0,0 +1,2 @@
+escape"\
+	test

test/bad/strings/escape05.px

@@ -0,0 +1,3 @@
+
+s = "escape\"\\\n\ttest"
+print s

test/good/strings/char06.out

@@ -0,0 +1,7 @@
+7
+true
+\'"
+ \
+("\\'\"\n\r\t\0", 1)
+['\\', '\'', '"', '\n', '\r', '\t', '\0']
+[92, 39, 34, 10, 13, 9, 0]

test/good/strings/char06.px

@@ -0,0 +1,9 @@
+
+s = "\\\'\"\n\r\t\0"
+c = ['\\', '\'', '\"', '\n', '\r', '\t', '\0']
+print s.length
+print s == c.join()
+print s[:4], c[0]
+print (s, 1)
+print c
+print c.map(_.code)

test/good/strings/escape01.out

@@ -1,2 +1,2 @@
-escape"\
-	test
+true
+6

test/good/strings/escape01.px

@@ -1,3 +1,4 @@
 
-s = "escape\"\\\n\ttest"
-print s
+s = ""
+print s.empty
+print "string".length + s.length

test/good/strings/escape02.out

@@ -1 +1 @@
-0.13.1.dev
+0.14.0