[GR-18163] Fix wrong constant/identifier detection in lexer for non-ASCII encodings (#2102)

PullRequest: truffleruby/2050

Author: eregon

CHANGELOG.md

@@ -16,6 +16,7 @@ Bug fixes:
 * Fixed `coverage` issue when `*eval` is used (#2078).
 * Use expanded load paths for feature matching (#1501).
 * Fixed `SystemStackError` sometimes replaced by an internal Java `NoClassDefFoundError` on JVM (#1743).
+* Fixed constant/identifier detection in lexer for non-ASCII encodings (#2079, #2102, @ivoanjo).
 
 Compatibility:
 

spec/ruby/language/variables_spec.rb

@@ -796,3 +796,32 @@ module VariableSpecs
     end
   end
 end
+
+describe 'Allowed characters' do
+  ruby_version_is "2.6" do
+    # new feature in 2.6 -- https://bugs.ruby-lang.org/issues/13770
+    it 'does not allow non-ASCII upcased characters at the beginning' do
+      -> do
+        eval <<-CODE
+          def test
+            ἍBB = 1
+          end
+        CODE
+      end.should raise_error(SyntaxError, /dynamic constant assignment/)
+    end
+  end
+
+  it 'allows non-ASCII lowercased characters at the beginning' do
+    result = nil
+
+    eval <<-CODE
+      def test
+        μ = 1
+      end
+
+      result = test
+    CODE
+
+    result.should == 1
+  end
+end

src/main/java/org/truffleruby/parser/lexer/RubyLexer.java

@@ -1802,7 +1802,7 @@ private int identifier(int c, boolean commandState) {
             }
             tempVal = createTokenRope();
 
-            if (result == 0 && Character.isUpperCase(tempVal.get(0) & 0xFF)) {
+            if (result == 0 && isFirstCodepointUppercase(tempVal)) {
                 result = RubyParser.tCONSTANT;
             } else {
                 result = RubyParser.tIDENTIFIER;
@@ -3503,12 +3503,12 @@ public boolean update_heredoc_indent(int c) {
     }
 
     public void validateFormalIdentifier(Rope identifier) {
-        int first = identifier.get(0) & 0xFF;
-
-        if (Character.isUpperCase(first)) {
+        if (isFirstCodepointUppercase(identifier)) {
             compile_error("formal argument cannot be a constant");
         }
 
+        int first = identifier.get(0) & 0xFF;
+
         switch (first) {
             case '@':
                 if (identifier.get(1) == '@') {
@@ -3675,4 +3675,18 @@ protected boolean isSpaceArg(int c, boolean spaceSeen) {
         return isARG() && spaceSeen && !Character.isWhitespace(c);
     }
 
+    /** Encoding-aware (including multi-byte encodings) check of first codepoint of a given rope, usually to determine
+     * if it is a constant */
+    private boolean isFirstCodepointUppercase(Rope rope) {
+        Encoding ropeEncoding = rope.encoding;
+        int firstByte = rope.get(0) & 0xFF;
+
+        if (ropeEncoding.isAsciiCompatible() && isASCII(firstByte)) {
+            return StringSupport.isAsciiUppercase((byte) firstByte);
+        } else {
+            byte[] ropeBytes = rope.getBytes();
+            int firstCharacter = ropeEncoding.mbcToCode(ropeBytes, 0, ropeBytes.length);
+            return ropeEncoding.isUpper(firstCharacter);
+        }
+    }
 }