further clarifications

Author: Nicolas Laurent

src/main/java/org/truffleruby/parser/lexer/HeredocTerm.java

@@ -104,9 +104,9 @@ public int parseString(RubyLexer lexer) {
             return error(lexer, nd_lit);
         }
 
-        // Found end marker for this heredoc, on the very first line
+        // Found end marker for this heredoc, at the start of a line
         if (lexer.was_bol() && lexer.whole_match_p(this.nd_lit, indent)) {
-            lexer.heredoc_restore(this);
+            lexer.heredoc_restore(this); // will also skip over the end marker
             lexer.setStrTerm(null);
             lexer.setState(EXPR_END);
             return RubyParser.tSTRING_END;
@@ -173,11 +173,15 @@ public int parseString(RubyLexer lexer) {
             RopeBuilder tok = new RopeBuilder();
             tok.setEncoding(lexer.getEncoding());
 
-            // TODO why is this needed at the start?
             if (c == '#') {
                 // interpolated variable or block begin
+                // This returns tSTRING_DVAR (if it finds $, @ or @@), tSTRING_DBEG (if it finds '{'), or 0 (none of
+                // these things were found).
                 int token = lexer.peekVariableName(RubyParser.tSTRING_DVAR, RubyParser.tSTRING_DBEG);
                 if (token != 0) {
+                    // Emit the token - note that the parser will unset RubyLexer#lex_strTerm while the variable or
+                    // block is being parse and restore it when it is done, allowing the rest of the heredoc to be
+                    // processed.
                     return token;
                 }
                 tok.append('#');
@@ -187,10 +191,12 @@ public int parseString(RubyLexer lexer) {
             do {
                 lexer.pushback(c);
 
-                Encoding enc[] = new Encoding[1];
+                Encoding[] enc = new Encoding[1];
                 enc[0] = lexer.getEncoding();
 
-                // parse the line into the buffer, as a regular string (with expansion)
+                // Parse the next string segment into the buffer, as a regular string (with expansion).
+                // The segment might terminate because of a newline, line continuation (\\) or because of a
+                // an interpolation (#{...}, #@foo, #$foo, etc).
                 if ((c = new StringTerm(flags, '\0', '\n', lexer.ruby_sourceline)
                         .parseStringIntoBuffer(lexer, tok, enc)) == EOF) {
                     if (lexer.eofp) {
@@ -205,7 +211,7 @@ public int parseString(RubyLexer lexer) {
                     return RubyParser.tSTRING_CONTENT;
                 }
 
-                // TODO is this a newline?
+                // append the terminating newline
                 tok.append(lexer.nextc());
 
                 if (lexer.getHeredocIndent() > 0) {
@@ -219,6 +225,7 @@ public int parseString(RubyLexer lexer) {
                 if ((c = lexer.nextc()) == EOF) {
                     return error(lexer, nd_lit);
                 }
+                // NOTE: The end marker is not processed here, but in the next call to HeredocTerm#parseString
             } while (!lexer.whole_match_p(nd_lit, indent));
             str = tok;
         }

src/main/java/org/truffleruby/parser/lexer/RubyLexer.java

@@ -3455,7 +3455,7 @@ public void setHeredocLineIndent(int heredoc_line_indent) {
     }
 
     /** Sets {@link #heredoc_indent}. Usually used to reset the indent to 0 in the parser after we've finished parsing a
-     * heredoc ({@link RubyParser#tSTRING_END} or {@link RubyParser#tSTRING_DEND} has been seen). */
+     * heredoc ({@link RubyParser#tSTRING_END} has been seen). */
     public void setHeredocIndent(int heredoc_indent) {
         this.heredoc_indent = heredoc_indent;
     }
@@ -3555,33 +3555,45 @@ public void tokaddmbc(int codepoint, RopeBuilder buffer) {
         buffer.append(bytes);
     }
 
-    /** Updates {@link #heredoc_line_indent} and {@link #heredoc_indent} based on the character {@code c} read on the
-     * current line. If the character is whitespace, increments {@link #heredoc_line_indent} with its width and return
-     * true. If not, sets {@link #heredoc_indent} to {@link #heredoc_line_indent} if line_indent is lesser, sets
-     * {@link #heredoc_line_indent} to -1, and returns false.
+    /** Updates {@link #heredoc_line_indent} and {@link #heredoc_indent} based on the current value of these two
+     * variables and of the character {@code c} read on the current line.
+     *
+     * <p>
+     * This always returns false if {@link #heredoc_line_indent} is -1, and the only effect to to reset
+     * {@link #heredoc_line_indent} to 0 if if the character is a newline.
+     *
+     * <p>
+     * Otherwise, if the character is a space or a tab, increments {@link #heredoc_line_indent} with its width and
+     * return true. In every other case, false is returned. Refer to the source code for more details.
+     *
+     * <p>
+     * Return false without further actions for newlines.
+     *
      * <p>
-     * Further invocations of this after {@link #heredoc_line_indent} has been set to 1 will return false. If {@code c}
-     * is a newline in this case, will reset {@link #heredoc_line_indent} to 0. */
+     * Otherwise, this is the first non-whitespace character, and {@link #heredoc_indent} is set to
+     * {@link #heredoc_line_indent} if the later is smaller. {@link #heredoc_line_indent} is set to -1. */
     public boolean update_heredoc_indent(int c) {
         if (heredoc_line_indent == -1) {
             if (c == '\n') {
                 heredoc_line_indent = 0;
             }
+            return false;
         } else if (c == ' ') {
             heredoc_line_indent++;
             return true;
         } else if (c == '\t') {
             int w = (heredoc_line_indent / TAB_WIDTH) + 1;
             heredoc_line_indent = w * TAB_WIDTH;
             return true;
-        } else if (c != '\n') {
+        } else if (c == '\n') {
+            return false;
+        } else {
             if (heredoc_indent > heredoc_line_indent) {
                 heredoc_indent = heredoc_line_indent;
             }
             heredoc_line_indent = -1;
+            return false;
         }
-
-        return false;
     }
 
     public void validateFormalIdentifier(Rope identifier) {
@@ -3633,7 +3645,7 @@ public boolean was_bol() {
     }
 
     /** Indicates whether the current line matches the given marker, after stripping away leading whitespace if
-     * {@code indent} is true. */
+     * {@code indent} is true. Does not advance the input position ({@link #lex_p}). */
     boolean whole_match_p(Rope eos, boolean indent) {
         int len = eos.byteLength();
         int p = lex_pbeg;