Move check for full parse into token_stream parser

Author: dtolnay

src/fallback.rs

@@ -1,4 +1,4 @@
-use crate::parse::{token_stream, Cursor};
+use crate::parse::{self, Cursor};
 use crate::{Delimiter, Spacing, TokenTree};
 #[cfg(span_locations)]
 use std::cell::RefCell;
@@ -139,12 +139,7 @@ impl FromStr for TokenStream {
         // Create a dummy file & add it to the source map
         let cursor = get_cursor(src);
 
-        let (rest, tokens) = token_stream(cursor)?;
-        if rest.is_empty() {
-            Ok(tokens)
-        } else {
-            Err(LexError)
-        }
+        parse::token_stream(cursor)
     }
 }
 

src/parse.rs

@@ -26,7 +26,7 @@ impl<'a> Cursor<'a> {
         self.rest.starts_with(s)
     }
 
-    pub(crate) fn is_empty(&self) -> bool {
+    fn is_empty(&self) -> bool {
         self.rest.is_empty()
     }
 
@@ -148,7 +148,7 @@ fn word_break(input: Cursor) -> Result<Cursor, LexError> {
     }
 }
 
-pub(crate) fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
+pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
     let mut trees = Vec::new();
     let mut stack = Vec::new();
 
@@ -217,8 +217,8 @@ pub(crate) fn token_stream(mut input: Cursor) -> PResult<TokenStream> {
         }
     }
 
-    if stack.is_empty() {
-        Ok((input, TokenStream { inner: trees }))
+    if stack.is_empty() && input.is_empty() {
+        Ok(TokenStream { inner: trees })
     } else {
         Err(LexError)
     }