Merge pull request #283 from dtolnay/lexspan

Add span to lex errors

Author: dtolnay

src/fallback.rs

@@ -35,7 +35,15 @@ pub(crate) struct TokenStream {
 }
 
 #[derive(Debug)]
-pub(crate) struct LexError;
+pub(crate) struct LexError {
+    pub(crate) span: Span,
+}
+
+impl LexError {
+    pub(crate) fn span(&self) -> Span {
+        self.span
+    }
+}
 
 impl TokenStream {
     pub fn new() -> TokenStream {

src/lib.rs

@@ -250,6 +250,12 @@ impl Debug for TokenStream {
     }
 }
 
+impl LexError {
+    pub fn span(&self) -> Span {
+        Span::_new(self.inner.span())
+    }
+}
+
 impl Debug for LexError {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         Debug::fmt(&self.inner, f)

src/parse.rs

@@ -167,8 +167,17 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
 
         let first = match input.bytes().next() {
             Some(first) => first,
-            None if stack.is_empty() => return Ok(TokenStream { inner: trees }),
-            None => return Err(LexError),
+            None => match stack.last() {
+                None => return Ok(TokenStream { inner: trees }),
+                #[cfg(span_locations)]
+                Some((lo, _frame)) => {
+                    return Err(LexError {
+                        span: Span { lo: *lo, hi: *lo },
+                    })
+                }
+                #[cfg(not(span_locations))]
+                Some(_frame) => return Err(LexError { span: Span {} }),
+            },
         };
 
         if let Some(open_delimiter) = match first {
@@ -189,14 +198,17 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
             b'}' => Some(Delimiter::Brace),
             _ => None,
         } {
-            input = input.advance(1);
-            let frame = stack.pop().ok_or(LexError)?;
+            let frame = match stack.pop() {
+                Some(frame) => frame,
+                None => return Err(lex_error(input)),
+            };
             #[cfg(span_locations)]
             let (lo, frame) = frame;
             let (open_delimiter, outer) = frame;
             if open_delimiter != close_delimiter {
-                return Err(LexError);
+                return Err(lex_error(input));
             }
+            input = input.advance(1);
             let mut g = Group::new(open_delimiter, TokenStream { inner: trees });
             g.set_span(Span {
                 #[cfg(span_locations)]
@@ -209,7 +221,7 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
         } else {
             let (rest, mut tt) = match leaf_token(input) {
                 Ok((rest, tt)) => (rest, tt),
-                Err(Reject) => return Err(LexError),
+                Err(Reject) => return Err(lex_error(input)),
             };
             tt.set_span(crate::Span::_new_stable(Span {
                 #[cfg(span_locations)]
@@ -223,6 +235,19 @@ pub(crate) fn token_stream(mut input: Cursor) -> Result<TokenStream, LexError> {
     }
 }
 
+fn lex_error(cursor: Cursor) -> LexError {
+    #[cfg(not(span_locations))]
+    let _ = cursor;
+    LexError {
+        span: Span {
+            #[cfg(span_locations)]
+            lo: cursor.off,
+            #[cfg(span_locations)]
+            hi: cursor.off,
+        },
+    }
+}
+
 fn leaf_token(input: Cursor) -> PResult<TokenTree> {
     if let Ok((input, l)) = literal(input) {
         // must be parsed before ident

src/wrapper.rs

@@ -107,8 +107,12 @@ impl FromStr for TokenStream {
 
 // Work around https://github.com/rust-lang/rust/issues/58736.
 fn proc_macro_parse(src: &str) -> Result<proc_macro::TokenStream, LexError> {
-    panic::catch_unwind(|| src.parse().map_err(LexError::Compiler))
-        .unwrap_or(Err(LexError::Fallback(fallback::LexError)))
+    let result = panic::catch_unwind(|| src.parse().map_err(LexError::Compiler));
+    result.unwrap_or_else(|_| {
+        Err(LexError::Fallback(fallback::LexError {
+            span: fallback::Span::call_site(),
+        }))
+    })
 }
 
 impl Display for TokenStream {
@@ -243,6 +247,15 @@ impl Debug for TokenStream {
     }
 }
 
+impl LexError {
+    pub(crate) fn span(&self) -> Span {
+        match self {
+            LexError::Compiler(_) => Span::call_site(),
+            LexError::Fallback(e) => Span::Fallback(e.span()),
+        }
+    }
+}
+
 impl From<proc_macro::LexError> for LexError {
     fn from(e: proc_macro::LexError) -> LexError {
         LexError::Compiler(e)
@@ -270,7 +283,12 @@ impl Display for LexError {
             #[cfg(lexerror_display)]
             LexError::Compiler(e) => Display::fmt(e, f),
             #[cfg(not(lexerror_display))]
-            LexError::Compiler(_e) => Display::fmt(&fallback::LexError, f),
+            LexError::Compiler(_e) => Display::fmt(
+                &fallback::LexError {
+                    span: fallback::Span::call_site(),
+                },
+                f,
+            ),
             LexError::Fallback(e) => Display::fmt(e, f),
         }
     }