|
2 | 2 |
|
3 | 3 | use parser::{Token, TokenSource};
|
4 | 4 | use std::cell::{Cell, Ref, RefCell};
|
5 |
| -use syntax::{lex_single_syntax_kind, SmolStr, SyntaxKind, SyntaxKind::*, T}; |
| 5 | +use syntax::{tokenize, SmolStr, SyntaxKind, SyntaxKind::*, T}; |
6 | 6 | use tt::buffer::{Cursor, TokenBuffer};
|
7 | 7 |
|
8 | 8 | #[derive(Debug, Clone, Eq, PartialEq)]
|
@@ -155,10 +155,17 @@ fn convert_delim(d: Option<tt::DelimiterKind>, closing: bool) -> TtToken {
|
155 | 155 | }
|
156 | 156 |
|
157 | 157 | fn convert_literal(l: &tt::Literal) -> TtToken {
|
158 |
| - let kind = lex_single_syntax_kind(&l.text) |
159 |
| - .map(|(kind, _error)| kind) |
160 |
| - .filter(|kind| kind.is_literal()) |
161 |
| - .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); |
| 158 | + let mut kinds = tokenize(&l.text).0.into_iter().map(|token| token.kind); |
| 159 | + |
| 160 | + let kind = match kinds.next() { |
| 161 | + Some(kind) if kind.is_literal() => Some(kind), |
| 162 | + Some(SyntaxKind::MINUS) => match kinds.next() { |
| 163 | + Some(kind) if kind.is_literal() => Some(kind), |
| 164 | + _ => None, |
| 165 | + }, |
| 166 | + _ => None, |
| 167 | + } |
| 168 | + .unwrap_or_else(|| panic!("Fail to convert given literal {:#?}", &l)); |
162 | 169 |
|
163 | 170 | TtToken { kind, is_joint_to_next: false, text: l.text.clone() }
|
164 | 171 | }
|
|
0 commit comments