Auto merge of #14837 - Veykril:rustc-lexer, r=Veykril

bors · bors · commit 9ce95674e8d1 · 2023-05-18T11:55:38.000Z
Support c string literals
diff --git a/crates/hir-def/src/body/pretty.rs b/crates/hir-def/src/body/pretty.rs
@@ -611,6 +611,7 @@ impl<'a> Printer<'a> {
         match literal {
             Literal::String(it) => w!(self, "{:?}", it),
             Literal::ByteString(it) => w!(self, "\"{}\"", it.escape_ascii()),
+            Literal::CString(it) => w!(self, "\"{}\\0\"", it),
             Literal::Char(it) => w!(self, "'{}'", it.escape_debug()),
             Literal::Bool(it) => w!(self, "{}", it),
             Literal::Int(i, suffix) => {
diff --git a/crates/hir-def/src/hir.rs b/crates/hir-def/src/hir.rs
@@ -85,6 +85,7 @@ impl fmt::Display for FloatTypeWrapper {
 pub enum Literal {
     String(Box<str>),
     ByteString(Box<[u8]>),
+    CString(Box<str>),
     Char(char),
     Bool(bool),
     Int(i128, Option<BuiltinInt>),
@@ -135,6 +136,10 @@ impl From<ast::LiteralKind> for Literal {
                 let text = s.value().map(Box::from).unwrap_or_else(Default::default);
                 Literal::String(text)
             }
+            LiteralKind::CString(s) => {
+                let text = s.value().map(Box::from).unwrap_or_else(Default::default);
+                Literal::CString(text)
+            }
             LiteralKind::Byte(b) => {
                 Literal::Uint(b.value().unwrap_or_default() as u128, Some(BuiltinUint::U8))
             }
diff --git a/crates/hir-def/src/lang_item.rs b/crates/hir-def/src/lang_item.rs
diff --git a/crates/hir-ty/src/consteval/tests.rs b/crates/hir-ty/src/consteval/tests.rs
@@ -1883,6 +1883,38 @@ fn byte_string() {
     );
 }
 
+#[test]
+fn c_string() {
+    check_number(
+        r#"
+//- minicore: index, slice
+#[lang = "CStr"]
+pub struct CStr {
+    inner: [u8]
+}
+const GOAL: u8 = {
+    let a = c"hello";
+    a.inner[0]
+};
+    "#,
+        104,
+    );
+    check_number(
+        r#"
+//- minicore: index, slice
+#[lang = "CStr"]
+pub struct CStr {
+    inner: [u8]
+}
+const GOAL: u8 = {
+    let a = c"hello";
+    a.inner[6]
+};
+    "#,
+        0,
+    );
+}
+
 #[test]
 fn consts() {
     check_number(
diff --git a/crates/hir-ty/src/infer/expr.rs b/crates/hir-ty/src/infer/expr.rs
@@ -13,7 +13,7 @@ use hir_def::{
     hir::{
         ArithOp, Array, BinaryOp, ClosureKind, Expr, ExprId, LabelId, Literal, Statement, UnaryOp,
     },
-    lang_item::LangItem,
+    lang_item::{LangItem, LangItemTarget},
     path::{GenericArg, GenericArgs},
     BlockId, ConstParamId, FieldId, ItemContainerId, Lookup,
 };
@@ -832,6 +832,20 @@ impl<'a> InferenceContext<'a> {
                     let array_type = TyKind::Array(byte_type, len).intern(Interner);
                     TyKind::Ref(Mutability::Not, static_lifetime(), array_type).intern(Interner)
                 }
+                Literal::CString(..) => TyKind::Ref(
+                    Mutability::Not,
+                    static_lifetime(),
+                    self.resolve_lang_item(LangItem::CStr)
+                        .and_then(LangItemTarget::as_struct)
+                        .map_or_else(
+                            || self.err_ty(),
+                            |strukt| {
+                                TyKind::Adt(AdtId(strukt.into()), Substitution::empty(Interner))
+                                    .intern(Interner)
+                            },
+                        ),
+                )
+                .intern(Interner),
                 Literal::Char(..) => TyKind::Scalar(Scalar::Char).intern(Interner),
                 Literal::Int(_v, ty) => match ty {
                     Some(int_ty) => {
diff --git a/crates/hir-ty/src/infer/pat.rs b/crates/hir-ty/src/infer/pat.rs
@@ -428,9 +428,10 @@ fn is_non_ref_pat(body: &hir_def::body::Body, pat: PatId) -> bool {
         // FIXME: ConstBlock/Path/Lit might actually evaluate to ref, but inference is unimplemented.
         Pat::Path(..) => true,
         Pat::ConstBlock(..) => true,
-        Pat::Lit(expr) => {
-            !matches!(body[*expr], Expr::Literal(Literal::String(..) | Literal::ByteString(..)))
-        }
+        Pat::Lit(expr) => !matches!(
+            body[*expr],
+            Expr::Literal(Literal::String(..) | Literal::CString(..) | Literal::ByteString(..))
+        ),
         Pat::Wild | Pat::Bind { .. } | Pat::Ref { .. } | Pat::Box { .. } | Pat::Missing => false,
     }
 }
diff --git a/crates/hir-ty/src/mir/lower.rs b/crates/hir-ty/src/mir/lower.rs
@@ -1133,15 +1133,26 @@ impl<'ctx> MirLowerCtx<'ctx> {
         let bytes = match l {
             hir_def::hir::Literal::String(b) => {
                 let b = b.as_bytes();
-                let mut data = vec![];
+                let mut data = Vec::with_capacity(mem::size_of::<usize>() * 2);
                 data.extend(0usize.to_le_bytes());
                 data.extend(b.len().to_le_bytes());
                 let mut mm = MemoryMap::default();
                 mm.insert(0, b.to_vec());
                 return Ok(Operand::from_concrete_const(data, mm, ty));
             }
+            hir_def::hir::Literal::CString(b) => {
+                let b = b.as_bytes();
+                let bytes = b.iter().copied().chain(iter::once(0)).collect::<Vec<_>>();
+
+                let mut data = Vec::with_capacity(mem::size_of::<usize>() * 2);
+                data.extend(0usize.to_le_bytes());
+                data.extend(bytes.len().to_le_bytes());
+                let mut mm = MemoryMap::default();
+                mm.insert(0, bytes);
+                return Ok(Operand::from_concrete_const(data, mm, ty));
+            }
             hir_def::hir::Literal::ByteString(b) => {
-                let mut data = vec![];
+                let mut data = Vec::with_capacity(mem::size_of::<usize>() * 2);
                 data.extend(0usize.to_le_bytes());
                 data.extend(b.len().to_le_bytes());
                 let mut mm = MemoryMap::default();
diff --git a/crates/hir-ty/src/tests/simple.rs b/crates/hir-ty/src/tests/simple.rs
@@ -3572,3 +3572,18 @@ fn main() {
 "#,
     );
 }
+
+#[test]
+fn cstring_literals() {
+    check_types(
+        r#"
+#[lang = "CStr"]
+pub struct CStr;
+
+fn main() {
+    c"ello";
+  //^^^^^^^ &CStr
+}
+"#,
+    );
+}
diff --git a/crates/ide-assists/src/handlers/raw_string.rs b/crates/ide-assists/src/handlers/raw_string.rs
@@ -20,6 +20,7 @@ use crate::{utils::required_hashes, AssistContext, AssistId, AssistKind, Assists
 // }
 // ```
 pub(crate) fn make_raw_string(acc: &mut Assists, ctx: &AssistContext<'_>) -> Option<()> {
+    // FIXME: This should support byte and c strings as well.
     let token = ctx.find_token_at_offset::<ast::String>()?;
     if token.is_raw() {
         return None;
diff --git a/crates/ide/src/extend_selection.rs b/crates/ide/src/extend_selection.rs
@@ -39,7 +39,7 @@ fn try_extend_selection(
 ) -> Option<TextRange> {
     let range = frange.range;
 
-    let string_kinds = [COMMENT, STRING, BYTE_STRING];
+    let string_kinds = [COMMENT, STRING, BYTE_STRING, C_STRING];
     let list_kinds = [
         RECORD_PAT_FIELD_LIST,
         MATCH_ARM_LIST,
diff --git a/crates/ide/src/syntax_highlighting.rs b/crates/ide/src/syntax_highlighting.rs
@@ -16,7 +16,10 @@ mod tests;
 use hir::{Name, Semantics};
 use ide_db::{FxHashMap, RootDatabase, SymbolKind};
 use syntax::{
-    ast, AstNode, AstToken, NodeOrToken, SyntaxKind::*, SyntaxNode, TextRange, WalkEvent, T,
+    ast::{self, IsString},
+    AstNode, AstToken, NodeOrToken,
+    SyntaxKind::*,
+    SyntaxNode, TextRange, WalkEvent, T,
 };
 
 use crate::{
@@ -440,7 +443,17 @@ fn traverse(
                 && ast::ByteString::can_cast(descended_token.kind())
             {
                 if let Some(byte_string) = ast::ByteString::cast(token) {
-                    highlight_escape_string(hl, &byte_string, range.start());
+                    if !byte_string.is_raw() {
+                        highlight_escape_string(hl, &byte_string, range.start());
+                    }
+                }
+            } else if ast::CString::can_cast(token.kind())
+                && ast::CString::can_cast(descended_token.kind())
+            {
+                if let Some(c_string) = ast::CString::cast(token) {
+                    if !c_string.is_raw() {
+                        highlight_escape_string(hl, &c_string, range.start());
+                    }
                 }
             } else if ast::Char::can_cast(token.kind())
                 && ast::Char::can_cast(descended_token.kind())
diff --git a/crates/ide/src/syntax_highlighting/highlight.rs b/crates/ide/src/syntax_highlighting/highlight.rs
@@ -26,7 +26,7 @@ pub(super) fn token(sema: &Semantics<'_, RootDatabase>, token: SyntaxToken) -> O
     }
 
     let highlight: Highlight = match token.kind() {
-        STRING | BYTE_STRING => HlTag::StringLiteral.into(),
+        STRING | BYTE_STRING | C_STRING => HlTag::StringLiteral.into(),
         INT_NUMBER if token.parent_ancestors().nth(1).map(|it| it.kind()) == Some(FIELD_EXPR) => {
             SymbolKind::Field.into()
         }
diff --git a/crates/ide/src/syntax_tree.rs b/crates/ide/src/syntax_tree.rs
@@ -1,5 +1,7 @@
-use ide_db::base_db::{FileId, SourceDatabase};
-use ide_db::RootDatabase;
+use ide_db::{
+    base_db::{FileId, SourceDatabase},
+    RootDatabase,
+};
 use syntax::{
     AstNode, NodeOrToken, SourceFile, SyntaxKind::STRING, SyntaxToken, TextRange, TextSize,
 };
diff --git a/crates/parser/src/grammar/expressions/atom.rs b/crates/parser/src/grammar/expressions/atom.rs
@@ -12,6 +12,8 @@ use super::*;
 //     let _ = r"d";
 //     let _ = b"e";
 //     let _ = br"f";
+//     let _ = c"g";
+//     let _ = cr"h";
 // }
 pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
     T![true],
@@ -22,6 +24,7 @@ pub(crate) const LITERAL_FIRST: TokenSet = TokenSet::new(&[
     CHAR,
     STRING,
     BYTE_STRING,
+    C_STRING,
 ]);
 
 pub(crate) fn literal(p: &mut Parser<'_>) -> Option<CompletedMarker> {
diff --git a/crates/parser/src/grammar/generic_args.rs b/crates/parser/src/grammar/generic_args.rs
@@ -28,6 +28,7 @@ const GENERIC_ARG_FIRST: TokenSet = TokenSet::new(&[
     BYTE,
     STRING,
     BYTE_STRING,
+    C_STRING,
 ])
 .union(types::TYPE_FIRST);
 
diff --git a/crates/parser/src/lexed_str.rs b/crates/parser/src/lexed_str.rs
@@ -277,7 +277,7 @@ impl<'a> Converter<'a> {
                 if !terminated {
                     err = "Missing trailing `\"` symbol to terminate the string literal";
                 }
-                STRING
+                C_STRING
             }
             rustc_lexer::LiteralKind::RawStr { n_hashes } => {
                 if n_hashes.is_none() {
@@ -295,7 +295,7 @@ impl<'a> Converter<'a> {
                 if n_hashes.is_none() {
                     err = "Invalid raw string literal";
                 }
-                STRING
+                C_STRING
             }
         };
 
diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rast
@@ -131,6 +131,30 @@ SOURCE_FILE
           LITERAL
             BYTE_STRING "br\"f\""
           SEMICOLON ";"
+        WHITESPACE "\n    "
+        LET_STMT
+          LET_KW "let"
+          WHITESPACE " "
+          WILDCARD_PAT
+            UNDERSCORE "_"
+          WHITESPACE " "
+          EQ "="
+          WHITESPACE " "
+          LITERAL
+            C_STRING "c\"g\""
+          SEMICOLON ";"
+        WHITESPACE "\n    "
+        LET_STMT
+          LET_KW "let"
+          WHITESPACE " "
+          WILDCARD_PAT
+            UNDERSCORE "_"
+          WHITESPACE " "
+          EQ "="
+          WHITESPACE " "
+          LITERAL
+            C_STRING "cr\"h\""
+          SEMICOLON ";"
         WHITESPACE "\n"
         R_CURLY "}"
   WHITESPACE "\n"
diff --git a/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rs b/crates/parser/test_data/parser/inline/ok/0085_expr_literals.rs
@@ -9,4 +9,6 @@ fn foo() {
     let _ = r"d";
     let _ = b"e";
     let _ = br"f";
+    let _ = c"g";
+    let _ = cr"h";
 }
diff --git a/crates/syntax/src/ast/expr_ext.rs b/crates/syntax/src/ast/expr_ext.rs
@@ -288,6 +288,7 @@ impl ast::ArrayExpr {
 pub enum LiteralKind {
     String(ast::String),
     ByteString(ast::ByteString),
+    CString(ast::CString),
     IntNumber(ast::IntNumber),
     FloatNumber(ast::FloatNumber),
     Char(ast::Char),
@@ -319,6 +320,9 @@ impl ast::Literal {
         if let Some(t) = ast::ByteString::cast(token.clone()) {
             return LiteralKind::ByteString(t);
         }
+        if let Some(t) = ast::CString::cast(token.clone()) {
+            return LiteralKind::CString(t);
+        }
         if let Some(t) = ast::Char::cast(token.clone()) {
             return LiteralKind::Char(t);
         }
diff --git a/crates/syntax/src/ast/generated/tokens.rs b/crates/syntax/src/ast/generated/tokens.rs
@@ -90,6 +90,27 @@ impl AstToken for ByteString {
     fn syntax(&self) -> &SyntaxToken { &self.syntax }
 }
 
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+pub struct CString {
+    pub(crate) syntax: SyntaxToken,
+}
+impl std::fmt::Display for CString {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        std::fmt::Display::fmt(&self.syntax, f)
+    }
+}
+impl AstToken for CString {
+    fn can_cast(kind: SyntaxKind) -> bool { kind == C_STRING }
+    fn cast(syntax: SyntaxToken) -> Option<Self> {
+        if Self::can_cast(syntax.kind()) {
+            Some(Self { syntax })
+        } else {
+            None
+        }
+    }
+    fn syntax(&self) -> &SyntaxToken { &self.syntax }
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, Hash)]
 pub struct IntNumber {
     pub(crate) syntax: SyntaxToken,
diff --git a/crates/syntax/src/ast/token_ext.rs b/crates/syntax/src/ast/token_ext.rs
diff --git a/crates/syntax/src/parsing/reparsing.rs b/crates/syntax/src/parsing/reparsing.rs
diff --git a/crates/syntax/src/tests/sourcegen_ast.rs b/crates/syntax/src/tests/sourcegen_ast.rs
diff --git a/crates/syntax/src/validation.rs b/crates/syntax/src/validation.rs

Original file line number	Diff line number	Diff line change
`@@ -428,9 +428,10 @@ fn is_non_ref_pat(body: &hir_def::body::Body, pat: PatId) -> bool {`
`428`	`428`	`// FIXME: ConstBlock/Path/Lit might actually evaluate to ref, but inference is unimplemented.`
`429`	`429`	`Pat::Path(..) => true,`
`430`	`430`	`Pat::ConstBlock(..) => true,`
`431`		`- Pat::Lit(expr) => {`
`432`		`- !matches!(body[*expr], Expr::Literal(Literal::String(..) \| Literal::ByteString(..)))`
`433`		`- }`
	`431`	`+ Pat::Lit(expr) => !matches!(`
	`432`	`+ body[*expr],`
	`433`	`+ Expr::Literal(Literal::String(..) \| Literal::CString(..) \| Literal::ByteString(..))`
	`434`	`+ ),`
`434`	`435`	`Pat::Wild \| Pat::Bind { .. } \| Pat::Ref { .. } \| Pat::Box { .. } \| Pat::Missing => false,`
`435`	`436`	`}`
`436`	`437`	`}`