tests: add test that roughly ensures that our lint messages conform with the diagnostics convention of the rustc dev guide

lint message should not start with uppercase letters
lint messages should not have punctuation at the end of the last line

https://rustc-dev-guide.rust-lang.org/diagnostics.html#diagnostic-structure

The test reads through all the .stderr files in the testsuit and checks lint messages that start with "help: ", "error: " etc.
There is also an exception list for special messages that are deemed acceptable.

changelog: make sure lint messages conform with the rustc dev guide and add test

Author: matthiaskrgr

Cargo.toml

@@ -42,6 +42,7 @@ tester = "0.9"
 clippy-mini-macro-test = { version = "0.2", path = "mini-macro" }
 serde = { version = "1.0", features = ["derive"] }
 derive-new = "0.5"
+regex = "1.4"
 
 # A noop dependency that changes in the Rust repository, it's a bit of a hack.
 # See the `src/tools/rustc-workspace-hack/README.md` file in `rust-lang/rust`

tests/lint_message_convention.rs

@@ -0,0 +1,102 @@
+use std::path::PathBuf;
+
+use regex::RegexSet;
+
+#[derive(Debug)]
+struct Message {
+    path: PathBuf,
+    bad_lines: Vec<String>,
+}
+
+impl Message {
+    fn new(path: PathBuf) -> Self {
+        let content: String = std::fs::read_to_string(&path).unwrap();
+        // we don't want the first letter after "error: ", "help: " ... to be capitalized
+        // also no puncutation (except for "?" ?) at the end of a line
+        let regex_set: RegexSet = RegexSet::new(&[
+            r"error: [A-Z]",
+            r"help: [A-Z]",
+            r"warning: [A-Z]",
+            r"note: [A-Z]",
+            r"try this: [A-Z]",
+            r"error: .*[.!]$",
+            r"help: .*[.!]$",
+            r"warning: .*[.!]$",
+            r"note: .*[.!]$",
+            r"try this: .*[.!]$",
+        ])
+        .unwrap();
+
+        // sometimes the first character is capitalized and it is legal (like in "Iterator...") or
+        // we want to ask a question ending in "?"
+        let exceptions_set: RegexSet = RegexSet::new(&[
+            r".*error: I see you're using a LinkedList! Perhaps you meant some other data structure?",
+            r".*C-like enum variant discriminant is not portable to 32-bit targets",
+            r".*Iterator::step_by(0) will panic at runtime",
+            r".*did you mean `unix`?",
+            r".*the arguments may be inverted...",
+            r".*Intel x86 assembly syntax used",
+            r".*AT&T x86 assembly syntax used",
+            r".*remove .* the return type...",
+            r"note: Clippy version: .*",
+        ])
+        .unwrap();
+
+        let bad_lines = content
+            .lines()
+            .filter(|line| regex_set.matches(line).matched_any())
+            // ignore exceptions
+            .filter(|line| !exceptions_set.matches(line).matched_any())
+            .map(|s| s.to_owned())
+            .collect::<Vec<String>>();
+
+        Message { path, bad_lines }
+    }
+}
+
+#[test]
+fn lint_message_convention() {
+    // make sure that lint messages:
+    // * are not capitalized
+    // * don't have puncuation at the end of the last sentence
+
+    // these directories have interesting tests
+    let test_dirs = ["ui", "ui-cargo", "ui-internal", "ui-toml"]
+        .iter()
+        .map(PathBuf::from)
+        .map(|p| {
+            let base = PathBuf::from("tests");
+            base.join(p)
+        });
+
+    // gather all .stderr files
+    let tests = test_dirs
+        .map(|dir| {
+            std::fs::read_dir(dir)
+                .expect("failed to read dir")
+                .map(|direntry| direntry.unwrap().path())
+        })
+        .flatten()
+        .filter(|file| matches!(file.extension().map(|s| s.to_str()), Some(Some("stderr"))));
+
+    // get all files that have any "bad lines" in them
+    let bad_tests: Vec<Message> = tests
+        .map(|path| Message::new(path))
+        .filter(|message| !message.bad_lines.is_empty())
+        .collect();
+
+    bad_tests.iter().for_each(|message| {
+        eprintln!(
+            "error: the test '{}' contained the following nonconforming lines :",
+            message.path.display()
+        );
+        message.bad_lines.iter().for_each(|line| eprintln!("{}", line));
+        eprintln!("\n\n");
+    });
+
+    eprintln!("\n\n\nLint message should not start with a capital letter and should not have punctuation at the end of the message unless multiple sentences are needed.");
+    eprintln!("Check out the rustc-dev-guide for more information:");
+    eprintln!("https://rustc-dev-guide.rust-lang.org/diagnostics.html#diagnostic-structure");
+
+    assert!(bad_tests.is_empty());
+}