Merge #7918

7918: Generalize file ensuring infrastructure r=matklad a=matklad

bors r+
🤖

Co-authored-by: Aleksey Kladov <aleksey.kladov@gmail.com>

Author: bors[bot]

crates/rust-analyzer/src/config.rs

@@ -859,12 +859,12 @@ fn manual(fields: &[(&'static str, &'static str, &[&str], &str)]) -> String {
 mod tests {
     use std::fs;
 
-    use test_utils::project_dir;
+    use test_utils::{ensure_file_contents, project_root};
 
     use super::*;
 
     #[test]
-    fn schema_in_sync_with_package_json() {
+    fn generate_package_json_config() {
         let s = Config::json_schema();
         let schema = format!("{:#}", s);
         let mut schema = schema
@@ -877,27 +877,26 @@ mod tests {
             .to_string();
         schema.push_str(",\n");
 
-        let package_json_path = project_dir().join("editors/code/package.json");
+        let package_json_path = project_root().join("editors/code/package.json");
         let mut package_json = fs::read_to_string(&package_json_path).unwrap();
 
         let start_marker = "                \"$generated-start\": false,\n";
         let end_marker = "                \"$generated-end\": false\n";
 
         let start = package_json.find(start_marker).unwrap() + start_marker.len();
         let end = package_json.find(end_marker).unwrap();
+
         let p = remove_ws(&package_json[start..end]);
         let s = remove_ws(&schema);
-
         if !p.contains(&s) {
             package_json.replace_range(start..end, &schema);
-            fs::write(&package_json_path, &mut package_json).unwrap();
-            panic!("new config, updating package.json")
+            ensure_file_contents(&package_json_path, &package_json)
         }
     }
 
     #[test]
-    fn schema_in_sync_with_docs() {
-        let docs_path = project_dir().join("docs/user/generated_config.adoc");
+    fn generate_config_documentation() {
+        let docs_path = project_root().join("docs/user/generated_config.adoc");
         let current = fs::read_to_string(&docs_path).unwrap();
         let expected = ConfigData::manual();
 

crates/syntax/src/tests.rs

@@ -7,7 +7,7 @@ use std::{
 use ast::NameOwner;
 use expect_test::expect_file;
 use rayon::prelude::*;
-use test_utils::{bench, bench_fixture, project_dir, skip_slow_tests};
+use test_utils::{bench, bench_fixture, project_root, skip_slow_tests};
 
 use crate::{ast, fuzz, tokenize, AstNode, SourceFile, SyntaxError, TextRange, TextSize, Token};
 
@@ -153,7 +153,7 @@ fn reparse_fuzz_tests() {
 /// Test that Rust-analyzer can parse and validate the rust-analyzer
 #[test]
 fn self_hosting_parsing() {
-    let dir = project_dir().join("crates");
+    let dir = project_root().join("crates");
     let files = walkdir::WalkDir::new(dir)
         .into_iter()
         .filter_entry(|entry| {
@@ -193,7 +193,7 @@ fn self_hosting_parsing() {
 }
 
 fn test_data_dir() -> PathBuf {
-    project_dir().join("crates/syntax/test_data")
+    project_root().join("crates/syntax/test_data")
 }
 
 fn assert_errors_are_present(errors: &[SyntaxError], path: &Path) {

crates/test_utils/src/bench_fixture.rs

@@ -4,7 +4,7 @@ use std::fs;
 
 use stdx::format_to;
 
-use crate::project_dir;
+use crate::project_root;
 
 pub fn big_struct() -> String {
     let n = 1_000;
@@ -32,11 +32,11 @@ struct S{} {{
 }
 
 pub fn glorious_old_parser() -> String {
-    let path = project_dir().join("bench_data/glorious_old_parser");
+    let path = project_root().join("bench_data/glorious_old_parser");
     fs::read_to_string(&path).unwrap()
 }
 
 pub fn numerous_macro_rules() -> String {
-    let path = project_dir().join("bench_data/numerous_macro_rules");
+    let path = project_root().join("bench_data/numerous_macro_rules");
     fs::read_to_string(&path).unwrap()
 }

crates/test_utils/src/lib.rs

@@ -14,11 +14,11 @@ mod fixture;
 use std::{
     convert::{TryFrom, TryInto},
     env, fs,
-    path::PathBuf,
+    path::{Path, PathBuf},
 };
 
 use profile::StopWatch;
-use stdx::lines_with_ends;
+use stdx::{is_ci, lines_with_ends};
 use text_size::{TextRange, TextSize};
 
 pub use dissimilar::diff as __diff;
@@ -288,14 +288,14 @@ pub fn skip_slow_tests() -> bool {
     if should_skip {
         eprintln!("ignoring slow test")
     } else {
-        let path = project_dir().join("./target/.slow_tests_cookie");
+        let path = project_root().join("./target/.slow_tests_cookie");
         fs::write(&path, ".").unwrap();
     }
     should_skip
 }
 
 /// Returns the path to the root directory of `rust-analyzer` project.
-pub fn project_dir() -> PathBuf {
+pub fn project_root() -> PathBuf {
     let dir = env!("CARGO_MANIFEST_DIR");
     PathBuf::from(dir).parent().unwrap().parent().unwrap().to_owned()
 }
@@ -353,3 +353,39 @@ pub fn bench(label: &'static str) -> impl Drop {
 
     Bencher { sw: StopWatch::start(), label }
 }
+
+/// Checks that the `file` has the specified `contents`. If that is not the
+/// case, updates the file and then fails the test.
+pub fn ensure_file_contents(file: &Path, contents: &str) {
+    if let Err(()) = try_ensure_file_contents(file, contents) {
+        panic!("Some files were not up-to-date");
+    }
+}
+
+/// Checks that the `file` has the specified `contents`. If that is not the
+/// case, updates the file and return an Error.
+pub fn try_ensure_file_contents(file: &Path, contents: &str) -> Result<(), ()> {
+    match std::fs::read_to_string(file) {
+        Ok(old_contents) if normalize_newlines(&old_contents) == normalize_newlines(contents) => {
+            return Ok(())
+        }
+        _ => (),
+    }
+    let display_path = file.strip_prefix(&project_root()).unwrap_or(file);
+    eprintln!(
+        "\n\x1b[31;1merror\x1b[0m: {} was not up-to-date, updating\n",
+        display_path.display()
+    );
+    if is_ci() {
+        eprintln!("    NOTE: run `cargo test` locally and commit the updated files\n");
+    }
+    if let Some(parent) = file.parent() {
+        let _ = std::fs::create_dir_all(parent);
+    }
+    std::fs::write(file, contents).unwrap();
+    Err(())
+}
+
+fn normalize_newlines(s: &str) -> String {
+    s.replace("\r\n", "\n")
+}

docs/dev/architecture.md

@@ -308,9 +308,8 @@ This sections talks about the things which are everywhere and nowhere in particu
 ### Code generation
 
 Some of the components of this repository are generated through automatic processes.
-`cargo xtask codegen` runs all generation tasks.
+Generated code is updated automatically on `cargo test`.
 Generated code is generally committed to the git repository.
-There are tests to check that the generated code is fresh.
 
 In particular, we generate:
 

xtask/src/codegen.rs

@@ -7,68 +7,66 @@
 
 mod gen_syntax;
 mod gen_parser_tests;
+mod gen_lint_completions;
 mod gen_assists_docs;
 mod gen_feature_docs;
-mod gen_lint_completions;
 mod gen_diagnostic_docs;
 
 use std::{
     fmt, mem,
     path::{Path, PathBuf},
 };
-use xshell::{cmd, pushenv, read_file, write_file};
+use xshell::{cmd, pushenv};
 
-use crate::{ensure_rustfmt, flags, project_root, Result};
+use crate::{ensure_rustfmt, project_root, Result};
 
 pub(crate) use self::{
-    gen_assists_docs::{generate_assists_docs, generate_assists_tests},
-    gen_diagnostic_docs::generate_diagnostic_docs,
-    gen_feature_docs::generate_feature_docs,
-    gen_lint_completions::generate_lint_completions,
-    gen_parser_tests::generate_parser_tests,
-    gen_syntax::generate_syntax,
+    gen_assists_docs::generate_assists_tests, gen_lint_completions::generate_lint_completions,
+    gen_parser_tests::generate_parser_tests, gen_syntax::generate_syntax,
 };
 
-#[derive(Debug, PartialEq, Eq, Clone, Copy)]
-pub(crate) enum Mode {
-    Overwrite,
-    Verify,
+pub(crate) fn docs() -> Result<()> {
+    // We don't commit docs to the repo, so we can just overwrite them.
+    gen_assists_docs::generate_assists_docs()?;
+    gen_feature_docs::generate_feature_docs()?;
+    gen_diagnostic_docs::generate_diagnostic_docs()?;
+    Ok(())
 }
 
-impl flags::Codegen {
-    pub(crate) fn run(self) -> Result<()> {
-        if self.features {
-            generate_lint_completions(Mode::Overwrite)?;
-        }
-        generate_syntax(Mode::Overwrite)?;
-        generate_parser_tests(Mode::Overwrite)?;
-        generate_assists_tests(Mode::Overwrite)?;
-        generate_assists_docs(Mode::Overwrite)?;
-        generate_feature_docs(Mode::Overwrite)?;
-        generate_diagnostic_docs(Mode::Overwrite)?;
-        Ok(())
-    }
+#[allow(unused)]
+fn used() {
+    generate_parser_tests();
+    generate_assists_tests();
+    generate_syntax();
+    generate_lint_completions();
 }
 
-/// A helper to update file on disk if it has changed.
-/// With verify = false,
-fn update(path: &Path, contents: &str, mode: Mode) -> Result<()> {
-    match read_file(path) {
-        Ok(old_contents) if normalize(&old_contents) == normalize(contents) => {
-            return Ok(());
+/// Checks that the `file` has the specified `contents`. If that is not the
+/// case, updates the file and then fails the test.
+pub(crate) fn ensure_file_contents(file: &Path, contents: &str) -> Result<()> {
+    match std::fs::read_to_string(file) {
+        Ok(old_contents) if normalize_newlines(&old_contents) == normalize_newlines(contents) => {
+            return Ok(())
         }
         _ => (),
     }
-    if mode == Mode::Verify {
-        anyhow::bail!("`{}` is not up-to-date", path.display());
+    let display_path = file.strip_prefix(&project_root()).unwrap_or(file);
+    eprintln!(
+        "\n\x1b[31;1merror\x1b[0m: {} was not up-to-date, updating\n",
+        display_path.display()
+    );
+    if std::env::var("CI").is_ok() {
+        eprintln!("    NOTE: run `cargo test` locally and commit the updated files\n");
     }
-    eprintln!("updating {}", path.display());
-    write_file(path, contents)?;
-    return Ok(());
-
-    fn normalize(s: &str) -> String {
-        s.replace("\r\n", "\n")
+    if let Some(parent) = file.parent() {
+        let _ = std::fs::create_dir_all(parent);
     }
+    std::fs::write(file, contents).unwrap();
+    anyhow::bail!("some file were not up to date")
+}
+
+fn normalize_newlines(s: &str) -> String {
+    s.replace("\r\n", "\n")
 }
 
 const PREAMBLE: &str = "Generated file, do not edit by hand, see `xtask/src/codegen`";

xtask/src/codegen/gen_assists_docs.rs

@@ -2,22 +2,25 @@
 
 use std::{fmt, path::Path};
 
+use xshell::write_file;
+
 use crate::{
-    codegen::{self, extract_comment_blocks_with_empty_lines, reformat, Location, Mode, PREAMBLE},
+    codegen::{self, extract_comment_blocks_with_empty_lines, reformat, Location, PREAMBLE},
     project_root, rust_files_in, Result,
 };
 
-pub(crate) fn generate_assists_tests(mode: Mode) -> Result<()> {
+pub(crate) fn generate_assists_tests() -> Result<()> {
     let assists = Assist::collect()?;
-    generate_tests(&assists, mode)
+    generate_tests(&assists)
 }
 
-pub(crate) fn generate_assists_docs(mode: Mode) -> Result<()> {
+pub(crate) fn generate_assists_docs() -> Result<()> {
     let assists = Assist::collect()?;
     let contents = assists.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
     let contents = format!("//{}\n{}\n", PREAMBLE, contents.trim());
     let dst = project_root().join("docs/user/generated_assists.adoc");
-    codegen::update(&dst, &contents, mode)
+    write_file(dst, &contents)?;
+    Ok(())
 }
 
 #[derive(Debug)]
@@ -111,7 +114,7 @@ impl fmt::Display for Assist {
     }
 }
 
-fn generate_tests(assists: &[Assist], mode: Mode) -> Result<()> {
+fn generate_tests(assists: &[Assist]) -> Result<()> {
     let mut buf = String::from("use super::check_doc_test;\n");
 
     for assist in assists.iter() {
@@ -135,7 +138,10 @@ r#####"
         buf.push_str(&test)
     }
     let buf = reformat(&buf)?;
-    codegen::update(&project_root().join("crates/ide_assists/src/tests/generated.rs"), &buf, mode)
+    codegen::ensure_file_contents(
+        &project_root().join("crates/ide_assists/src/tests/generated.rs"),
+        &buf,
+    )
 }
 
 fn hide_hash_comments(text: &str) -> String {

xtask/src/codegen/gen_diagnostic_docs.rs

@@ -2,18 +2,20 @@
 
 use std::{fmt, path::PathBuf};
 
+use xshell::write_file;
+
 use crate::{
-    codegen::{self, extract_comment_blocks_with_empty_lines, Location, Mode, PREAMBLE},
+    codegen::{extract_comment_blocks_with_empty_lines, Location, PREAMBLE},
     project_root, rust_files, Result,
 };
 
-pub(crate) fn generate_diagnostic_docs(mode: Mode) -> Result<()> {
+pub(crate) fn generate_diagnostic_docs() -> Result<()> {
     let diagnostics = Diagnostic::collect()?;
     let contents =
         diagnostics.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
     let contents = format!("//{}\n{}\n", PREAMBLE, contents.trim());
     let dst = project_root().join("docs/user/generated_diagnostic.adoc");
-    codegen::update(&dst, &contents, mode)?;
+    write_file(&dst, &contents)?;
     Ok(())
 }
 

xtask/src/codegen/gen_feature_docs.rs

@@ -2,17 +2,19 @@
 
 use std::{fmt, path::PathBuf};
 
+use xshell::write_file;
+
 use crate::{
-    codegen::{self, extract_comment_blocks_with_empty_lines, Location, Mode, PREAMBLE},
+    codegen::{extract_comment_blocks_with_empty_lines, Location, PREAMBLE},
     project_root, rust_files, Result,
 };
 
-pub(crate) fn generate_feature_docs(mode: Mode) -> Result<()> {
+pub(crate) fn generate_feature_docs() -> Result<()> {
     let features = Feature::collect()?;
     let contents = features.into_iter().map(|it| it.to_string()).collect::<Vec<_>>().join("\n\n");
     let contents = format!("//{}\n{}\n", PREAMBLE, contents.trim());
     let dst = project_root().join("docs/user/generated_features.adoc");
-    codegen::update(&dst, &contents, mode)?;
+    write_file(&dst, &contents)?;
     Ok(())
 }