rustc: implement argsfiles for command line

This makes `rustc` support `@path` arguments on the command line. The `path` is opened and the file is interpreted
as new command line options which are logically inserted at that point in the command-line. The options in the file
are one per line. The file is UTF-8 encoded, and may have either Unix or Windows line endings.
It does not support recursive use of `@path`.

This is useful for very large command lines, or when command-lines are being generated into files by other tooling.

Author: jsgf

src/doc/rustc/src/command-line-arguments.md

@@ -304,3 +304,10 @@ to customize the output:
 
 Note that it is invalid to combine the `--json` argument with the `--color`
 argument, and it is required to combine `--json` with `--error-format=json`.
+
+## `@path`: load command-line flags from a path
+
+If you specify `@path` on the command-line, then it will open `path` and read
+command line options from it. These options are one per line; a blank line indicates
+an empty option. The file can use Unix or Windows style line endings, and must be
+encoded as UTF-8.

src/librustc_driver/args/mod.rs

@@ -0,0 +1,145 @@
+#![allow(dead_code)]
+
+use std::env;
+use std::error;
+use std::fmt;
+use std::fs;
+use std::io;
+use std::str;
+
+#[cfg(test)]
+mod tests;
+
+/// States for parsing text
+#[derive(Debug, Copy, Clone, Eq, PartialEq)]
+enum State {
+    Normal, // within normal text
+    Cr,     // just saw \r
+    Lf,     // just saw \n
+}
+
+struct FileArgs {
+    path: String,
+    input: Vec<u8>,
+    offset: usize,
+}
+
+impl FileArgs {
+    pub fn new(path: String, input: Vec<u8>) -> Self {
+        FileArgs { path, input, offset: 0 }
+    }
+}
+
+impl Iterator for FileArgs {
+    type Item = Result<String, Error>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        if self.offset >= self.input.len() {
+            // All done
+            return None;
+        }
+
+        use State::*;
+        let mut state = Normal;
+        let start = self.offset;
+        let mut end = start;
+
+        for (idx, b) in self.input[start..].iter().enumerate() {
+            let idx = start + idx + 1;
+
+            self.offset = idx;
+
+            match (b, state) {
+                (b'\r', Normal) => state = Cr,
+                (b'\n', Normal) => state = Lf,
+
+                (b'\r', Lf) | (b'\n', Cr) => {
+                    // Two-character line break (accept \r\n and \n\r(?)), so consume them both
+                    break;
+                }
+
+                (_, Cr) | (_, Lf) => {
+                    // Peeked at character after single-character line break, so rewind to visit it
+                    // next time around.
+                    self.offset = idx - 1;
+                    break;
+                }
+
+                (_, _) => {
+                    end = idx;
+                    state = Normal;
+                }
+            }
+        }
+
+        Some(
+            String::from_utf8(self.input[start..end].to_vec())
+                .map_err(|_| Error::Utf8Error(Some(self.path.clone()))),
+        )
+    }
+}
+
+pub struct ArgsIter {
+    base: env::ArgsOs,
+    file: Option<FileArgs>,
+}
+
+impl ArgsIter {
+    pub fn new() -> Self {
+        ArgsIter { base: env::args_os(), file: None }
+    }
+}
+
+impl Iterator for ArgsIter {
+    type Item = Result<String, Error>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        loop {
+            if let Some(ref mut file) = &mut self.file {
+                match file.next() {
+                    Some(res) => return Some(res.map_err(From::from)),
+                    None => self.file = None,
+                }
+            }
+
+            let arg =
+                self.base.next().map(|arg| arg.into_string().map_err(|_| Error::Utf8Error(None)));
+            match arg {
+                Some(Err(err)) => return Some(Err(err)),
+                Some(Ok(ref arg)) if arg.starts_with("@") => {
+                    // can't not be utf-8 now
+                    let path = str::from_utf8(&arg.as_bytes()[1..]).unwrap();
+                    let file = match fs::read(path) {
+                        Ok(file) => file,
+                        Err(err) => return Some(Err(Error::IOError(path.to_string(), err))),
+                    };
+                    self.file = Some(FileArgs::new(path.to_string(), file));
+                }
+                Some(Ok(arg)) => return Some(Ok(arg)),
+                None => return None,
+            }
+        }
+    }
+}
+
+#[derive(Debug)]
+pub enum Error {
+    Utf8Error(Option<String>),
+    IOError(String, io::Error),
+}
+
+impl fmt::Display for Error {
+    fn fmt(&self, fmt: &mut fmt::Formatter<'_>) -> fmt::Result {
+        match self {
+            Error::Utf8Error(None) => write!(fmt, "Utf8 error"),
+            Error::Utf8Error(Some(path)) => write!(fmt, "Utf8 error in {}", path),
+            Error::IOError(path, err) => write!(fmt, "IO Error: {}: {}", path, err),
+        }
+    }
+}
+
+impl error::Error for Error {
+    fn description(&self) -> &'static str {
+        "argument error"
+    }
+}

src/librustc_driver/args/tests.rs

@@ -0,0 +1,137 @@
+use super::*;
+
+fn want_args(v: impl IntoIterator<Item = &'static str>) -> Vec<String> {
+    v.into_iter().map(String::from).collect()
+}
+
+fn got_args(file: &[u8]) -> Result<Vec<String>, Error> {
+    FileArgs::new(String::new(), file.to_vec()).collect()
+}
+
+#[test]
+fn nothing() {
+    let file = b"";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec![]));
+}
+
+#[test]
+fn empty() {
+    let file = b"\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec![""]));
+}
+
+#[test]
+fn simple() {
+    let file = b"foo";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo"]));
+}
+
+#[test]
+fn simple_eol() {
+    let file = b"foo\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo"]));
+}
+
+#[test]
+fn multi() {
+    let file = b"foo\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_eol() {
+    let file = b"foo\nbar\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_empty() {
+    let file = b"foo\n\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_eol() {
+    let file = b"foo\n\nbar\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_start() {
+    let file = b"\nfoo\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["", "foo", "bar"]));
+}
+
+#[test]
+fn multi_empty_end() {
+    let file = b"foo\nbar\n\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar", ""]));
+}
+
+fn simple_eol_crlf() {
+    let file = b"foo\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo"]));
+}
+
+#[test]
+fn multi_crlf() {
+    let file = b"foo\r\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_eol_crlf() {
+    let file = b"foo\r\nbar\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar"]));
+}
+
+#[test]
+fn multi_empty_crlf() {
+    let file = b"foo\r\n\r\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_eol_crlf() {
+    let file = b"foo\r\n\r\nbar\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "", "bar"]));
+}
+
+#[test]
+fn multi_empty_start_crlf() {
+    let file = b"\r\nfoo\r\nbar";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["", "foo", "bar"]));
+}
+
+#[test]
+fn multi_empty_end_crlf() {
+    let file = b"foo\r\nbar\r\n\r\n";
+
+    assert_eq!(got_args(file).unwrap(), want_args(vec!["foo", "bar", ""]));
+}
+
+#[test]
+fn bad_utf8() {
+    let file = b"foo\x80foo";
+
+    match got_args(file).unwrap_err() {
+        Error::Utf8Error(_) => (),
+        bad => panic!("bad err: {:?}", bad),
+    }
+}

src/librustc_driver/lib.rs

@@ -66,6 +66,7 @@ use syntax::symbol::sym;
 use syntax_pos::{DUMMY_SP, MultiSpan, FileName};
 
 pub mod pretty;
+mod args;
 
 /// Exit status code used for successful compilation and help output.
 pub const EXIT_SUCCESS: i32 = 0;
@@ -777,11 +778,17 @@ fn usage(verbose: bool, include_unstable_options: bool) {
     } else {
         "\n    --help -v           Print the full set of options rustc accepts"
     };
-    println!("{}\nAdditional help:
+    let at_path = if verbose {
+        "    @path               Read newline separated options from `path`\n"
+    } else {
+        ""
+    };
+    println!("{}{}\nAdditional help:
     -C help             Print codegen options
     -W help             \
               Print 'lint' options and default settings{}{}\n",
              options.usage(message),
+             at_path,
              nightly_help,
              verbose_help);
 }
@@ -1186,10 +1193,10 @@ pub fn main() {
     init_rustc_env_logger();
     let mut callbacks = TimePassesCallbacks::default();
     let result = report_ices_to_stderr_if_any(|| {
-        let args = env::args_os().enumerate()
-            .map(|(i, arg)| arg.into_string().unwrap_or_else(|arg| {
+        let args = args::ArgsIter::new().enumerate()
+            .map(|(i, arg)| arg.unwrap_or_else(|err| {
                 early_error(ErrorOutputType::default(),
-                            &format!("Argument {} is not valid Unicode: {:?}", i, arg))
+                            &format!("Argument {} is not valid: {}", i, err))
             }))
             .collect::<Vec<_>>();
         run_compiler(&args, &mut callbacks, None, None)

src/test/ui/commandline-argfile-badutf8.args

@@ -0,0 +1,2 @@
+--cfg
+unbroken�

src/test/ui/commandline-argfile-badutf8.rs

@@ -0,0 +1,13 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-fail
+// compile-flags: --cfg cmdline_set @{{src-base}}/commandline-argfile-badutf8.args
+
+#[cfg(not(cmdline_set))]
+compile_error!("cmdline_set not set");
+
+#[cfg(not(unbroken))]
+compile_error!("unbroken not set");
+
+fn main() {
+}

src/test/ui/commandline-argfile-badutf8.stderr

@@ -0,0 +1,2 @@
+error: Argument 19 is not valid: Utf8 error in $DIR/commandline-argfile-badutf8.args
+

src/test/ui/commandline-argfile-missing.rs

@@ -0,0 +1,13 @@
+// Check to see if we can get parameters from an @argsfile file
+//
+// build-fail
+// compile-flags: --cfg cmdline_set @{{src-base}}/commandline-argfile-missing.args
+
+#[cfg(not(cmdline_set))]
+compile_error!("cmdline_set not set");
+
+#[cfg(not(unbroken))]
+compile_error!("unbroken not set");
+
+fn main() {
+}

src/test/ui/commandline-argfile-missing.stderr

@@ -0,0 +1,2 @@
+error: Argument 18 is not valid: IO Error: $DIR/commandline-argfile-missing.args: No such file or directory (os error 2)
+

src/test/ui/commandline-argfile.args

@@ -0,0 +1,2 @@
+--cfg
+unbroken