From 75e34d406b57050c598f05abd03daf8cedd537d9 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Mon, 23 Nov 2020 16:50:29 +0100 Subject: [PATCH 01/10] Generate response file when cross compiling from Windows to Android --- src/lib.rs | 88 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 88 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index fcf5584ea..ea9a5b87f 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,6 +121,7 @@ pub struct Build { extra_warnings: Option, env_cache: Arc>>>, apple_sdk_root_cache: Arc>>, + response_file_cache: Arc, OsString>>>, } /// Represents the types of errors that may occur while using cc-rs. @@ -314,6 +315,7 @@ impl Build { warnings_into_errors: false, env_cache: Arc::new(Mutex::new(HashMap::new())), apple_sdk_root_cache: Arc::new(Mutex::new(HashMap::new())), + response_file_cache: Arc::new(Mutex::new(HashMap::new())), } } @@ -1177,12 +1179,18 @@ impl Build { fn compile_object(&self, obj: &Object) -> Result<(), Error> { let is_asm = obj.src.extension().and_then(|s| s.to_str()) == Some("asm"); let target = self.get_target()?; + let host = self.get_host()?; let msvc = target.contains("msvc"); let compiler = self.try_get_compiler()?; let clang = compiler.family == ToolFamily::Clang; let (mut cmd, name) = if msvc && is_asm { self.msvc_macro_assembler()? } else { + let compiler = if host.contains("windows") && target.contains("android") { + self.generate_windows_to_android_response_file(compiler, obj) + } else { + compiler + }; let mut cmd = compiler.to_command(); for &(ref a, ref b) in self.env.iter() { cmd.env(a, b); @@ -2563,6 +2571,86 @@ impl Build { cache.insert(sdk.into(), ret.clone()); Ok(ret) } + + fn generate_windows_to_android_response_file(&self, compiler: Tool, obj: &Object) -> Tool { + let mut cache = self + .response_file_cache + .lock() + .expect("response_file_cache lock failed"); + if let Some(response_file_path) = cache.get(&compiler.args) { + println!("CACHED!"); + return Tool { + path: compiler.path, + cc_wrapper_path: compiler.cc_wrapper_path, + cc_wrapper_args: compiler.cc_wrapper_args, + args: vec![response_file_path.clone()], + env: compiler.env, + family: compiler.family, + cuda: compiler.cuda, + removed_args: Vec::default(), + }; + } + + // Similar to https://github.com/rust-lang/rust/pull/47507 + // and https://github.com/rust-lang/rust/pull/48548 + let estimated_command_line_len = compiler + .args + .iter() + .map(|a| a.as_os_str().len()) + .sum::(); + + if estimated_command_line_len > 1024 * 6 { + let mut args = String::from("\u{FEFF}"); // BOM + for arg in compiler.args.iter() { + if compiler.removed_args.iter().any(|a| a == arg) { + continue; + } + + args.push('"'); + for c in arg.to_str().unwrap().chars() { + if c == ' ' { + args.push('\\') + } + args.push(c) + } + args.push('"'); + args.push(' '); + } + + let args = args.replace("\\", "\\\\"); + let mut utf16le = Vec::new(); + for code_unit in args.encode_utf16() { + utf16le.push(code_unit as u8); + utf16le.push((code_unit >> 8) as u8); + } + + let mut dst = obj.dst.clone(); + dst.set_extension("args"); + let args_file = OsString::from(dst); + fs::File::create(&args_file) + .unwrap() + .write_all(&utf16le) + .unwrap(); + + let mut args_file_arg = OsString::from("@"); + args_file_arg.push(args_file); + + cache.insert(compiler.args.clone(), args_file_arg.clone()); + + Tool { + path: compiler.path, + cc_wrapper_path: compiler.cc_wrapper_path, + cc_wrapper_args: compiler.cc_wrapper_args, + args: vec![args_file_arg], + env: compiler.env, + family: compiler.family, + cuda: compiler.cuda, + removed_args: Vec::default(), + } + } else { + compiler + } + } } impl Default for Build { From 5cc37cb8575ee1728f33b911fa46796612e9c984 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Wed, 25 Nov 2020 13:51:29 +0100 Subject: [PATCH 02/10] Call clang directly when cross-compiling from Windows to Android --- src/lib.rs | 116 +++++++++++++---------------------------------------- 1 file changed, 28 insertions(+), 88 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 7e1e7d48d..f65797806 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -121,7 +121,6 @@ pub struct Build { extra_warnings: Option, env_cache: Arc>>>, apple_sdk_root_cache: Arc>>, - response_file_cache: Arc, OsString>>>, } /// Represents the types of errors that may occur while using cc-rs. @@ -315,7 +314,6 @@ impl Build { warnings_into_errors: false, env_cache: Arc::new(Mutex::new(HashMap::new())), apple_sdk_root_cache: Arc::new(Mutex::new(HashMap::new())), - response_file_cache: Arc::new(Mutex::new(HashMap::new())), } } @@ -1179,18 +1177,12 @@ impl Build { fn compile_object(&self, obj: &Object) -> Result<(), Error> { let is_asm = obj.src.extension().and_then(|s| s.to_str()) == Some("asm"); let target = self.get_target()?; - let host = self.get_host()?; let msvc = target.contains("msvc"); let compiler = self.try_get_compiler()?; let clang = compiler.family == ToolFamily::Clang; let (mut cmd, name) = if msvc && is_asm { self.msvc_macro_assembler()? } else { - let compiler = if host.contains("windows") && target.contains("android") { - self.generate_windows_to_android_response_file(compiler, obj) - } else { - compiler - }; let mut cmd = compiler.to_command(); for &(ref a, ref b) in self.env.iter() { cmd.env(a, b); @@ -2104,6 +2096,31 @@ impl Build { tool }; + // New "standalone" C/C++ cross-compiler executables from recent Android NDK + // are just shell scripts that call main clang binary (from Android NDK) with + // proper `--target` argument. + // + // For example, armv7a-linux-androideabi16-clang passes + // `--target=armv7a-linux-androideabi16` to clang. + // + // As the shell script calls the main clang binary, the command line limit length + // on Windows is restricted to around 8k characters instead of around 32k characters. + // To remove this limit, we call the main clang binary directly and contruct the + // `--target=` ourselves. + if host.contains("windows") && android_clang_compiler_uses_target_arg_internally(&tool.path) + { + let mut f = || -> Option { + let file_name = tool.path.file_name()?.to_str()?.to_owned(); + let (target, clang) = file_name.split_at(file_name.rfind("-")?); + + tool.path.set_file_name(clang.trim_start_matches("-")); + tool.path.set_extension("exe"); + tool.args.push(format!("--target={}", target).into()); + Some(true) + }; + f(); + } + // If we found `cl.exe` in our environment, the tool we're returning is // an MSVC-like tool, *and* no env vars were set then set env vars for // the tool that we're returning. @@ -2580,86 +2597,6 @@ impl Build { cache.insert(sdk.into(), ret.clone()); Ok(ret) } - - fn generate_windows_to_android_response_file(&self, compiler: Tool, obj: &Object) -> Tool { - let mut cache = self - .response_file_cache - .lock() - .expect("response_file_cache lock failed"); - if let Some(response_file_path) = cache.get(&compiler.args) { - println!("CACHED!"); - return Tool { - path: compiler.path, - cc_wrapper_path: compiler.cc_wrapper_path, - cc_wrapper_args: compiler.cc_wrapper_args, - args: vec![response_file_path.clone()], - env: compiler.env, - family: compiler.family, - cuda: compiler.cuda, - removed_args: Vec::default(), - }; - } - - // Similar to https://github.com/rust-lang/rust/pull/47507 - // and https://github.com/rust-lang/rust/pull/48548 - let estimated_command_line_len = compiler - .args - .iter() - .map(|a| a.as_os_str().len()) - .sum::(); - - if estimated_command_line_len > 1024 * 6 { - let mut args = String::from("\u{FEFF}"); // BOM - for arg in compiler.args.iter() { - if compiler.removed_args.iter().any(|a| a == arg) { - continue; - } - - args.push('"'); - for c in arg.to_str().unwrap().chars() { - if c == ' ' { - args.push('\\') - } - args.push(c) - } - args.push('"'); - args.push(' '); - } - - let args = args.replace("\\", "\\\\"); - let mut utf16le = Vec::new(); - for code_unit in args.encode_utf16() { - utf16le.push(code_unit as u8); - utf16le.push((code_unit >> 8) as u8); - } - - let mut dst = obj.dst.clone(); - dst.set_extension("args"); - let args_file = OsString::from(dst); - fs::File::create(&args_file) - .unwrap() - .write_all(&utf16le) - .unwrap(); - - let mut args_file_arg = OsString::from("@"); - args_file_arg.push(args_file); - - cache.insert(compiler.args.clone(), args_file_arg.clone()); - - Tool { - path: compiler.path, - cc_wrapper_path: compiler.cc_wrapper_path, - cc_wrapper_args: compiler.cc_wrapper_args, - args: vec![args_file_arg], - env: compiler.env, - family: compiler.family, - cuda: compiler.cuda, - removed_args: Vec::default(), - } - } else { - compiler - } - } } impl Default for Build { @@ -3092,6 +3029,9 @@ fn autodetect_android_compiler(target: &str, host: &str, gnu: &str, clang: &str) let gnu_compiler = format!("{}-{}", target, gnu); let clang_compiler = format!("{}-{}", target, clang); + println!("CLANG_COMPILER: {}", clang_compiler); + println!("TARGET: {}", target); + // On Windows, the Android clang compiler is provided as a `.cmd` file instead // of a `.exe` file. `std::process::Command` won't run `.cmd` files unless the // `.cmd` is explicitly appended to the command name, so we do that here. From 45c73513d1a461ec817967f1fae719463b81a452 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Wed, 25 Nov 2020 13:58:29 +0100 Subject: [PATCH 03/10] Remove debug println!s --- src/lib.rs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index f65797806..1267cf029 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3029,9 +3029,6 @@ fn autodetect_android_compiler(target: &str, host: &str, gnu: &str, clang: &str) let gnu_compiler = format!("{}-{}", target, gnu); let clang_compiler = format!("{}-{}", target, clang); - println!("CLANG_COMPILER: {}", clang_compiler); - println!("TARGET: {}", target); - // On Windows, the Android clang compiler is provided as a `.cmd` file instead // of a `.exe` file. `std::process::Command` won't run `.cmd` files unless the // `.cmd` is explicitly appended to the command name, so we do that here. From ebf1b456f282749b0c40128dec47116dfae5cea1 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Thu, 26 Nov 2020 10:39:18 +0100 Subject: [PATCH 04/10] Add -mstackrealign argument to some i686-linux-android targets --- src/lib.rs | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index 1267cf029..025859de5 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2116,6 +2116,13 @@ impl Build { tool.path.set_file_name(clang.trim_start_matches("-")); tool.path.set_extension("exe"); tool.args.push(format!("--target={}", target).into()); + + for version in 16..=23 { + if target.contains(&format!("i686-linux-android{}", version)) { + tool.args.push("-mstackrealign".into()); + } + } + Some(true) }; f(); From 6343b7c0df71b6e19ad6a81705588e06681e0482 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Thu, 26 Nov 2020 10:50:17 +0100 Subject: [PATCH 05/10] Cargo fmt --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 025859de5..dce4059f2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2120,7 +2120,7 @@ impl Build { for version in 16..=23 { if target.contains(&format!("i686-linux-android{}", version)) { tool.args.push("-mstackrealign".into()); - } + } } Some(true) From d8c2fb62e38a8ed7f5b75a48d5a988d3450d75dc Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Tue, 1 Dec 2020 11:56:51 +0100 Subject: [PATCH 06/10] Address feedback --- src/lib.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index dce4059f2..b3e8b25b1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2109,9 +2109,9 @@ impl Build { // `--target=` ourselves. if host.contains("windows") && android_clang_compiler_uses_target_arg_internally(&tool.path) { - let mut f = || -> Option { - let file_name = tool.path.file_name()?.to_str()?.to_owned(); - let (target, clang) = file_name.split_at(file_name.rfind("-")?); + if let Some(path) = tool.path.file_name() { + let file_name = path.to_str().unwrap().to_owned(); + let (target, clang) = file_name.split_at(file_name.rfind("-").unwrap()); tool.path.set_file_name(clang.trim_start_matches("-")); tool.path.set_extension("exe"); @@ -2122,10 +2122,7 @@ impl Build { tool.args.push("-mstackrealign".into()); } } - - Some(true) }; - f(); } // If we found `cl.exe` in our environment, the tool we're returning is From 58925ece51ba528d25bdaa25e054d7a71ea5f616 Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Tue, 1 Dec 2020 22:59:03 +0100 Subject: [PATCH 07/10] Address feedback --- src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/lib.rs b/src/lib.rs index b3e8b25b1..c7627395d 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2117,6 +2117,8 @@ impl Build { tool.path.set_extension("exe"); tool.args.push(format!("--target={}", target).into()); + // Additionally, shell scripts for target i686-linux-android versions 16 to 24 + // pass the `mstackrealign` option so we do that here as well. for version in 16..=23 { if target.contains(&format!("i686-linux-android{}", version)) { tool.args.push("-mstackrealign".into()); From ba599193c5cd5a7a32652eb21f21c315b24ffb7d Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Wed, 2 Dec 2020 18:52:07 +0100 Subject: [PATCH 08/10] Address feedback --- src/lib.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index c7627395d..391cc1db6 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2119,8 +2119,11 @@ impl Build { // Additionally, shell scripts for target i686-linux-android versions 16 to 24 // pass the `mstackrealign` option so we do that here as well. - for version in 16..=23 { - if target.contains(&format!("i686-linux-android{}", version)) { + if target.contains("i686-linux-android") { + let (_, version) = target.split_at(target.rfind("d").unwrap() + 1); + let version: u32 = version.parse().unwrap(); + + if version > 15 && version < 25 { tool.args.push("-mstackrealign".into()); } } From 8704a8e55465bac7b9bde59d36b656356a7d68fe Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Wed, 2 Dec 2020 18:55:23 +0100 Subject: [PATCH 09/10] Remove parse unwrap --- src/lib.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/lib.rs b/src/lib.rs index 391cc1db6..5755ab7f1 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2121,11 +2121,11 @@ impl Build { // pass the `mstackrealign` option so we do that here as well. if target.contains("i686-linux-android") { let (_, version) = target.split_at(target.rfind("d").unwrap() + 1); - let version: u32 = version.parse().unwrap(); - - if version > 15 && version < 25 { - tool.args.push("-mstackrealign".into()); - } + if let Ok(version) = version.parse::() { + if version > 15 && version < 25 { + tool.args.push("-mstackrealign".into()); + } + } } }; } From f4a1bce9c16e9071104bc65e75b6922496a1ca1a Mon Sep 17 00:00:00 2001 From: Max de Danschutter Date: Wed, 2 Dec 2020 18:57:37 +0100 Subject: [PATCH 10/10] Cargo fmt --- src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 5755ab7f1..9814ea7dd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2125,7 +2125,7 @@ impl Build { if version > 15 && version < 25 { tool.args.push("-mstackrealign".into()); } - } + } } }; }