diff --git a/src/compiler/cicc.rs b/src/compiler/cicc.rs index bf4901f648..cdba52a1f8 100644 --- a/src/compiler/cicc.rs +++ b/src/compiler/cicc.rs @@ -319,6 +319,7 @@ pub fn generate_compile_commands( arguments, env_vars: env_vars.to_owned(), cwd: cwd.to_owned(), + share_jobserver: false, }; #[cfg(not(feature = "dist-client"))] diff --git a/src/compiler/compiler.rs b/src/compiler/compiler.rs index 50c4a4ecdd..0939ef08a5 100644 --- a/src/compiler/compiler.rs +++ b/src/compiler/compiler.rs @@ -166,6 +166,14 @@ pub struct SingleCompileCommand { pub arguments: Vec, pub env_vars: Vec<(OsString, OsString)>, pub cwd: PathBuf, + /// Whether this compiler participates in the GNU make jobserver. + /// + /// Deliberately a field rather than a defaulted builder method: the + /// compiler then makes every frontend answer, so a new one cannot + /// silently get this wrong. Getting it wrong in the `true` direction + /// costs a `fork` per compile; in the `false` direction it costs `rustc` + /// its parallelism limit, which is what the jobserver exists to enforce. + pub share_jobserver: bool, } #[async_trait] @@ -196,6 +204,7 @@ impl CompileCommandImpl for SingleCompileCommand { arguments, env_vars, cwd, + share_jobserver, } = self; // Resolve compiler avoiding ccache wrappers to prevent double-caching. let resolved_executable = resolve_compiler_avoiding_wrapper(executable, env_vars); @@ -204,6 +213,9 @@ impl CompileCommandImpl for SingleCompileCommand { .env_clear() .envs(env_vars.clone()) .current_dir(cwd); + if *share_jobserver { + cmd.share_jobserver(); + } run_input_output(cmd, None).await } } diff --git a/src/compiler/cudafe.rs b/src/compiler/cudafe.rs index 9ca1eb6631..d6440ac4c1 100644 --- a/src/compiler/cudafe.rs +++ b/src/compiler/cudafe.rs @@ -154,6 +154,7 @@ pub fn generate_compile_commands( arguments, env_vars: env_vars.to_owned(), cwd: cwd.to_owned(), + share_jobserver: false, }; #[cfg(not(feature = "dist-client"))] diff --git a/src/compiler/diab.rs b/src/compiler/diab.rs index a11e578606..8212930036 100644 --- a/src/compiler/diab.rs +++ b/src/compiler/diab.rs @@ -378,6 +378,7 @@ pub fn generate_compile_commands( arguments, env_vars: env_vars.to_owned(), cwd: cwd.to_owned(), + share_jobserver: false, }; Ok((command, None, Cacheable::Yes)) diff --git a/src/compiler/gcc.rs b/src/compiler/gcc.rs index 8a832b5d68..eb378c1e24 100644 --- a/src/compiler/gcc.rs +++ b/src/compiler/gcc.rs @@ -999,6 +999,7 @@ where arguments, env_vars: env_vars.to_owned(), cwd: cwd.to_owned(), + share_jobserver: false, }; #[cfg(not(feature = "dist-client"))] diff --git a/src/compiler/msvc.rs b/src/compiler/msvc.rs index 1e58a077ba..21edfe3021 100644 --- a/src/compiler/msvc.rs +++ b/src/compiler/msvc.rs @@ -1141,6 +1141,7 @@ fn generate_compile_commands( arguments, env_vars: env_vars.to_owned(), cwd: cwd.to_owned(), + share_jobserver: false, }; #[cfg(not(feature = "dist-client"))] diff --git a/src/compiler/rust.rs b/src/compiler/rust.rs index 779e9dd79e..1a1a24a173 100644 --- a/src/compiler/rust.rs +++ b/src/compiler/rust.rs @@ -1798,6 +1798,11 @@ impl Compilation for RustCompilation { .collect(), env_vars: env_vars.to_owned(), cwd: cwd.to_owned(), + // rustc reads `CARGO_MAKEFLAGS` and runs codegen on a thread pool + // sized by the jobserver. Without one, every concurrent rustc + // spawns as many threads as there are CPUs, which is the + // oversubscription sccache's own jobserver exists to prevent. + share_jobserver: true, }; #[cfg(not(feature = "dist-client"))] diff --git a/src/compiler/tasking_vx.rs b/src/compiler/tasking_vx.rs index b3fff8238a..3e1ed1bb15 100644 --- a/src/compiler/tasking_vx.rs +++ b/src/compiler/tasking_vx.rs @@ -391,6 +391,7 @@ fn generate_compile_commands( arguments, env_vars: env_vars.to_owned(), cwd: cwd.to_owned(), + share_jobserver: false, }; Ok((command, None, Cacheable::Yes)) diff --git a/src/dist/mod.rs b/src/dist/mod.rs index 6bc1024aa8..0fa99d90dd 100644 --- a/src/dist/mod.rs +++ b/src/dist/mod.rs @@ -326,6 +326,8 @@ pub fn try_compile_command_to_dist( arguments, env_vars, cwd, + // The jobserver is a local resource; a remote worker has its own. + share_jobserver: _, } = command; Some(CompileCommand { executable: executable.into_os_string().into_string().ok()?, diff --git a/src/mock_command.rs b/src/mock_command.rs index 3ad5ecc3f4..2f4494e3f3 100644 --- a/src/mock_command.rs +++ b/src/mock_command.rs @@ -111,6 +111,15 @@ pub trait RunCommand: fmt::Debug + Send { fn stdout(&mut self, cfg: Stdio) -> &mut Self; /// Set the process' stderr from `cfg`. fn stderr(&mut self, cfg: Stdio) -> &mut Self; + /// Hand sccache's jobserver down to this child. + /// + /// Only worth doing for a child that implements the protocol -- in + /// practice `rustc`, which reads `CARGO_MAKEFLAGS`. It is not free: see + /// [`AsyncCommand::spawn`] for why sharing costs a `fork` instead of a + /// `posix_spawn`. + fn share_jobserver(&mut self) -> &mut Self { + self + } /// Execute the process and return a process object. async fn spawn(&mut self) -> Result; } @@ -180,6 +189,7 @@ impl CommandChild for Child { pub struct AsyncCommand { inner: Option, jobserver: Client, + share_jobserver: bool, } impl AsyncCommand { @@ -187,6 +197,7 @@ impl AsyncCommand { AsyncCommand { inner: Some(Command::new(program)), jobserver, + share_jobserver: false, } } @@ -246,13 +257,28 @@ impl RunCommand for AsyncCommand { self.inner().stderr(cfg); self } + fn share_jobserver(&mut self) -> &mut AsyncCommand { + self.share_jobserver = true; + self + } async fn spawn(&mut self) -> Result { let mut inner = self.inner.take().unwrap(); inner.env_remove("MAKEFLAGS"); inner.env_remove("MFLAGS"); inner.env_remove("CARGO_MAKEFLAGS"); - self.jobserver.configure(&mut inner); + // `configure` registers a `pre_exec` closure to clear `CLOEXEC` on the + // jobserver's file descriptors, and any `pre_exec` makes `std` fall + // back from `posix_spawn` to `fork`+`exec`. Almost nothing sccache + // spawns can use a jobserver -- preprocessors, version probes and + // C/C++ compilers all ignore it -- so the default is not to share, + // and the callers that spawn `rustc` ask for it. + if self.share_jobserver { + self.jobserver.configure(&mut inner); + } + // The token is acquired either way: it rate-limits how many children + // *we* run, which is separate from whether the child can acquire + // tokens of its own. let token = self.jobserver.acquire().await?; let mut inner = tokio::process::Command::from(inner); let child = inner @@ -667,4 +693,40 @@ mod test { ); assert_eq!(exit_status(0), spawn_on_thread(creator, true)); } + + /// The jobserver reaches a child through `CARGO_MAKEFLAGS`, so reading it + /// back out of the child is what "did the child get the jobserver?" means. + #[cfg(unix)] + fn child_sees_cargo_makeflags(share: bool) -> bool { + let client = Client::new_num(1); + let mut creator = ::new(&client); + let runtime = tokio::runtime::Runtime::new().unwrap(); + let mut cmd = creator.new_command("/bin/sh"); + cmd.args(&["-c", "printf %s \"$CARGO_MAKEFLAGS\""]); + if share { + cmd.share_jobserver(); + } + let output: std::process::Output = runtime + .block_on(async { crate::util::run_input_output(cmd, None).await }) + .unwrap(); + !output.stdout.is_empty() + } + + #[test] + #[cfg(unix)] + fn jobserver_is_withheld_by_default() { + assert!( + !child_sees_cargo_makeflags(false), + "a child should not inherit the jobserver unless it asks" + ); + } + + #[test] + #[cfg(unix)] + fn share_jobserver_hands_it_over() { + assert!( + child_sees_cargo_makeflags(true), + "share_jobserver() should give the child CARGO_MAKEFLAGS" + ); + } }