Optimize child process spawns by skipping job server configs

src/compiler/cicc.rs

@@ -319,6 +319,7 @@ pub fn generate_compile_commands(
         arguments,
         env_vars: env_vars.to_owned(),
         cwd: cwd.to_owned(),
+        share_jobserver: false,
     };
 
     #[cfg(not(feature = "dist-client"))]

src/compiler/compiler.rs

@@ -166,6 +166,14 @@ pub struct SingleCompileCommand {
     pub arguments: Vec<OsString>,
     pub env_vars: Vec<(OsString, OsString)>,
     pub cwd: PathBuf,
+    /// Whether this compiler participates in the GNU make jobserver.
+    ///
+    /// Deliberately a field rather than a defaulted builder method: the
+    /// compiler then makes every frontend answer, so a new one cannot
+    /// silently get this wrong. Getting it wrong in the `true` direction
+    /// costs a `fork` per compile; in the `false` direction it costs `rustc`
+    /// its parallelism limit, which is what the jobserver exists to enforce.
+    pub share_jobserver: bool,
 }
 
 #[async_trait]
@@ -196,6 +204,7 @@ impl CompileCommandImpl for SingleCompileCommand {
             arguments,
             env_vars,
             cwd,
+            share_jobserver,
         } = self;
         // Resolve compiler avoiding ccache wrappers to prevent double-caching.
         let resolved_executable = resolve_compiler_avoiding_wrapper(executable, env_vars);
@@ -204,6 +213,9 @@ impl CompileCommandImpl for SingleCompileCommand {
             .env_clear()
             .envs(env_vars.clone())
             .current_dir(cwd);
+        if *share_jobserver {
+            cmd.share_jobserver();
+        }
         run_input_output(cmd, None).await
     }
 }

src/compiler/cudafe.rs

@@ -154,6 +154,7 @@ pub fn generate_compile_commands(
         arguments,
         env_vars: env_vars.to_owned(),
         cwd: cwd.to_owned(),
+        share_jobserver: false,
     };
 
     #[cfg(not(feature = "dist-client"))]

src/compiler/diab.rs

@@ -378,6 +378,7 @@ pub fn generate_compile_commands(
         arguments,
         env_vars: env_vars.to_owned(),
         cwd: cwd.to_owned(),
+        share_jobserver: false,
     };
 
     Ok((command, None, Cacheable::Yes))

src/compiler/gcc.rs

@@ -999,6 +999,7 @@ where
         arguments,
         env_vars: env_vars.to_owned(),
         cwd: cwd.to_owned(),
+        share_jobserver: false,
     };
 
     #[cfg(not(feature = "dist-client"))]

src/compiler/msvc.rs

@@ -1141,6 +1141,7 @@ fn generate_compile_commands(
         arguments,
         env_vars: env_vars.to_owned(),
         cwd: cwd.to_owned(),
+        share_jobserver: false,
     };
 
     #[cfg(not(feature = "dist-client"))]

src/compiler/rust.rs

@@ -1798,6 +1798,11 @@ impl<T: CommandCreatorSync> Compilation<T> for RustCompilation {
                 .collect(),
             env_vars: env_vars.to_owned(),
             cwd: cwd.to_owned(),
+            // rustc reads `CARGO_MAKEFLAGS` and runs codegen on a thread pool
+            // sized by the jobserver. Without one, every concurrent rustc
+            // spawns as many threads as there are CPUs, which is the
+            // oversubscription sccache's own jobserver exists to prevent.
+            share_jobserver: true,
         };
 
         #[cfg(not(feature = "dist-client"))]

src/compiler/tasking_vx.rs

@@ -391,6 +391,7 @@ fn generate_compile_commands(
         arguments,
         env_vars: env_vars.to_owned(),
         cwd: cwd.to_owned(),
+        share_jobserver: false,
     };
 
     Ok((command, None, Cacheable::Yes))

src/dist/mod.rs

@@ -326,6 +326,8 @@ pub fn try_compile_command_to_dist(
         arguments,
         env_vars,
         cwd,
+        // The jobserver is a local resource; a remote worker has its own.
+        share_jobserver: _,
     } = command;
     Some(CompileCommand {
         executable: executable.into_os_string().into_string().ok()?,

src/mock_command.rs

@@ -111,6 +111,15 @@ pub trait RunCommand: fmt::Debug + Send {
     fn stdout(&mut self, cfg: Stdio) -> &mut Self;
     /// Set the process' stderr from `cfg`.
     fn stderr(&mut self, cfg: Stdio) -> &mut Self;
+    /// Hand sccache's jobserver down to this child.
+    ///
+    /// Only worth doing for a child that implements the protocol -- in
+    /// practice `rustc`, which reads `CARGO_MAKEFLAGS`. It is not free: see
+    /// [`AsyncCommand::spawn`] for why sharing costs a `fork` instead of a
+    /// `posix_spawn`.
+    fn share_jobserver(&mut self) -> &mut Self {
+        self
+    }
     /// Execute the process and return a process object.
     async fn spawn(&mut self) -> Result<Self::C>;
 }
@@ -180,13 +189,15 @@ impl CommandChild for Child {
 pub struct AsyncCommand {
     inner: Option<Command>,
     jobserver: Client,
+    share_jobserver: bool,
 }
 
 impl AsyncCommand {
     pub fn new<S: AsRef<OsStr>>(program: S, jobserver: Client) -> AsyncCommand {
         AsyncCommand {
             inner: Some(Command::new(program)),
             jobserver,
+            share_jobserver: false,
         }
     }
 
@@ -246,13 +257,28 @@ impl RunCommand for AsyncCommand {
         self.inner().stderr(cfg);
         self
     }
+    fn share_jobserver(&mut self) -> &mut AsyncCommand {
+        self.share_jobserver = true;
+        self
+    }
     async fn spawn(&mut self) -> Result<Child> {
         let mut inner = self.inner.take().unwrap();
         inner.env_remove("MAKEFLAGS");
         inner.env_remove("MFLAGS");
         inner.env_remove("CARGO_MAKEFLAGS");
-        self.jobserver.configure(&mut inner);
+        // `configure` registers a `pre_exec` closure to clear `CLOEXEC` on the
+        // jobserver's file descriptors, and any `pre_exec` makes `std` fall
+        // back from `posix_spawn` to `fork`+`exec`. Almost nothing sccache
+        // spawns can use a jobserver -- preprocessors, version probes and
+        // C/C++ compilers all ignore it -- so the default is not to share,
+        // and the callers that spawn `rustc` ask for it.
+        if self.share_jobserver {
+            self.jobserver.configure(&mut inner);
+        }
 
+        // The token is acquired either way: it rate-limits how many children
+        // *we* run, which is separate from whether the child can acquire
+        // tokens of its own.
         let token = self.jobserver.acquire().await?;
         let mut inner = tokio::process::Command::from(inner);
         let child = inner
@@ -667,4 +693,40 @@ mod test {
         );
         assert_eq!(exit_status(0), spawn_on_thread(creator, true));
     }
+
+    /// The jobserver reaches a child through `CARGO_MAKEFLAGS`, so reading it
+    /// back out of the child is what "did the child get the jobserver?" means.
+    #[cfg(unix)]
+    fn child_sees_cargo_makeflags(share: bool) -> bool {
+        let client = Client::new_num(1);
+        let mut creator = <ProcessCommandCreator as CommandCreator>::new(&client);
+        let runtime = tokio::runtime::Runtime::new().unwrap();
+        let mut cmd = creator.new_command("/bin/sh");
+        cmd.args(&["-c", "printf %s \"$CARGO_MAKEFLAGS\""]);
+        if share {
+            cmd.share_jobserver();
+        }
+        let output: std::process::Output = runtime
+            .block_on(async { crate::util::run_input_output(cmd, None).await })
+            .unwrap();
+        !output.stdout.is_empty()
+    }
+
+    #[test]
+    #[cfg(unix)]
+    fn jobserver_is_withheld_by_default() {
+        assert!(
+            !child_sees_cargo_makeflags(false),
+            "a child should not inherit the jobserver unless it asks"
+        );
+    }
+
+    #[test]
+    #[cfg(unix)]
+    fn share_jobserver_hands_it_over() {
+        assert!(
+            child_sees_cargo_makeflags(true),
+            "share_jobserver() should give the child CARGO_MAKEFLAGS"
+        );
+    }
 }