diff --git a/Cargo.lock b/Cargo.lock index 3eb26acddf..cfd5584039 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6284,6 +6284,7 @@ dependencies = [ "tracing", "tracing-subscriber", "url", + "which", ] [[package]] diff --git a/crates/pixi_build_backend/Cargo.toml b/crates/pixi_build_backend/Cargo.toml index 8b5e2fcb84..7dc869cb84 100644 --- a/crates/pixi_build_backend/Cargo.toml +++ b/crates/pixi_build_backend/Cargo.toml @@ -40,6 +40,7 @@ tokio = { workspace = true, features = ["macros"] } tracing = { workspace = true } tracing-subscriber = { workspace = true } url = { workspace = true } +which = { workspace = true } pixi_build_types = { workspace = true } diff --git a/crates/pixi_build_backend/src/cache.rs b/crates/pixi_build_backend/src/cache.rs index c354464f0c..c5405bd0e7 100644 --- a/crates/pixi_build_backend/src/cache.rs +++ b/crates/pixi_build_backend/src/cache.rs @@ -22,6 +22,28 @@ pub fn sccache_envs(env: &HashMap) -> Option> { if res.is_empty() { None } else { Some(res) } } +/// Ensure the binaries for a globally-configured compiler cache are available +/// on `PATH`. +/// +/// A compiler cache that comes from the user's global pixi config is a +/// per-machine preference, so it is used as a compiler launcher only and is +/// deliberately *not* added to the build requirements (doing so would make the +/// lockfile depend on who runs the resolve). That means the tool has to be +/// installed on the machine already; if it is missing we fail with an +/// actionable hint instead of silently building without the cache. +pub fn ensure_compiler_cache_on_path(tools: &[SourcePackageName]) -> miette::Result<()> { + for tool in tools { + let name = tool.as_str(); + if which::which(name).is_err() { + return Err(miette::miette!( + help = format!("install it with `pixi global install {name}`"), + "the global `compiler-cache` config requests `{name}`, but it was not found on PATH", + )); + } + } + Ok(()) +} + pub fn add_sccache<'a, P: ProjectModel>( dependencies: &mut Dependencies<'a, ::Spec>, sccache_tools: &'a [SourcePackageName], @@ -33,3 +55,26 @@ pub fn add_sccache<'a, P: ProjectModel>( } } } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn ensure_compiler_cache_on_path_errors_when_missing() { + let tool = SourcePackageName::from(PackageName::new_unchecked( + "pixi-definitely-not-installed-cache", + )); + let err = ensure_compiler_cache_on_path(std::slice::from_ref(&tool)) + .expect_err("a non-existent tool must not be found on PATH"); + + // The hint should point users at `pixi global install`. Collapse + // whitespace first since miette wraps long lines in its rendering. + let rendered = format!("{err:?}"); + let collapsed = rendered.split_whitespace().collect::>().join(" "); + assert!( + collapsed.contains("pixi global install"), + "error should suggest `pixi global install`, got: {rendered}" + ); + } +} diff --git a/crates/pixi_build_cmake/src/build_script.j2 b/crates/pixi_build_cmake/src/build_script.j2 index 5cdaf6ddc9..1e09626237 100644 --- a/crates/pixi_build_cmake/src/build_script.j2 +++ b/crates/pixi_build_cmake/src/build_script.j2 @@ -26,6 +26,14 @@ ] -%} {% endif -%} +{# Add sccache compiler launchers if available -#} +{%- if has_sccache -%} +{%- set cmake_args = cmake_args + [ + "-DCMAKE_C_COMPILER_LAUNCHER=sccache", + "-DCMAKE_CXX_COMPILER_LAUNCHER=sccache", +] -%} +{% endif -%} + {#- Output version information -#} ninja --version cmake --version @@ -56,3 +64,6 @@ fi cmake --build . --target install {% endif -%} +{%- if has_sccache %} +sccache --show-stats +{%- endif %} diff --git a/crates/pixi_build_cmake/src/build_script.rs b/crates/pixi_build_cmake/src/build_script.rs index c64952f712..702dcf9c57 100644 --- a/crates/pixi_build_cmake/src/build_script.rs +++ b/crates/pixi_build_cmake/src/build_script.rs @@ -10,6 +10,8 @@ pub struct BuildScriptContext { /// This is used to determine if the build script /// should include Python-related logic. pub has_host_python: bool, + /// Whether sccache is available and should be used as a compiler launcher. + pub has_sccache: bool, } #[derive(Copy, Clone, Serialize)] @@ -48,6 +50,7 @@ mod test { source_dir: String::from("my-prefix-dir"), extra_args: extra_args.clone(), has_host_python, + has_sccache: false, }; let script = context.render(); @@ -70,4 +73,24 @@ mod test { insta::assert_snapshot!(script); }); } + + #[rstest] + fn test_build_script_sccache( + #[values(BuildPlatform::Windows, BuildPlatform::Unix)] build_platform: BuildPlatform, + ) { + let context = BuildScriptContext { + build_platform, + source_dir: String::from("my-prefix-dir"), + extra_args: vec![], + has_host_python: false, + has_sccache: true, + }; + let script = context.render(); + + let mut settings = insta::Settings::clone_current(); + settings.set_snapshot_suffix(format!("{build_platform}")); + settings.bind(|| { + insta::assert_snapshot!(script); + }); + } } diff --git a/crates/pixi_build_cmake/src/config.rs b/crates/pixi_build_cmake/src/config.rs index 9e286853af..38bee9fa4c 100644 --- a/crates/pixi_build_cmake/src/config.rs +++ b/crates/pixi_build_cmake/src/config.rs @@ -4,12 +4,64 @@ use indexmap::IndexMap; use pixi_build_backend::generated_recipe::BackendConfig; use serde::{Deserialize, Serialize}; +/// The compiler cache to use during builds. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum CompilerCache { + /// Use sccache as the compiler cache. + Sccache, +} + +/// A `compiler-cache` setting together with where it came from. +/// +/// The two forms are deserialized from the same `compiler-cache` key but carry +/// different consequences, so the build can keep the lockfile deterministic: +/// +/// - [`Self::Package`] — written by the package itself as a bare string +/// (`compiler-cache = "sccache"`). The cache tool is added to the build +/// requirements and therefore captured in the lockfile. +/// - [`Self::Default`] — injected by the command dispatcher as +/// `{ "default": "sccache" }` from the user's global/project pixi config. As +/// a per-machine preference it is used as a compiler launcher only and is +/// never added to the locked build requirements, so the tool must already be +/// on `PATH`. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum CompilerCacheConfig { + /// Set in the package manifest; locked as a build dependency. + Package(CompilerCache), + /// Injected default from pixi config; launcher only, not locked. + Default { + /// The cache requested by the global/project config. + default: CompilerCache, + }, +} + +impl CompilerCacheConfig { + /// The requested cache tool, regardless of where the setting came from. + pub fn cache(&self) -> &CompilerCache { + match self { + Self::Package(cache) | Self::Default { default: cache } => cache, + } + } + + /// Whether the cache tool should be added to the locked build + /// requirements. Only a package-local setting is locked; an injected + /// per-machine default is used as a launcher only. + pub fn lock_as_dependency(&self) -> bool { + matches!(self, Self::Package(_)) + } +} + #[derive(Debug, Default, Deserialize, Serialize, Clone)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub struct CMakeBackendConfig { /// Extra args for CMake invocation #[serde(default)] pub extra_args: Vec, + /// System environment variables (populated at runtime, not serialized) + #[serde(skip)] + pub system_env: IndexMap, /// Environment Variables #[serde(default)] pub env: IndexMap, @@ -23,6 +75,25 @@ pub struct CMakeBackendConfig { /// List of compilers to use (e.g., ["c", "cxx", "cuda"]) /// If not specified, a default will be used pub compilers: Option>, + /// The compiler cache to use. A bare `compiler-cache = "sccache"` in the + /// package manifest is locked as a build dependency; a value injected from + /// the user's pixi config is used as a launcher only. See + /// [`CompilerCacheConfig`]. + pub compiler_cache: Option, +} + +fn collect_system_env() -> IndexMap { + std::env::vars().collect() +} + +impl CMakeBackendConfig { + /// Create a new `CMakeBackendConfig` with the current system environment. + pub fn new_with_system_environment() -> Self { + Self { + system_env: collect_system_env(), + ..Self::default() + } + } } impl BackendConfig for CMakeBackendConfig { @@ -43,6 +114,7 @@ impl BackendConfig for CMakeBackendConfig { } else { target_config.extra_args.clone() }, + system_env: collect_system_env(), env: { let mut merged_env = self.env.clone(); merged_env.extend(target_config.env.clone()); @@ -58,6 +130,10 @@ impl BackendConfig for CMakeBackendConfig { .compilers .clone() .or_else(|| self.compilers.clone()), + compiler_cache: target_config + .compiler_cache + .clone() + .or_else(|| self.compiler_cache.clone()), }) } } @@ -68,7 +144,7 @@ mod tests { use serde_json::json; use std::path::PathBuf; - use super::CMakeBackendConfig; + use super::{CMakeBackendConfig, CompilerCache, CompilerCacheConfig}; #[test] fn test_ensure_deserialize_from_empty() { @@ -76,6 +152,38 @@ mod tests { serde_json::from_value::(json_data).unwrap(); } + #[test] + fn test_compiler_cache_distinguishes_package_from_injected_default() { + // A bare string is what a package writes in its manifest: locked. + let package = serde_json::from_value::(json!({ + "compiler-cache": "sccache" + })) + .unwrap() + .compiler_cache + .unwrap(); + assert_eq!(package, CompilerCacheConfig::Package(CompilerCache::Sccache)); + assert!(package.lock_as_dependency()); + + // The tagged form is what the command dispatcher injects from global + // config: a launcher only, never locked. + let injected = serde_json::from_value::(json!({ + "compiler-cache": { "default": "sccache" } + })) + .unwrap() + .compiler_cache + .unwrap(); + assert_eq!( + injected, + CompilerCacheConfig::Default { + default: CompilerCache::Sccache + } + ); + assert!(!injected.lock_as_dependency()); + + // Both still resolve to the same underlying cache tool. + assert_eq!(package.cache(), injected.cache()); + } + #[test] fn test_merge_with_target_config() { let mut base_env = indexmap::IndexMap::new(); @@ -88,6 +196,7 @@ mod tests { debug_dir: Some(PathBuf::from("/base/debug")), extra_input_globs: vec!["*.base".to_string()], compilers: Some(vec!["cxx".to_string()]), + ..CMakeBackendConfig::default() }; let mut target_env = indexmap::IndexMap::new(); @@ -100,6 +209,7 @@ mod tests { debug_dir: None, extra_input_globs: vec!["*.target".to_string()], compilers: Some(vec!["c".to_string(), "cuda".to_string()]), + ..CMakeBackendConfig::default() }; let merged = base_config @@ -144,6 +254,7 @@ mod tests { debug_dir: Some(PathBuf::from("/base/debug")), extra_input_globs: vec!["*.base".to_string()], compilers: Some(vec!["cxx".to_string()]), + ..CMakeBackendConfig::default() }; let empty_target_config = CMakeBackendConfig::default(); diff --git a/crates/pixi_build_cmake/src/main.rs b/crates/pixi_build_cmake/src/main.rs index 791bcebccc..b04b31c39d 100644 --- a/crates/pixi_build_cmake/src/main.rs +++ b/crates/pixi_build_cmake/src/main.rs @@ -3,9 +3,10 @@ mod config; mod inputs; use build_script::{BuildPlatform, BuildScriptContext}; -use config::CMakeBackendConfig; +use config::{CMakeBackendConfig, CompilerCache, CompilerCacheConfig}; use miette::IntoDiagnostic; use pixi_build_backend::{ + cache::{ensure_compiler_cache_on_path, sccache_envs, sccache_tools}, compilers::default_compiler_variants, generated_recipe::{DefaultMetadataProvider, GenerateRecipe, GeneratedRecipe, PythonParams}, intermediate_backend::IntermediateBackendInstantiator, @@ -18,7 +19,7 @@ use rattler_build_recipe::stage0::{Item, Script, SerializableMatchSpec, Value}; use rattler_build_types::NormalizedKey; use rattler_conda_types::PackageName; use rattler_conda_types::{ChannelUrl, Platform}; -use std::collections::HashSet; +use std::collections::{HashMap, HashSet}; use std::path::PathBuf; use std::{ collections::{BTreeMap, BTreeSet}, @@ -129,6 +130,65 @@ impl GenerateRecipe for CMakeGenerator { "python", ))); + // Enable sccache when the resolved configuration requests it. Where the + // setting came from decides how the tool is provided: a package-local + // `compiler-cache` is added to the build requirements (and therefore + // the lockfile) so the build is reproducible everywhere, while a + // globally-injected default is a per-machine preference used as a + // launcher only — adding it to the requirements would make the lockfile + // flip-flop depending on who runs the resolve, so the tool must already + // be on `PATH` instead. + let has_sccache = matches!( + config.compiler_cache.as_ref().map(CompilerCacheConfig::cache), + Some(CompilerCache::Sccache) + ); + let mut sccache_secrets: BTreeSet = BTreeSet::new(); + + if let Some(compiler_cache) = &config.compiler_cache { + // Mark any `SCCACHE_*` variables present in the system environment + // (but not explicitly set in the backend config `env`) as secrets so + // they are not leaked into the build recipe. + let system_env_vars: HashMap = config + .system_env + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + if let Some(system_sccache_keys) = sccache_envs(&system_env_vars) { + sccache_secrets = config + .system_env + .keys() + .filter(|key| { + system_sccache_keys.contains(&key.as_str()) + && !config.env.contains_key(*key) + }) + .cloned() + .collect(); + } + + if compiler_cache.lock_as_dependency() { + // Add sccache to the build requirements if not already present. + let sccache_dep: Vec> = sccache_tools() + .iter() + .map(|tool| { + Item::Value(Value::new_concrete( + SerializableMatchSpec::from(tool.as_str()), + None, + )) + }) + .collect(); + let existing_reqs: Vec<_> = requirements.build.clone().into_iter().collect(); + requirements.build.extend( + sccache_dep + .into_iter() + .filter(|dep| !existing_reqs.contains(dep)), + ); + } else { + // Globally configured: leave the locked build requirements + // untouched and require the tool to be installed on the machine. + ensure_compiler_cache_on_path(&sccache_tools())?; + } + } + let build_script = BuildScriptContext { build_platform: if Platform::current().is_windows() { BuildPlatform::Windows @@ -138,9 +198,12 @@ impl GenerateRecipe for CMakeGenerator { source_dir: manifest_root.display().to_string(), extra_args: config.extra_args.clone(), has_host_python, + has_sccache, } .render(); + sccache_secrets.extend(model.secrets.iter().cloned()); + generated_recipe.recipe.build.script = Script::from_content(build_script) .with_env( config @@ -149,7 +212,7 @@ impl GenerateRecipe for CMakeGenerator { .map(|(k, v)| (k.clone(), Value::new_concrete(v.clone(), None))) .collect(), ) - .with_secrets(model.secrets.iter().cloned().collect()); + .with_secrets(sccache_secrets.into_iter().collect()); Ok(generated_recipe) } @@ -751,4 +814,58 @@ mod tests { "Default stdlib should be c" ); } + + #[tokio::test] + async fn test_sccache_is_enabled() { + let project_model = project_fixture!({ + "name": "foobar", + "version": "0.1.0", + "targets": { + "defaultTarget": { + "runDependencies": { + "boltons": { + "binary": { + "version": "*" + } + } + } + }, + } + }); + + // SCCACHE_* env vars in system_env should be marked as secrets when + // compiler_cache is set to sccache. + let env = IndexMap::from([("SCCACHE_BUCKET".to_string(), "my-bucket".to_string())]); + let system_env = IndexMap::from([ + ("SCCACHE_SYSTEM".to_string(), "SOME_VALUE".to_string()), + ("SCCACHE_BUCKET".to_string(), "system-bucket".to_string()), + ]); + + let generated_recipe = CMakeGenerator::default() + .generate_recipe( + &project_model, + &CMakeBackendConfig { + env, + system_env, + compiler_cache: Some(CompilerCacheConfig::Package(CompilerCache::Sccache)), + ..CMakeBackendConfig::default() + }, + PathBuf::from("."), + Platform::Linux64, + None, + &HashSet::new(), + vec![], + None, + ) + .await + .expect("Failed to generate recipe"); + + // Verify that sccache is added to the build requirements and the + // system SCCACHE_* variables are recorded as secrets when + // compiler_cache = "sccache" is set. + insta::assert_yaml_snapshot!(generated_recipe.recipe, { + ".source[0].path" => "[ ... path ... ]", + ".build.script.content" => "[ ... script ... ]", + }); + } } diff --git a/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__build_script__test__build_script_sccache@unix.snap b/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__build_script__test__build_script_sccache@unix.snap new file mode 100644 index 0000000000..e734792023 --- /dev/null +++ b/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__build_script__test__build_script_sccache@unix.snap @@ -0,0 +1,25 @@ +--- +source: crates/pixi_build_cmake/src/build_script.rs +expression: script +--- +ninja --version +cmake --version + +mkdir -p build +pushd build + +if [ ! -f "build.ninja" ]; then + cmake $CMAKE_ARGS \ + -GNinja \ + -S "my-prefix-dir" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PREFIX \ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_C_COMPILER_LAUNCHER=sccache \ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache +fi + +cmake --build . --target install + +sccache --show-stats diff --git a/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__build_script__test__build_script_sccache@windows.snap b/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__build_script__test__build_script_sccache@windows.snap new file mode 100644 index 0000000000..e031ea2240 --- /dev/null +++ b/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__build_script__test__build_script_sccache@windows.snap @@ -0,0 +1,29 @@ +--- +source: crates/pixi_build_cmake/src/build_script.rs +expression: script +--- +ninja --version +cmake --version + +if not exist build mkdir build +pushd build + +if not exist build.ninja ( + cmake %CMAKE_ARGS% ^ + -GNinja ^ + -S "my-prefix-dir" ^ + -DCMAKE_BUILD_TYPE=Release ^ + -DCMAKE_INSTALL_PREFIX=%LIBRARY_PREFIX% ^ + -DCMAKE_EXPORT_COMPILE_COMMANDS=ON ^ + -DBUILD_SHARED_LIBS=ON ^ + -DCMAKE_C_COMPILER_LAUNCHER=sccache ^ + -DCMAKE_CXX_COMPILER_LAUNCHER=sccache + @if errorlevel 1 exit 1 +) + +cmake --build . --target install +@if errorlevel 1 exit 1 + + + +sccache --show-stats diff --git a/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__tests__sccache_is_enabled.snap b/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__tests__sccache_is_enabled.snap new file mode 100644 index 0000000000..7821ad78b3 --- /dev/null +++ b/crates/pixi_build_cmake/src/snapshots/pixi_build_cmake__tests__sccache_is_enabled.snap @@ -0,0 +1,49 @@ +--- +source: crates/pixi_build_cmake/src/main.rs +expression: generated_recipe.recipe +--- +schema_version: 1 +package: + name: foobar + version: 0.1.0 +build: + script: + env: + SCCACHE_BUCKET: my-bucket + secrets: + - SCCACHE_SYSTEM + content: "[ ... script ... ]" + python: + entry_points: [] + skip_pyc_compilation: [] + use_python_app_entrypoint: false + skip: [] + always_copy_files: [] + always_include_files: [] + merge_build_and_host_envs: false + files: [] + dynamic_linking: + rpaths: [] + binary_relocation: true + missing_dso_allowlist: [] + rpath_allowlist: [] + variant: + use_keys: [] + ignore_keys: [] + prefix_detection: + force_file_type: + text: [] + binary: [] + ignore: false + ignore_binary_files: false + post_process: [] +requirements: + build: + - "${{ compiler('cxx') }}" + - cmake + - ninja + - sccache + run: + - boltons +about: {} +extra: {} diff --git a/crates/pixi_build_rust/src/config.rs b/crates/pixi_build_rust/src/config.rs index 150fa4b946..de2d389253 100644 --- a/crates/pixi_build_rust/src/config.rs +++ b/crates/pixi_build_rust/src/config.rs @@ -4,6 +4,55 @@ use std::path::{Path, PathBuf}; use serde::{Deserialize, Serialize}; +/// The compiler cache to use during builds. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum CompilerCache { + /// Use sccache as the compiler cache. + Sccache, +} + +/// A `compiler-cache` setting together with where it came from. +/// +/// The two forms are deserialized from the same `compiler-cache` key but carry +/// different consequences, so the build can keep the lockfile deterministic: +/// +/// - [`Self::Package`] — written by the package itself as a bare string +/// (`compiler-cache = "sccache"`). The cache tool is added to the build +/// requirements and therefore captured in the lockfile. +/// - [`Self::Default`] — injected by the command dispatcher as +/// `{ "default": "sccache" }` from the user's global/project pixi config. As +/// a per-machine preference it is used as a compiler launcher only and is +/// never added to the locked build requirements, so the tool must already be +/// on `PATH`. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq, Eq)] +#[serde(untagged)] +pub enum CompilerCacheConfig { + /// Set in the package manifest; locked as a build dependency. + Package(CompilerCache), + /// Injected default from pixi config; launcher only, not locked. + Default { + /// The cache requested by the global/project config. + default: CompilerCache, + }, +} + +impl CompilerCacheConfig { + /// The requested cache tool, regardless of where the setting came from. + pub fn cache(&self) -> &CompilerCache { + match self { + Self::Package(cache) | Self::Default { default: cache } => cache, + } + } + + /// Whether the cache tool should be added to the locked build + /// requirements. Only a package-local setting is locked; an injected + /// per-machine default is used as a launcher only. + pub fn lock_as_dependency(&self) -> bool { + matches!(self, Self::Package(_)) + } +} + #[derive(Debug, Clone, Deserialize, Serialize)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] pub struct RustBackendConfig { @@ -34,6 +83,11 @@ pub struct RustBackendConfig { /// Example: `binaries = ["rattler-build"]` #[serde(default)] pub binaries: Vec, + /// The compiler cache to use. A bare `compiler-cache = "sccache"` in the + /// package manifest is locked as a build dependency; a value injected from + /// the user's pixi config is used as a launcher only. See + /// [`CompilerCacheConfig`]. + pub compiler_cache: Option, } fn collect_system_env() -> IndexMap { @@ -60,6 +114,7 @@ impl RustBackendConfig { ignore_cargo_manifest: Default::default(), compilers: Default::default(), binaries: Default::default(), + compiler_cache: Default::default(), } } @@ -115,6 +170,10 @@ impl BackendConfig for RustBackendConfig { .compilers .clone() .or_else(|| self.compilers.clone()), + compiler_cache: target_config + .compiler_cache + .clone() + .or_else(|| self.compiler_cache.clone()), }) } } @@ -183,6 +242,7 @@ mod tests { ignore_cargo_manifest: None, compilers: Some(vec!["rust".to_string()]), binaries: vec![], + compiler_cache: None, }; let mut target_env = indexmap::IndexMap::new(); @@ -198,6 +258,7 @@ mod tests { ignore_cargo_manifest: Some(true), compilers: Some(vec!["c".to_string(), "rust".to_string()]), binaries: vec![], + compiler_cache: None, }; let merged = base_config @@ -245,6 +306,7 @@ mod tests { ignore_cargo_manifest: None, compilers: Some(vec!["rust".to_string()]), binaries: vec![], + compiler_cache: None, }; let empty_target_config = RustBackendConfig::new_with_clean_environment(); diff --git a/crates/pixi_build_rust/src/main.rs b/crates/pixi_build_rust/src/main.rs index e19644f4fd..0618c62eba 100644 --- a/crates/pixi_build_rust/src/main.rs +++ b/crates/pixi_build_rust/src/main.rs @@ -9,7 +9,7 @@ use miette::IntoDiagnostic; use pixi_build_backend::variants::NormalizedKey; use pixi_build_backend::{ Variable, - cache::{sccache_envs, sccache_tools}, + cache::{ensure_compiler_cache_on_path, sccache_envs, sccache_tools}, compilers::default_compiler_variants, generated_recipe::{GenerateRecipe, GeneratedRecipe, PythonParams}, intermediate_backend::IntermediateBackendInstantiator, @@ -118,16 +118,17 @@ impl GenerateRecipe for RustGenerator { .map(|(k, v)| (k.clone(), v.clone())) .collect::>(); - let all_env_vars = config_env - .clone() - .into_iter() - .chain(system_env_vars.clone()) - .collect(); - let mut sccache_secrets: BTreeSet = BTreeSet::new(); - // Verify if user has set any sccache environment variables - if sccache_envs(&all_env_vars).is_some() { + // Enable sccache when the resolved configuration requests it. Where the + // setting came from decides how the tool is provided: a package-local + // `compiler-cache` is added to the build requirements (and therefore + // the lockfile) so the build is reproducible everywhere, while a + // globally-injected default is a per-machine preference used as a + // launcher only — adding it to the requirements would make the lockfile + // flip-flop depending on who runs the resolve, so the tool must already + // be on `PATH` instead. + if let Some(compiler_cache) = &config.compiler_cache { // check if we set some sccache in system env vars if let Some(system_sccache_keys) = sccache_envs(&system_env_vars) { // If sccache_envs are used in the system environment variables, @@ -146,25 +147,31 @@ impl GenerateRecipe for RustGenerator { sccache_secrets = system_sccache_keys; }; - let sccache_dep: Vec> = sccache_tools() - .iter() - .map(|tool| { - Item::Value(Value::new_concrete( - SerializableMatchSpec::from(tool.as_str()), - None, - )) - }) - .collect(); - - // Add sccache tools to the build requirements - // only if they are not already present - let existing_reqs: Vec<_> = requirements.build.clone().into_iter().collect(); + if compiler_cache.lock_as_dependency() { + let sccache_dep: Vec> = sccache_tools() + .iter() + .map(|tool| { + Item::Value(Value::new_concrete( + SerializableMatchSpec::from(tool.as_str()), + None, + )) + }) + .collect(); - requirements.build.extend( - sccache_dep - .into_iter() - .filter(|dep| !existing_reqs.contains(dep)), - ); + // Add sccache tools to the build requirements + // only if they are not already present + let existing_reqs: Vec<_> = requirements.build.clone().into_iter().collect(); + + requirements.build.extend( + sccache_dep + .into_iter() + .filter(|dep| !existing_reqs.contains(dep)), + ); + } else { + // Globally configured: leave the locked build requirements + // untouched and require the tool to be installed on the machine. + ensure_compiler_cache_on_path(&sccache_tools())?; + } has_sccache = true; } @@ -258,6 +265,7 @@ mod tests { use rattler_conda_types::PackageName; use super::*; + use config::{CompilerCache, CompilerCacheConfig}; #[tokio::test] async fn test_binaries_flag_is_rendered() { @@ -487,6 +495,7 @@ mod tests { env, system_env, ignore_cargo_manifest: Some(true), + compiler_cache: Some(CompilerCacheConfig::Package(CompilerCache::Sccache)), ..RustBackendConfig::new_with_clean_environment() }, PathBuf::from("."), @@ -499,13 +508,6 @@ mod tests { .await .expect("Failed to generate recipe"); - // Clean up environment variables - // SAFETY: We're in a test and cleaning up the environment after the test - unsafe { - std::env::remove_var("SCCACHE_SYSTEM"); - std::env::remove_var("SCCACHE_BUCKET"); - } - // Verify that sccache is added to the build requirements // when some env variables are set insta::assert_yaml_snapshot!(generated_recipe.recipe, { diff --git a/crates/pixi_command_dispatcher/Cargo.toml b/crates/pixi_command_dispatcher/Cargo.toml index 8b2df538e5..720091a749 100644 --- a/crates/pixi_command_dispatcher/Cargo.toml +++ b/crates/pixi_command_dispatcher/Cargo.toml @@ -55,6 +55,7 @@ pixi_compute_env_vars = { workspace = true } pixi_compute_network = { workspace = true } pixi_compute_reporters = { workspace = true } pixi_compute_sources = { workspace = true } +pixi_config = { workspace = true } pixi_consts = { workspace = true } pixi_git = { workspace = true } pixi_glob = { workspace = true } @@ -71,7 +72,6 @@ pixi_variant = { workspace = true } indexmap = { workspace = true } insta = { workspace = true, features = ["json"] } pixi_build_backend_passthrough = { workspace = true } -pixi_config = { workspace = true } pixi_test_utils = { workspace = true } regex = { workspace = true } slotmap = { workspace = true } diff --git a/crates/pixi_command_dispatcher/src/command_dispatcher/builder.rs b/crates/pixi_command_dispatcher/src/command_dispatcher/builder.rs index 591ba40f24..a3c96fe6ae 100644 --- a/crates/pixi_command_dispatcher/src/command_dispatcher/builder.rs +++ b/crates/pixi_command_dispatcher/src/command_dispatcher/builder.rs @@ -10,7 +10,8 @@ use crate::compute_data::{ }; use crate::environment::WorkspaceEnvRegistry; use crate::injected_config::{ - BackendOverrideKey, ChannelConfigKey, EnabledProtocolsKey, ToolBuildEnvironmentKey, + BackendOverrideKey, ChannelConfigKey, CompilerCacheKey, EnabledProtocolsKey, + ToolBuildEnvironmentKey, }; use crate::reporter::{ BackendSourceBuildReporter, BuildBackendMetadataReporter, CondaSolveReporter, GatewayReporter, @@ -30,6 +31,7 @@ use pixi_compute_env_vars::EnvVarsKey; use pixi_compute_sources::{ GitCheckoutReporter, GitCheckoutSemaphore, RootDir, UrlCheckoutReporter, UrlCheckoutSemaphore, }; +use pixi_config::CompilerCache; use pixi_git::resolver::GitResolver; use pixi_glob::GlobHashCache; use pixi_path::{AbsPathBuf, AbsPresumedDirPathBuf}; @@ -57,6 +59,8 @@ pub struct CommandDispatcherBuilder { execute_link_scripts: bool, channel_config: Option, enabled_protocols: Option, + /// Default compiler cache injected into backend configurations. + compiler_cache: Option, /// Allow symbolic links during package installation. allow_symbolic_links: Option, /// Allow hard links during package installation. @@ -318,6 +322,21 @@ impl CommandDispatcherBuilder { } } + /// Sets the default compiler cache to use for all builds dispatched + /// through this instance. Injected into the compute engine as + /// `CompilerCacheKey` and threaded into each backend's `compiler-cache` + /// configuration in the tagged `{ "default": }` form. Because this + /// is a per-machine default (not part of the package), backends use it as a + /// launcher only and must not add it to the locked build requirements; a + /// package's own `compiler-cache` (a bare string) is what governs the + /// locked dependency. + pub fn with_compiler_cache(self, compiler_cache: Option) -> Self { + Self { + compiler_cache, + ..self + } + } + /// Sets whether symbolic links are allowed during package installation. pub fn with_allow_symbolic_links(self, allow: Option) -> Self { Self { @@ -411,6 +430,7 @@ impl CommandDispatcherBuilder { ChannelConfig::default_with_root_dir(path.to_path_buf()) }); let enabled_protocols = self.enabled_protocols.unwrap_or_default(); + let compiler_cache = self.compiler_cache; let workspace_env_registry = Arc::new(WorkspaceEnvRegistry::new()); @@ -435,6 +455,7 @@ impl CommandDispatcherBuilder { conda_solve_semaphore, backend_source_build_semaphore, workspace_env_registry, + compiler_cache, }); // Build the compute engine, populating its global data store with @@ -529,6 +550,7 @@ impl CommandDispatcherBuilder { BackendOverrideKey, Arc::new(data.build_backend_overrides.clone()), ); + engine.inject(CompilerCacheKey, Arc::new(data.compiler_cache.clone())); CommandDispatcher { _dump_guard: Arc::new(DepGraphDumpGuard { diff --git a/crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs b/crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs index e1372e54fc..fb716a020b 100644 --- a/crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs +++ b/crates/pixi_command_dispatcher/src/command_dispatcher/mod.rs @@ -165,6 +165,11 @@ pub(crate) struct CommandDispatcherData { /// projection compute bodies can resolve refs via /// `ctx.global_data().workspace_env_registry().get(id)`. pub workspace_env_registry: Arc, + + /// The default compiler cache injected into backend configurations. If + /// set, this is merged into the backend configuration as a default (a + /// package's own `pixi.toml` config takes precedence). + pub compiler_cache: Option, } impl Default for CommandDispatcher { diff --git a/crates/pixi_command_dispatcher/src/injected_config.rs b/crates/pixi_command_dispatcher/src/injected_config.rs index d502eed3a8..ba72714a43 100644 --- a/crates/pixi_command_dispatcher/src/injected_config.rs +++ b/crates/pixi_command_dispatcher/src/injected_config.rs @@ -12,6 +12,7 @@ use derive_more::Display; use pixi_build_discovery::EnabledProtocols; use pixi_build_frontend::BackendOverride; use pixi_compute_engine::InjectedKey; +use pixi_config::CompilerCache; use rattler_conda_types::ChannelConfig; use crate::BuildEnvironment; @@ -52,3 +53,16 @@ pub struct BackendOverrideKey; impl InjectedKey for BackendOverrideKey { type Value = Arc; } + +/// Injected default [`CompilerCache`] for the dispatcher's engine. Injected +/// into each backend's `compiler-cache` configuration in the tagged +/// `{ "default": }` form, so backends can tell a global preference +/// apart from a package-local setting (a bare string) and avoid locking the +/// per-machine default into the build requirements. +#[derive(Clone, Debug, Display, Hash, PartialEq, Eq)] +#[display("CompilerCache")] +pub struct CompilerCacheKey; + +impl InjectedKey for CompilerCacheKey { + type Value = Arc>; +} diff --git a/crates/pixi_command_dispatcher/src/instantiate_backend_key.rs b/crates/pixi_command_dispatcher/src/instantiate_backend_key.rs index dea4c7b1ee..2482b250b2 100644 --- a/crates/pixi_command_dispatcher/src/instantiate_backend_key.rs +++ b/crates/pixi_command_dispatcher/src/instantiate_backend_key.rs @@ -24,6 +24,7 @@ use pixi_build_types::{ procedures::initialize::InitializeParams, }; use pixi_compute_engine::{ComputeCtx, Key}; +use pixi_config::CompilerCache; use pixi_path::AbsPresumedDirPathBuf; use pixi_record::PixiRecord; use pixi_spec::{BinarySpec, ResolvedExcludeNewer, SourceAnchor, SpecConversionError}; @@ -39,7 +40,7 @@ use tokio::sync::Mutex; use crate::compute_data::HasInstantiateBackendReporter; use crate::discovered_backend::DiscoveredBackendKey; use crate::ephemeral_env::{EphemeralEnvError, EphemeralEnvKey, EphemeralEnvSpec}; -use crate::injected_config::ToolBuildEnvironmentKey; +use crate::injected_config::{CompilerCacheKey, ToolBuildEnvironmentKey}; use crate::reporter::InstantiateBackendReporter; use crate::resolved_backend_command::{ResolvedBackendCommand, ResolvedBackendCommandKey}; use pixi_compute_cache_dirs::CacheDirsKey; @@ -258,6 +259,10 @@ impl InstantiateBackendKey { .to_owned() .into_std_path_buf(); + // Default compiler cache, merged into the backend configuration + // below unless the package already sets `compiler-cache` itself. + let compiler_cache = ctx.compute(&CompilerCacheKey).await; + // Apply the engine's backend override to the resolved spec. let resolved_command = ctx .compute(&ResolvedBackendCommandKey::new(resolved_spec.clone())) @@ -272,6 +277,7 @@ impl InstantiateBackendKey { &source_dir, &discovered.init_params, cache_dir_root, + &compiler_cache, ); } ResolvedBackendCommand::Spec(CommandSpec::System(system_spec)) => ( @@ -305,6 +311,7 @@ impl InstantiateBackendKey { tool, api_version, cache_dir_root, + &compiler_cache, ) .await } @@ -327,6 +334,7 @@ impl InstantiateBackendKey { source_dir: &std::path::Path, init_params: &BackendInitializationParams, cache_dir_root: PathBuf, + compiler_cache: &Option, ) -> Result> { let project_model = self .project_model_overrides @@ -338,7 +346,10 @@ impl InstantiateBackendKey { workspace_directory: Some(init_params.workspace_root.clone()), cache_directory: Some(cache_dir_root), project_model, - configuration: init_params.configuration.clone(), + configuration: inject_compiler_cache_default( + init_params.configuration.clone(), + compiler_cache, + ), target_configuration: init_params.target_configuration.clone(), }) .map_err(|e| Arc::new(InstantiateBackendError::InMemory(Arc::new(*e))))?; @@ -568,6 +579,7 @@ async fn spawn_json_rpc( tool: Tool, api_version: PixiBuildApiVersion, cache_dir_root: PathBuf, + compiler_cache: &Option, ) -> Result> { let project_model = project_model_overrides.apply(init_params.project_model.clone()); let backend = JsonRpcBackend::setup( @@ -575,7 +587,7 @@ async fn spawn_json_rpc( init_params.manifest_path.clone(), init_params.workspace_root.clone(), project_model, - init_params.configuration.clone(), + inject_compiler_cache_default(init_params.configuration.clone(), compiler_cache), init_params.target_configuration.clone(), Some(cache_dir_root), tool, @@ -588,6 +600,36 @@ async fn spawn_json_rpc( )))) } +/// Inject the dispatcher-wide default `compiler_cache` into the backend +/// `configuration` JSON. If `compiler_cache` is `None`, or the package already +/// set `compiler-cache` itself, the configuration is returned unchanged. +/// +/// The default originates from the user's global pixi config (a per-machine +/// preference), so it is injected under the same `compiler-cache` key but in +/// the tagged `{ "default": }` form rather than the bare-string form a +/// package writes. Backends use that shape to tell the two apart: only a +/// package-local `compiler-cache` adds a compiler cache to the locked build +/// requirements; a globally-injected default is a launcher only, or the +/// lockfile would flip-flop depending on who runs the resolve. +fn inject_compiler_cache_default( + configuration: Option, + compiler_cache: &Option, +) -> Option { + let Some(compiler_cache) = compiler_cache else { + return configuration; + }; + + let compiler_cache_value = + serde_json::to_value(compiler_cache).expect("CompilerCache serialization cannot fail"); + + let mut config = configuration.unwrap_or_else(|| serde_json::Value::Object(Default::default())); + if let Some(obj) = config.as_object_mut() { + obj.entry("compiler-cache") + .or_insert(serde_json::json!({ "default": compiler_cache_value })); + } + Some(config) +} + #[cfg(test)] mod tests { use super::*; diff --git a/crates/pixi_config/src/lib.rs b/crates/pixi_config/src/lib.rs index 4415cb69cf..a54a31dd9d 100644 --- a/crates/pixi_config/src/lib.rs +++ b/crates/pixi_config/src/lib.rs @@ -1685,6 +1685,14 @@ impl Serialize for PackageFormatAndCompression { } } +/// The compiler cache to use during builds. +#[derive(Clone, Debug, Deserialize, Serialize, PartialEq, Eq)] +#[serde(rename_all = "kebab-case")] +pub enum CompilerCache { + /// Use sccache as the compiler cache. + Sccache, +} + #[derive(Clone, Debug, Deserialize, Serialize, Default, PartialEq, Eq)] #[serde(rename_all = "kebab-case")] pub struct BuildConfig { @@ -1692,11 +1700,21 @@ pub struct BuildConfig { #[serde(default)] #[serde(skip_serializing_if = "Option::is_none")] pub package_format: Option, + + /// The compiler cache to use during builds. If set, the specified cache is + /// used in all build backends that support it (e.g. cmake, rust). Can be + /// set globally in `~/.config/pixi/config.toml` or per-project in + /// `.pixi/config.toml`. + /// + /// Example: `compiler-cache = "sccache"` + #[serde(default)] + #[serde(skip_serializing_if = "Option::is_none")] + pub compiler_cache: Option, } impl BuildConfig { pub fn is_default(&self) -> bool { - self.package_format.is_none() + self.package_format.is_none() && self.compiler_cache.is_none() } pub fn merge(&self, other: Self) -> Self { Self { @@ -1705,6 +1723,11 @@ impl BuildConfig { .as_ref() .or(self.package_format.as_ref()) .cloned(), + compiler_cache: other + .compiler_cache + .as_ref() + .or(self.compiler_cache.as_ref()) + .cloned(), } } } @@ -1990,6 +2013,9 @@ impl Config { pub fn get_keys(&self) -> &[&str] { &[ "authentication-override-file", + "build", + "build.compiler-cache", + "build.package-format", "cache", "cache.build-tool-environments", "cache.conda-packages", @@ -2602,6 +2628,37 @@ impl Config { self.cache.expand_paths()?; self.cache.validate()?; } + key if key.starts_with("build") => { + if key == "build" { + if let Some(value) = value { + self.build = serde_json::de::from_str(&value).into_diagnostic()?; + } else { + self.build = BuildConfig::default(); + } + return Ok(()); + } else if !key.starts_with("build.") { + return Err(err); + } + let subkey = key.strip_prefix("build.").unwrap(); + match subkey { + "compiler-cache" => { + self.build.compiler_cache = value + .map(|v| match v.as_str() { + "sccache" => Ok(CompilerCache::Sccache), + _ => Err(miette!("invalid compiler cache: {v}")), + }) + .transpose()?; + } + "package-format" => { + self.build.package_format = value + .map(|v| { + PackageFormatAndCompression::from_str(&v).map_err(|e| miette!(e)) + }) + .transpose()?; + } + _ => return Err(err), + } + } _ => return Err(err), } diff --git a/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap b/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap index 0a067a5d2e..36d08f7015 100644 --- a/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap +++ b/crates/pixi_config/src/snapshots/pixi_config__tests__config_merge_multiple.snap @@ -1,6 +1,5 @@ --- source: crates/pixi_config/src/lib.rs -assertion_line: 2831 expression: debug --- Config { @@ -148,6 +147,7 @@ Config { compression_level: Highest, }, ), + compiler_cache: None, }, tool_platform: None, cache: CacheConfig { diff --git a/crates/pixi_core/src/workspace/mod.rs b/crates/pixi_core/src/workspace/mod.rs index acc8dead82..f061bf7d15 100644 --- a/crates/pixi_core/src/workspace/mod.rs +++ b/crates/pixi_core/src/workspace/mod.rs @@ -668,6 +668,7 @@ impl Workspace { .with_pixi_install_reporter(rayon_primer.clone()) .with_pixi_solve_reporter(rayon_primer.clone()) .with_instantiate_backend_reporter(rayon_primer) + .with_compiler_cache(self.config.build.compiler_cache.clone()) .with_tool_platform(tool_platform, tool_virtual_packages)) } diff --git a/docs/build/backends/pixi-build-cmake.md b/docs/build/backends/pixi-build-cmake.md index 330b091964..044f8527ae 100644 --- a/docs/build/backends/pixi-build-cmake.md +++ b/docs/build/backends/pixi-build-cmake.md @@ -179,8 +179,38 @@ The CMake backend follows this build process: - `-DCMAKE_EXPORT_COMPILE_COMMANDS=ON`: Export compile commands for tooling - `-DBUILD_SHARED_LIBS=ON`: Build shared libraries by default - `-DPython_EXECUTABLE=$PYTHON`: Use the conda Python executable if it's part of the host dependencies. + - `-DCMAKE_C_COMPILER_LAUNCHER=sccache` / `-DCMAKE_CXX_COMPILER_LAUNCHER=sccache`: Set when `compiler-cache = "sccache"` is configured (see below). 3. **Build**: Executes `cmake --build` to compile the project 4. **Install**: Installs the built artifacts to the conda package +5. **Cache Statistics**: Displays `sccache --show-stats` if sccache is enabled + +## Compiler Cache + +You can speed up builds significantly by enabling a compiler cache. Configure it globally in +`~/.config/pixi/config.toml` or per-project in `.pixi/config.toml`: + +```toml title="config.toml" +[build] +compiler-cache = "sccache" +``` + +Or per-package in `pixi.toml`: + +```toml title="pixi.toml" +[package.build.config] +compiler-cache = "sccache" +``` + +In both cases the C/C++ compiler launchers are set to `sccache` and any `SCCACHE_*` environment +variables (e.g. for remote S3 cache configuration) are picked up from the environment and treated +as secrets. The two scopes differ in how `sccache` itself is provided: + +- **Per-package** (`pixi.toml`): `sccache` is added to the build dependencies automatically, so it + is recorded in the lockfile and resolves the same way on every machine. +- **Global / per-project** (`config.toml`): this is a per-machine preference, so it is *not* added + to the build dependencies — that would make the lockfile depend on who runs the resolve. Instead + `sccache` must already be on `PATH`; install it with `pixi global install sccache`. The build + fails with that hint if it is missing. ## Input tracking diff --git a/docs/build/backends/pixi-build-rust.md b/docs/build/backends/pixi-build-rust.md index 293a93c118..135e587d63 100644 --- a/docs/build/backends/pixi-build-rust.md +++ b/docs/build/backends/pixi-build-rust.md @@ -19,7 +19,7 @@ This backend automatically generates conda packages from Rust projects by: - **Using Cargo**: Leverages Rust's native build system for compilation and installation - **Cargo.toml Integration**: Automatically reads package metadata (name, version, description, license, etc.) from your `Cargo.toml` file when not specified in `pixi.toml` - **Cross-platform support**: Works consistently across Linux, macOS, and Windows -- **Optimization support**: Automatically detects and integrates with `sccache` for faster compilation +- **Optimization support**: Integrates with `sccache` for faster compilation when `compiler-cache = "sccache"` is configured - **OpenSSL integration**: Handles OpenSSL linking when available in the environment ## Basic Usage @@ -260,13 +260,39 @@ binaries = ["my-cli"] # Result for linux-64: only ["my-cli"] ``` +### `compiler-cache` + +- **Type**: `String` +- **Default**: unset + +The compiler cache to use. When set to `"sccache"`, the backend sets up `sccache` as +`RUSTC_WRAPPER`. It can also be set globally in `~/.config/pixi/config.toml` or per-project in +`.pixi/config.toml` under `[build]`; the per-package value takes precedence. + +```toml title="pixi.toml" +[package.build.config] +compiler-cache = "sccache" +``` + +How `sccache` is provided depends on where the setting comes from: + +- **Per-package** (`pixi.toml`): `sccache` is added to the build dependencies automatically, so it + is recorded in the lockfile and resolves the same way on every machine. +- **Global / per-project** (`config.toml`): this is a per-machine preference, so it is *not* added + to the build dependencies — that would make the lockfile depend on who runs the resolve. Instead + `sccache` must already be on `PATH`; install it with `pixi global install sccache`. The build + fails with that hint if it is missing. + +Any `SCCACHE_*` environment variables (e.g. for remote S3 cache configuration) are picked up from +the environment and treated as secrets. + ## Build Process The Rust backend follows this build process: 1. **Environment Setup**: Configures OpenSSL paths if available in the environment -2. **Compiler Caching**: Sets up `sccache` as `RUSTC_WRAPPER` if available for faster compilation +2. **Compiler Caching**: Sets up `sccache` as `RUSTC_WRAPPER` when `compiler-cache = "sccache"` is configured (see below) 3. **Build and Install**: Executes `cargo install` with the following default options: - `--locked`: Use the exact versions from `Cargo.lock` - `--root "$PREFIX"`: Install to the conda package prefix diff --git a/docs/reference/pixi_configuration.md b/docs/reference/pixi_configuration.md index 1a4eed41db..b5d31473b6 100644 --- a/docs/reference/pixi_configuration.md +++ b/docs/reference/pixi_configuration.md @@ -348,6 +348,60 @@ architecture for which there is fewer support for certain build backends. The virtual packages for the tool platform are detected from the current system. If the tool platform is for a different operating system than the current system, no virtual packages will be used. +## Build + +Configuration options that control how packages are built. + +### `build.compiler-cache` + +The compiler cache to use for all builds. When set, every build backend that supports compiler caching +(currently `pixi-build-cmake` and `pixi-build-rust`) will automatically use it. +Setting this globally avoids having to configure it per package. + +| Value | Description | +|---|---| +| `"sccache"` | Use [sccache](https://github.com/mozilla/sccache) as the compiler cache | + +```toml title="config.toml" +--8<-- "docs/source_files/pixi_config_tomls/main_config.toml:build" +``` + +The `compiler-cache` can also be set per-package in `pixi.toml` under `[package.build.config]`, +which takes precedence over the global and project-local config: + +```toml title="pixi.toml" +[package.build.config] +compiler-cache = "sccache" +``` + +!!! note "Global config does not lock the cache tool" + When `compiler-cache` is set here (globally or project-local) rather than per-package, the + cache tool is **not** added to a package's build dependencies — doing so would make the + lockfile change depending on whether the machine running the resolve has the cache configured. + The tool is used as a compiler launcher only and must already be available on `PATH`; install + it with `pixi global install sccache`. A per-package `compiler-cache` instead adds the tool to + the build dependencies so it is captured in the lockfile. The build fails with an install hint + if a globally-configured cache tool is not found. + +!!! tip "sccache credentials" + `SCCACHE_*` environment variables (e.g. `SCCACHE_BUCKET`, `SCCACHE_S3_KEY_ID`) are still + read from the environment for remote cache configuration. They are automatically treated + as secrets and not embedded in the build recipe. + +### `build.package-format` + +The package format and compression level to use when building conda packages. +The format is `` or `:`. + +| Format | Compression levels | +|---|---| +| `conda` (default) | `default`, `fast`, `max`, or a numeric zstd level (-7 to 22) | +| `tar-bz2` | `default`, `fast`, `max`, or a numeric bzip2 level (1-9) | + +```toml title="config.toml" +--8<-- "docs/source_files/pixi_config_tomls/main_config.toml:build" +``` + ### `cache` The `[cache]` table lets you redirect specific pixi caches independently. diff --git a/docs/source_files/pixi_config_tomls/main_config.toml b/docs/source_files/pixi_config_tomls/main_config.toml index 46516e06ac..10283f4b48 100644 --- a/docs/source_files/pixi_config_tomls/main_config.toml +++ b/docs/source_files/pixi_config_tomls/main_config.toml @@ -39,6 +39,19 @@ run-post-link-scripts = "false" # set to "insecure" to allow running post-link s tool-platform = "win-64" # force tools like build backends to be installed for a specific platform # --8<-- [end:tool-platform] +# --8<-- [start:build] +[build] +# The compiler cache to use for all builds (cmake, rust, …). +# Supported values: "sccache" +compiler-cache = "sccache" + +# The package format and compression level to use when building conda packages. +# Format: "" or ":" +# Formats: "conda" (default), "tar-bz2" +# Levels: "default", "max", "fast", or a numeric level (e.g. "conda:max", "tar-bz2:9") +package-format = "conda" +# --8<-- [end:build] + # --8<-- [start:cache] [cache] # Override for the cache root. Equivalent to setting PIXI_CACHE_DIR.