diff --git a/packages/worker/src/__unit__/env.test.ts b/packages/worker/src/__unit__/env.test.ts index 84baaa7a24..d8f5cb5a73 100644 --- a/packages/worker/src/__unit__/env.test.ts +++ b/packages/worker/src/__unit__/env.test.ts @@ -3,12 +3,34 @@ import os from 'os'; import config from '../config'; import { getBuildEnv, getGradleMemoryOptions } from '../env'; +import { + DEFAULT_RUNTIME_SETTINGS, + applyRuntimeSettings, + resetRuntimeSettings, +} from '../runtimeSettings'; describe(getBuildEnv.name, () => { const originalSentryDsn = config.sentry.dsn; + const originalPlatform = process.platform; + const originalCacheUrls = { + EAS_BUILD_NPM_CACHE_URL: process.env.EAS_BUILD_NPM_CACHE_URL, + NVM_NODEJS_ORG_MIRROR: process.env.NVM_NODEJS_ORG_MIRROR, + EAS_BUILD_MAVEN_CACHE_URL: process.env.EAS_BUILD_MAVEN_CACHE_URL, + EAS_BUILD_COCOAPODS_CACHE_URL: process.env.EAS_BUILD_COCOAPODS_CACHE_URL, + }; + + beforeEach(() => { + applyRuntimeSettings(DEFAULT_RUNTIME_SETTINGS); + }); afterEach(() => { config.sentry.dsn = originalSentryDsn; + restoreEnv('EAS_BUILD_NPM_CACHE_URL', originalCacheUrls.EAS_BUILD_NPM_CACHE_URL); + restoreEnv('NVM_NODEJS_ORG_MIRROR', originalCacheUrls.NVM_NODEJS_ORG_MIRROR); + restoreEnv('EAS_BUILD_MAVEN_CACHE_URL', originalCacheUrls.EAS_BUILD_MAVEN_CACHE_URL); + restoreEnv('EAS_BUILD_COCOAPODS_CACHE_URL', originalCacheUrls.EAS_BUILD_COCOAPODS_CACHE_URL); + mockProcessPlatform(originalPlatform); + resetRuntimeSettings(); jest.restoreAllMocks(); }); @@ -62,6 +84,200 @@ describe(getBuildEnv.name, () => { expect(env.EXPO_PRECOMPILED_MODULES_PATH).toBeUndefined(); }); + it('adds precompiled modules env vars for iOS jobs when enabled', () => { + applyRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: true, maven: true }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: true, + }); + + const env = getBuildEnv({ + job: { + platform: Platform.IOS, + type: Workflow.GENERIC, + builderEnvironment: { + env: {}, + }, + } as any, + projectId: 'project-id', + metadata: { + buildProfile: 'production', + gitCommitHash: 'abc123', + username: 'expo-user', + } as any, + buildId: 'build-id', + }); + + expect(env.EAS_USE_PRECOMPILED_MODULES).toBe('1'); + }); + + it('does not add precompiled modules env vars for Android jobs when enabled', () => { + applyRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: true, maven: true }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: true, + }); + + const env = getBuildEnv({ + job: { + platform: Platform.ANDROID, + type: Workflow.MANAGED, + builderEnvironment: { + env: {}, + }, + username: 'expo-user', + } as any, + projectId: 'project-id', + metadata: { + buildProfile: 'production', + gitCommitHash: 'abc123', + username: 'expo-user', + } as any, + buildId: 'build-id', + }); + + expect(env.EAS_USE_PRECOMPILED_MODULES).toBeUndefined(); + }); + + it('leaves job-provided precompiled modules env vars in override position', () => { + applyRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: true, maven: true }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: true, + }); + + const job = { + platform: Platform.IOS, + type: Workflow.GENERIC, + builderEnvironment: { + env: { + EAS_USE_PRECOMPILED_MODULES: '0', + }, + }, + } as any; + const baseEnv = getBuildEnv({ + job, + projectId: 'project-id', + metadata: { + buildProfile: 'production', + gitCommitHash: 'abc123', + username: 'expo-user', + } as any, + buildId: 'build-id', + }); + + expect({ ...baseEnv, ...job.builderEnvironment.env }.EAS_USE_PRECOMPILED_MODULES).toBe('0'); + }); + + it('does not expose disabled Linux cache URLs in build env', () => { + mockProcessPlatform('linux'); + process.env.EAS_BUILD_NPM_CACHE_URL = 'https://npm.example'; + process.env.NVM_NODEJS_ORG_MIRROR = 'https://node.example'; + process.env.EAS_BUILD_MAVEN_CACHE_URL = 'https://maven.example'; + applyRuntimeSettings({ + caches: { + linux: { npm: false, nodejs: false, maven: false }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: false, + }); + + const env = getBuildEnv({ + job: { + platform: Platform.ANDROID, + type: Workflow.MANAGED, + builderEnvironment: { + env: {}, + }, + username: 'expo-user', + } as any, + projectId: 'project-id', + metadata: { + buildProfile: 'production', + gitCommitHash: 'abc123', + username: 'expo-user', + } as any, + buildId: 'build-id', + }); + + expect(env.NPM_CACHE_URL).toBeUndefined(); + expect(env.NVM_NODEJS_ORG_MIRROR).toBeUndefined(); + expect(env.EAS_BUILD_NPM_CACHE_URL).toBeUndefined(); + expect(env.EAS_BUILD_MAVEN_CACHE_URL).toBeUndefined(); + }); + + it('does not expose disabled Darwin cache URLs in build env', () => { + mockProcessPlatform('darwin'); + process.env.EAS_BUILD_NPM_CACHE_URL = 'https://npm.example'; + process.env.NVM_NODEJS_ORG_MIRROR = 'https://node.example'; + process.env.EAS_BUILD_COCOAPODS_CACHE_URL = 'https://pods.example'; + applyRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: true, maven: true }, + darwin: { npm: false, nodejs: false, cocoapods: false }, + }, + iosPrecompiledModules: false, + }); + + const env = getBuildEnv({ + job: { + platform: Platform.IOS, + type: Workflow.GENERIC, + builderEnvironment: { + env: {}, + }, + } as any, + projectId: 'project-id', + metadata: { + buildProfile: 'production', + gitCommitHash: 'abc123', + username: 'expo-user', + } as any, + buildId: 'build-id', + }); + + expect(env.NPM_CACHE_URL).toBeUndefined(); + expect(env.NVM_NODEJS_ORG_MIRROR).toBeUndefined(); + expect(env.EAS_BUILD_NPM_CACHE_URL).toBeUndefined(); + expect(env.EAS_BUILD_COCOAPODS_CACHE_URL).toBeUndefined(); + }); + + it('exposes enabled cache URLs inferred from worker environment variables', () => { + mockProcessPlatform('linux'); + process.env.EAS_BUILD_NPM_CACHE_URL = 'https://npm.example'; + process.env.NVM_NODEJS_ORG_MIRROR = 'https://node.example'; + process.env.EAS_BUILD_MAVEN_CACHE_URL = 'https://maven.example'; + + const env = getBuildEnv({ + job: { + platform: Platform.ANDROID, + type: Workflow.MANAGED, + builderEnvironment: { + env: {}, + }, + username: 'expo-user', + } as any, + projectId: 'project-id', + metadata: { + buildProfile: 'production', + gitCommitHash: 'abc123', + username: 'expo-user', + } as any, + buildId: 'build-id', + }); + + expect(env.NPM_CACHE_URL).toBe('https://npm.example'); + expect(env.EAS_BUILD_NPM_CACHE_URL).toBe('https://npm.example'); + expect(env.NVM_NODEJS_ORG_MIRROR).toBe('https://node.example'); + expect(env.EAS_BUILD_MAVEN_CACHE_URL).toBe('https://maven.example'); + }); + it('sizes Gradle memory options from total memory', () => { const totalMemory = jest.spyOn(os, 'totalmem'); @@ -84,3 +300,18 @@ describe(getBuildEnv.name, () => { }); }); }); + +function mockProcessPlatform(platform: NodeJS.Platform): void { + Object.defineProperty(process, 'platform', { + configurable: true, + value: platform, + }); +} + +function restoreEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } +} diff --git a/packages/worker/src/__unit__/runtimeEnvironment.test.ts b/packages/worker/src/__unit__/runtimeEnvironment.test.ts index 951766d076..261bb0fa4e 100644 --- a/packages/worker/src/__unit__/runtimeEnvironment.test.ts +++ b/packages/worker/src/__unit__/runtimeEnvironment.test.ts @@ -1,10 +1,18 @@ // @ts-nocheck import { Android, Ios, Job } from '@expo/eas-build-job'; +import templateFile from '@expo/template-file'; import spawn, { SpawnResult } from '@expo/turtle-spawn'; -import { pathExists } from 'fs-extra'; -import { prepareRuntimeEnvironment } from '../runtimeEnvironment'; +import { mkdirp, pathExists } from 'fs-extra'; + +import config from '../config'; +import { + prepareRuntimeEnvironment, + prepareRuntimeEnvironmentConfigFiles, +} from '../runtimeEnvironment'; +import { applyRuntimeSettings, resetRuntimeSettings } from '../runtimeSettings'; jest.mock('fs-extra'); +jest.mock('@expo/template-file'); jest.mock('@expo/turtle-spawn'); const spawnResult: SpawnResult = { @@ -31,10 +39,86 @@ const ctx = { const builderConfig: Ios.BuilderEnvironment | Android.BuilderEnvironment = {}; describe('prepareRuntimeEnvironment', () => { + const originalEnvironment = config.env; + const originalPlatform = process.platform; + const originalCacheUrls = { + EAS_BUILD_NPM_CACHE_URL: process.env.EAS_BUILD_NPM_CACHE_URL, + EAS_BUILD_MAVEN_CACHE_URL: process.env.EAS_BUILD_MAVEN_CACHE_URL, + }; + beforeEach(() => { jest.mocked(spawn).mockReset(); }); + afterEach(() => { + config.env = originalEnvironment; + restoreEnv('EAS_BUILD_NPM_CACHE_URL', originalCacheUrls.EAS_BUILD_NPM_CACHE_URL); + restoreEnv('EAS_BUILD_MAVEN_CACHE_URL', originalCacheUrls.EAS_BUILD_MAVEN_CACHE_URL); + mockProcessPlatform(originalPlatform); + resetRuntimeSettings(); + jest.restoreAllMocks(); + }); + + describe(prepareRuntimeEnvironmentConfigFiles.name, () => { + beforeEach(() => { + config.env = 'production'; + process.env.EAS_BUILD_NPM_CACHE_URL = 'https://npm.example'; + process.env.EAS_BUILD_MAVEN_CACHE_URL = 'https://maven.example'; + }); + + it('does not prepare disabled Linux cache config files', async () => { + mockProcessPlatform('linux'); + applyRuntimeSettings({ + caches: { + linux: { npm: false, nodejs: true, maven: false }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: false, + }); + + await prepareRuntimeEnvironmentConfigFiles(); + + expect(spawn).not.toHaveBeenCalledWith('npm', [ + 'config', + 'set', + 'registry', + 'https://npm.example', + ]); + expect(templateFile).not.toHaveBeenCalled(); + expect(mkdirp).not.toHaveBeenCalled(); + }); + + it('prepares enabled Linux cache config files', async () => { + mockProcessPlatform('linux'); + applyRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: true, maven: true }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: false, + }); + + await prepareRuntimeEnvironmentConfigFiles(); + + expect(spawn).toHaveBeenCalledWith('npm', [ + 'config', + 'set', + 'registry', + 'https://npm.example', + ]); + expect(templateFile).toHaveBeenCalledWith( + expect.stringContaining('yarnrc.yml'), + { URL: 'https://npm.example' }, + expect.stringContaining('.yarnrc.yml') + ); + expect(templateFile).toHaveBeenCalledWith( + expect.stringContaining('init.gradle'), + { URL: 'https://maven.example' }, + expect.stringContaining('init.gradle') + ); + }); + }); + describe('installNode', () => { describe('prepareRuntimeEnvironment', () => { beforeEach(() => { @@ -188,3 +272,18 @@ describe('prepareRuntimeEnvironment', () => { }); }); }); + +function mockProcessPlatform(platform: NodeJS.Platform): void { + Object.defineProperty(process, 'platform', { + configurable: true, + value: platform, + }); +} + +function restoreEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } +} diff --git a/packages/worker/src/__unit__/runtimeSettings.test.ts b/packages/worker/src/__unit__/runtimeSettings.test.ts new file mode 100644 index 0000000000..5d32b23af9 --- /dev/null +++ b/packages/worker/src/__unit__/runtimeSettings.test.ts @@ -0,0 +1,174 @@ +import { Response } from 'node-fetch'; +import fetch from 'node-fetch'; + +import { Environment } from '../constants'; +import { + DEFAULT_RUNTIME_SETTINGS, + applyRuntimeSettings, + getRuntimeSettings, + getRuntimeSettingsCacheUrl, + getRuntimeSettingsUrl, + loadRuntimeSettingsAsync, + parseRuntimeSettings, + resetRuntimeSettings, + shouldUseCache, +} from '../runtimeSettings'; + +jest.mock('node-fetch', () => { + const actual = jest.requireActual('node-fetch'); + return { + __esModule: true, + ...actual, + default: jest.fn(), + }; +}); + +describe('runtimeSettings', () => { + const logger = { + info: jest.fn(), + warn: jest.fn(), + }; + const originalCacheUrls = { + EAS_BUILD_NPM_CACHE_URL: process.env.EAS_BUILD_NPM_CACHE_URL, + NPM_CACHE_URL: process.env.NPM_CACHE_URL, + NVM_NODEJS_ORG_MIRROR: process.env.NVM_NODEJS_ORG_MIRROR, + EAS_BUILD_MAVEN_CACHE_URL: process.env.EAS_BUILD_MAVEN_CACHE_URL, + EAS_BUILD_COCOAPODS_CACHE_URL: process.env.EAS_BUILD_COCOAPODS_CACHE_URL, + }; + + afterEach(() => { + resetRuntimeSettings(); + jest.mocked(fetch).mockReset(); + jest.restoreAllMocks(); + restoreEnv('EAS_BUILD_NPM_CACHE_URL', originalCacheUrls.EAS_BUILD_NPM_CACHE_URL); + restoreEnv('NPM_CACHE_URL', originalCacheUrls.NPM_CACHE_URL); + restoreEnv('NVM_NODEJS_ORG_MIRROR', originalCacheUrls.NVM_NODEJS_ORG_MIRROR); + restoreEnv('EAS_BUILD_MAVEN_CACHE_URL', originalCacheUrls.EAS_BUILD_MAVEN_CACHE_URL); + restoreEnv('EAS_BUILD_COCOAPODS_CACHE_URL', originalCacheUrls.EAS_BUILD_COCOAPODS_CACHE_URL); + }); + + it('resolves hardcoded GCS URLs for staging and production only', () => { + expect(getRuntimeSettingsUrl(Environment.STAGING)).toBe( + 'https://storage.googleapis.com/eas-workflows-staging/runtime-settings.json' + ); + expect(getRuntimeSettingsUrl(Environment.PRODUCTION)).toBe( + 'https://storage.googleapis.com/eas-workflows-production/runtime-settings.json' + ); + expect(getRuntimeSettingsUrl(Environment.DEVELOPMENT)).toBeNull(); + expect(getRuntimeSettingsUrl(Environment.TEST)).toBeNull(); + }); + + it('uses defaults when remote settings are unavailable', async () => { + jest.mocked(fetch).mockResolvedValue(new Response('missing', { status: 404 })); + + await expect(loadRuntimeSettingsAsync(Environment.STAGING, logger)).resolves.toEqual( + DEFAULT_RUNTIME_SETTINGS + ); + expect(getRuntimeSettings()).toEqual(DEFAULT_RUNTIME_SETTINGS); + expect(logger.warn).toHaveBeenCalled(); + }); + + it('uses defaults when remote settings are invalid', async () => { + jest.mocked(fetch).mockResolvedValue( + new Response( + JSON.stringify({ + caches: { + linux: { npm: true, nodejs: true, maven: true, extra: true }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: false, + }), + { status: 200 } + ) + ); + + await expect(loadRuntimeSettingsAsync(Environment.STAGING, logger)).resolves.toEqual( + DEFAULT_RUNTIME_SETTINGS + ); + expect(getRuntimeSettings()).toEqual(DEFAULT_RUNTIME_SETTINGS); + expect(logger.warn).toHaveBeenCalled(); + }); + + it('validates accepted and rejected runtime settings JSON', () => { + expect( + parseRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: false, maven: true }, + darwin: { npm: true, nodejs: true, cocoapods: false }, + }, + iosPrecompiledModules: true, + }) + ).toEqual({ + caches: { + linux: { npm: true, nodejs: false, maven: true }, + darwin: { npm: true, nodejs: true, cocoapods: false }, + }, + iosPrecompiledModules: true, + }); + + expect(() => + parseRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: true, maven: true }, + darwin: { npm: true, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: 'enabled', + }) + ).toThrow(); + }); + + it('gates caches per worker platform', () => { + applyRuntimeSettings({ + caches: { + linux: { npm: false, nodejs: true, maven: false }, + darwin: { npm: true, nodejs: false, cocoapods: false }, + }, + iosPrecompiledModules: false, + }); + + expect(shouldUseCache('npm', 'linux')).toBe(false); + expect(shouldUseCache('maven', 'linux')).toBe(false); + expect(shouldUseCache('nodejs', 'linux')).toBe(true); + + expect(shouldUseCache('nodejs', 'darwin')).toBe(false); + expect(shouldUseCache('cocoapods', 'darwin')).toBe(false); + expect(shouldUseCache('npm', 'darwin')).toBe(true); + expect(shouldUseCache('maven', 'darwin')).toBe(false); + expect(shouldUseCache('cocoapods', 'linux')).toBe(false); + }); + + it('requires runtime settings to be loaded before use', () => { + resetRuntimeSettings(); + + expect(() => getRuntimeSettings()).toThrow('Runtime settings must be loaded before use'); + expect(() => shouldUseCache('npm')).toThrow('Runtime settings must be loaded before use'); + }); + + it('infers enabled cache URLs from environment variables', () => { + process.env.EAS_BUILD_NPM_CACHE_URL = 'https://npm.example'; + process.env.NVM_NODEJS_ORG_MIRROR = 'https://node.example'; + process.env.EAS_BUILD_MAVEN_CACHE_URL = 'https://maven.example'; + process.env.EAS_BUILD_COCOAPODS_CACHE_URL = 'https://pods.example'; + applyRuntimeSettings({ + caches: { + linux: { npm: true, nodejs: true, maven: false }, + darwin: { npm: false, nodejs: true, cocoapods: true }, + }, + iosPrecompiledModules: false, + }); + + expect(getRuntimeSettingsCacheUrl('npm', 'linux')).toBe('https://npm.example'); + expect(getRuntimeSettingsCacheUrl('nodejs', 'linux')).toBe('https://node.example'); + expect(getRuntimeSettingsCacheUrl('maven', 'linux')).toBeNull(); + expect(getRuntimeSettingsCacheUrl('cocoapods', 'darwin')).toBe('https://pods.example'); + expect(getRuntimeSettingsCacheUrl('npm', 'darwin')).toBeNull(); + }); +}); + +function restoreEnv(key: string, value: string | undefined): void { + if (value === undefined) { + delete process.env[key]; + } else { + process.env[key] = value; + } +} diff --git a/packages/worker/src/config.ts b/packages/worker/src/config.ts index 9532519f2f..7f79a6d422 100644 --- a/packages/worker/src/config.ts +++ b/packages/worker/src/config.ts @@ -82,22 +82,6 @@ export default { dataPlaneURL: env('RUDDERSTACK_DATA_PLANE_URL', { defaultValue: null }), writeKey: env('RUDDERSTACK_WRITE_KEY', { defaultValue: null }), }, - npmCacheUrl: env('WORKER_RUNTIME_CONFIG_BASE64', { - transform: createBase64EnvTransformer('npmCacheUrl'), - defaultValue: null, - }), - nodeJsCacheUrl: env('WORKER_RUNTIME_CONFIG_BASE64', { - transform: createBase64EnvTransformer('nodeJsCacheUrl'), - defaultValue: null, - }), - mavenCacheUrl: env('WORKER_RUNTIME_CONFIG_BASE64', { - transform: createBase64EnvTransformer('mavenCacheUrl'), - defaultValue: null, - }), - cocoapodsCacheUrl: env('WORKER_RUNTIME_CONFIG_BASE64', { - transform: createBase64EnvTransformer('cocoapodsCacheUrl'), - defaultValue: null, - }), runMetricsServer: env('WORKER_RUNTIME_CONFIG_BASE64', { transform: createBase64EnvTransformer('runMetricsServer'), defaultValue: null, diff --git a/packages/worker/src/env.ts b/packages/worker/src/env.ts index 851e62e625..eff78b3e5d 100644 --- a/packages/worker/src/env.ts +++ b/packages/worker/src/env.ts @@ -7,6 +7,11 @@ import path from 'path'; import config from './config'; import { Environment } from './constants'; import { androidImagesWithJavaVersionLowerThen11 } from './external/turtle'; +import { + getRuntimeSettings, + getRuntimeSettingsCacheUrl, + getRuntimeSettingsCacheUrlEnvVars, +} from './runtimeSettings'; import { getAccessedEnvs } from './utils/env'; // keep in sync with local-build-plugin env vars @@ -35,9 +40,14 @@ export function getBuildEnv({ setEnv(env, 'EAS_BUILD_PLATFORM', job.platform); setEnv(env, 'EAS_CLI_SENTRY_DSN', config.sentry.dsn); // NPM_CACHE_URL is deprecated - setEnv(env, 'NPM_CACHE_URL', config.npmCacheUrl); - setEnv(env, 'NVM_NODEJS_ORG_MIRROR', config.nodeJsCacheUrl); - setEnv(env, 'EAS_BUILD_NPM_CACHE_URL', config.npmCacheUrl); + const npmCacheUrl = getRuntimeSettingsCacheUrl('npm'); + const nodeJsCacheUrl = getRuntimeSettingsCacheUrl('nodejs'); + const mavenCacheUrl = getRuntimeSettingsCacheUrl('maven'); + const cocoapodsCacheUrl = getRuntimeSettingsCacheUrl('cocoapods'); + + setEnv(env, 'NPM_CACHE_URL', npmCacheUrl); + setEnv(env, 'NVM_NODEJS_ORG_MIRROR', nodeJsCacheUrl); + setEnv(env, 'EAS_BUILD_NPM_CACHE_URL', npmCacheUrl); setEnv(env, 'EAS_BUILD_PROFILE', metadata.buildProfile); setEnv(env, 'EAS_BUILD_GIT_COMMIT_HASH', metadata.gitCommitHash); setEnv(env, 'EAS_BUILD_ID', buildId); @@ -47,8 +57,11 @@ export function getBuildEnv({ const runnerPlatform = job.platform; if (runnerPlatform === Platform.IOS) { - setEnv(env, 'EAS_BUILD_COCOAPODS_CACHE_URL', config.cocoapodsCacheUrl); + setEnv(env, 'EAS_BUILD_COCOAPODS_CACHE_URL', cocoapodsCacheUrl); setEnv(env, 'COMPILER_INDEX_STORE_ENABLE', 'NO'); + if (shouldUsePrecompiledModules(job)) { + setEnv(env, 'EAS_USE_PRECOMPILED_MODULES', '1'); + } if (job.builderEnvironment?.env?.EAS_USE_CACHE === '1') { setEnv(env, 'USE_CCACHE', '1'); @@ -69,7 +82,7 @@ export function getBuildEnv({ setEnv(env, 'ANDROID_CCACHE', binPath); } } - setEnv(env, 'EAS_BUILD_MAVEN_CACHE_URL', config.mavenCacheUrl); + setEnv(env, 'EAS_BUILD_MAVEN_CACHE_URL', mavenCacheUrl); } if (config.env !== Environment.TEST) { @@ -135,8 +148,20 @@ export function getBuildEnv({ return env; } +function shouldUsePrecompiledModules(job: Job): boolean { + if (job.platform !== Platform.IOS) { + return false; + } + + return getRuntimeSettings().iosPrecompiledModules; +} + function getFilteredEnv(): Env { - const envToFilter = [...getAccessedEnvs(), 'KUBERNETES_*']; + const envToFilter = [ + ...getAccessedEnvs(), + ...getRuntimeSettingsCacheUrlEnvVars(), + 'KUBERNETES_*', + ]; const envToReturn = micromatch( Object.keys(process.env), envToFilter.map(env => `!${env}`) diff --git a/packages/worker/src/external/turtle.ts b/packages/worker/src/external/turtle.ts index 371897a029..1d0ee8165e 100644 --- a/packages/worker/src/external/turtle.ts +++ b/packages/worker/src/external/turtle.ts @@ -24,10 +24,6 @@ export namespace Worker { type JobRunWorkerRuntimeConfig = { gcsSignedUploadUrlForLogs: GCS.SignedUrl; - nodeJsCacheUrl: string | undefined; - npmCacheUrl: string | undefined; - mavenCacheUrl: string | undefined; - cocoapodsCacheUrl: string | undefined; runMetricsServer: boolean; type: 'jobRun'; @@ -42,10 +38,6 @@ export namespace Worker { gcsSignedUploadUrlForBuildCache?: GCS.SignedUrl; gcsSignedBuildCacheDownloadUrl?: string; - nodeJsCacheUrl: string | undefined; - npmCacheUrl: string | undefined; - mavenCacheUrl: string | undefined; - cocoapodsCacheUrl: string | undefined; runMetricsServer: boolean; type?: never; diff --git a/packages/worker/src/main.ts b/packages/worker/src/main.ts index fcceabffb1..5b50346e46 100644 --- a/packages/worker/src/main.ts +++ b/packages/worker/src/main.ts @@ -2,11 +2,13 @@ import config from './config'; import logger from './logger'; import { startServer } from './metricsServer'; import { prepareRuntimeEnvironmentConfigFiles } from './runtimeEnvironment'; +import { loadRuntimeSettingsAsync } from './runtimeSettings'; import sentry from './sentry'; import { prepareWorkingdir } from './workingdir'; import startWsServer from './ws'; async function main(): Promise { + await loadRuntimeSettingsAsync(config.env, logger); await prepareRuntimeEnvironmentConfigFiles(); await prepareWorkingdir(); startWsServer(); diff --git a/packages/worker/src/runtimeEnvironment.ts b/packages/worker/src/runtimeEnvironment.ts index d5d9361240..aa10fc55da 100644 --- a/packages/worker/src/runtimeEnvironment.ts +++ b/packages/worker/src/runtimeEnvironment.ts @@ -8,6 +8,7 @@ import path from 'path'; import { v4 as uuidv4 } from 'uuid'; import config from './config'; +import { getRuntimeSettingsCacheUrl } from './runtimeSettings'; class SystemDepsInstallError extends errors.UserError { constructor(dependency: string) { @@ -23,25 +24,28 @@ export async function prepareRuntimeEnvironmentConfigFiles(): Promise { return; } - if (config.npmCacheUrl) { + const npmCacheUrl = getRuntimeSettingsCacheUrl('npm'); + const mavenCacheUrl = getRuntimeSettingsCacheUrl('maven'); + + if (npmCacheUrl) { // create ~/.npmrc - await spawn('npm', ['config', 'set', 'registry', config.npmCacheUrl]); + await spawn('npm', ['config', 'set', 'registry', npmCacheUrl]); // create ~/.yarnrc.yml await templateFile( path.join(__dirname, '../src/templates/yarnrc.yml'), { - URL: config.npmCacheUrl, + URL: npmCacheUrl, }, path.join(os.homedir(), '.yarnrc.yml') ); } - if (config.mavenCacheUrl) { + if (mavenCacheUrl) { await fs.mkdirp(path.join(os.homedir(), '.gradle')); await templateFile( path.join(__dirname, '../src/templates/init.gradle'), { - URL: config.mavenCacheUrl, + URL: mavenCacheUrl, }, path.join(os.homedir(), '.gradle/init.gradle') ); diff --git a/packages/worker/src/runtimeSettings.ts b/packages/worker/src/runtimeSettings.ts new file mode 100644 index 0000000000..61fdd52d12 --- /dev/null +++ b/packages/worker/src/runtimeSettings.ts @@ -0,0 +1,167 @@ +import fetch from 'node-fetch'; +import { z } from 'zod'; + +import { Environment } from './constants'; + +const RUNTIME_SETTINGS_FILENAME = 'runtime-settings.json'; + +const CACHE_URL_ENV_VARS = { + npm: ['EAS_BUILD_NPM_CACHE_URL', 'NPM_CACHE_URL'], + nodejs: ['NVM_NODEJS_ORG_MIRROR'], + maven: ['EAS_BUILD_MAVEN_CACHE_URL'], + cocoapods: ['EAS_BUILD_COCOAPODS_CACHE_URL'], +} as const satisfies Record; + +export const RuntimeSettingsSchema = z + .object({ + caches: z + .object({ + linux: z + .object({ + npm: z.boolean(), + nodejs: z.boolean(), + maven: z.boolean(), + }) + .strict(), + darwin: z + .object({ + npm: z.boolean(), + nodejs: z.boolean(), + cocoapods: z.boolean(), + }) + .strict(), + }) + .strict(), + iosPrecompiledModules: z.boolean(), + }) + .strict(); + +export type RuntimeSettings = z.infer; +export type RuntimeSettingsCacheName = 'npm' | 'nodejs' | 'maven' | 'cocoapods'; +export type RuntimeSettingsCacheUrlEnvVar = + (typeof CACHE_URL_ENV_VARS)[RuntimeSettingsCacheName][number]; + +export const DEFAULT_RUNTIME_SETTINGS: RuntimeSettings = { + caches: { + linux: { + npm: true, + nodejs: true, + maven: true, + }, + darwin: { + npm: true, + nodejs: true, + cocoapods: true, + }, + }, + iosPrecompiledModules: false, +}; + +let runtimeSettings: RuntimeSettings | null = null; + +type Logger = { + info: (obj: object, msg?: string) => void; + warn: (obj: object, msg?: string) => void; +}; + +export function getRuntimeSettingsUrl(environment: Environment): string | null { + if (environment !== Environment.STAGING && environment !== Environment.PRODUCTION) { + return null; + } + + return `https://storage.googleapis.com/eas-workflows-${environment}/${RUNTIME_SETTINGS_FILENAME}`; +} + +export async function loadRuntimeSettingsAsync( + environment: Environment, + logger: Logger +): Promise { + const url = getRuntimeSettingsUrl(environment); + if (!url) { + applyRuntimeSettings(DEFAULT_RUNTIME_SETTINGS); + return getRuntimeSettings(); + } + + try { + const response = await fetch(url, { + signal: AbortSignal.timeout(5000), + }); + + if (!response.ok) { + logger.warn( + { url, status: response.status }, + 'Failed to fetch worker runtime settings, using defaults' + ); + applyRuntimeSettings(DEFAULT_RUNTIME_SETTINGS); + return getRuntimeSettings(); + } + + const settings = parseRuntimeSettings(await response.json()); + applyRuntimeSettings(settings); + logger.info({ url, settings }, 'Loaded worker runtime settings'); + return getRuntimeSettings(); + } catch (err) { + logger.warn({ err, url }, 'Failed to load worker runtime settings, using defaults'); + applyRuntimeSettings(DEFAULT_RUNTIME_SETTINGS); + return getRuntimeSettings(); + } +} + +export function parseRuntimeSettings(value: unknown): RuntimeSettings { + return RuntimeSettingsSchema.parse(value); +} + +export function applyRuntimeSettings(settings: RuntimeSettings): void { + runtimeSettings = settings; +} + +export function resetRuntimeSettings(): void { + runtimeSettings = null; +} + +export function getRuntimeSettings(): RuntimeSettings { + if (!runtimeSettings) { + throw new Error('Runtime settings must be loaded before use'); + } + return runtimeSettings; +} + +export function shouldUseCache( + cacheName: RuntimeSettingsCacheName, + platform: NodeJS.Platform = process.platform +): boolean { + const settings = getRuntimeSettings(); + if (platform === 'darwin') { + if (cacheName === 'maven') { + return false; + } + return settings.caches.darwin[cacheName]; + } + + if (cacheName === 'cocoapods') { + return false; + } + return settings.caches.linux[cacheName]; +} + +export function getRuntimeSettingsCacheUrl( + cacheName: RuntimeSettingsCacheName, + platform: NodeJS.Platform = process.platform +): string | null { + if (!shouldUseCache(cacheName, platform)) { + return null; + } + + for (const envVar of CACHE_URL_ENV_VARS[cacheName]) { + const value = process.env[envVar]; + if (value) { + return value; + } + } + + return null; +} + +export function getRuntimeSettingsCacheUrlEnvVars(): RuntimeSettingsCacheUrlEnvVar[] { + return Object.values(CACHE_URL_ENV_VARS).flat(); +} diff --git a/scripts/worker-runtime-settings b/scripts/worker-runtime-settings new file mode 100755 index 0000000000..369ac422d0 --- /dev/null +++ b/scripts/worker-runtime-settings @@ -0,0 +1,117 @@ +#!/usr/bin/env bash + +set -eo pipefail + +CACHE_CONTROL="no-cache, no-store, must-revalidate" + +usage() { + cat </dev/null 2>&1; then + echo "gsutil is required." >&2 + exit 1 + fi +} + +parse_env() { + local env="" + while [[ $# -gt 0 ]]; do + case "$1" in + --env) + env="${2:-}" + shift 2 + ;; + *) + shift + ;; + esac + done + + case "$env" in + staging|production) + echo "$env" + ;; + *) + echo "Expected --env staging|production." >&2 + exit 1 + ;; + esac +} + +object_uri() { + local env="$1" + echo "gs://eas-workflows-${env}/runtime-settings.json" +} + +confirm_production() { + local env="$1" + if [[ "$env" != "production" ]]; then + return + fi + + local confirmation="" + read -r -p 'Type "production" to upload production runtime settings: ' confirmation + if [[ "$confirmation" != "production" ]]; then + echo "Production upload canceled." >&2 + exit 1 + fi +} + +upload_file() { + local env="$1" + local file="$2" + local uri + uri="$(object_uri "$env")" + + confirm_production "$env" + gsutil \ + -h "Cache-Control:${CACHE_CONTROL}" \ + -h "Content-Type:application/json" \ + cp "$file" "$uri" + + local downloaded + downloaded="$(mktemp)" + gsutil cp "$uri" "$downloaded" >/dev/null + if ! cmp -s "$file" "$downloaded"; then + rm -f "$downloaded" + echo "Read-after-write check failed: uploaded file differs from remote object." >&2 + exit 1 + fi + rm -f "$downloaded" +} + +main() { + require_gsutil + + local command="${1:-}" + shift || true + + case "$command" in + edit) + local env tmpfile editor + env="$(parse_env "$@")" + tmpfile="$(mktemp)" + gsutil cp "$(object_uri "$env")" "$tmpfile" + editor="${EDITOR:-vi}" + "$editor" "$tmpfile" + upload_file "$env" "$tmpfile" + rm -f "$tmpfile" + ;; + -h|--help|"") + usage + ;; + *) + usage >&2 + exit 1 + ;; + esac +} + +main "$@"