diff --git a/.semaphore/semaphore-scheduled-builds.yml b/.semaphore/semaphore-scheduled-builds.yml index 840e62b176..5ce986c52c 100644 --- a/.semaphore/semaphore-scheduled-builds.yml +++ b/.semaphore/semaphore-scheduled-builds.yml @@ -49,12 +49,29 @@ global_job_config: if [[ -n "${SEMAPHORE_GIT_BRANCH}" && -z "$SKIP_CACHE_RESTORE" ]]; then echo "Looking for cached working copy for branch ${SEMAPHORE_GIT_BRANCH}" gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - if gcloud storage cp "$gcs_branch_cache_dir/working-copy.tar.zstd" /tmp/working-copy.tar.zstd; then + # On master, use Semaphore's free built-in cache; other branches still + # use GCS. (For PRs, SEMAPHORE_GIT_BRANCH is the base branch, so PRs + # targeting master restore master's Semaphore cache.) Note: `cache + # restore` exits 0 even on a miss, so we test for the restored file. + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then use_sem_cache=true; else use_sem_cache=false; fi + rm -f /tmp/working-copy.tar.zstd + if ${use_sem_cache}; then + cache restore "working-copy-${SEMAPHORE_GIT_BRANCH}" || true + else + gcloud storage cp "$gcs_branch_cache_dir/working-copy.tar.zstd" /tmp/working-copy.tar.zstd || true + fi + if [[ -f /tmp/working-copy.tar.zstd ]]; then echo "Restoring cache for branch ${SEMAPHORE_GIT_BRANCH}" tar --use-compress-program=zstd -xf /tmp/working-copy.tar.zstd & tar_pid=$! downloaded_build_cache=false if [[ -n "$BUILD_CACHE_GROUP" && -n "${SEMAPHORE_GIT_PR_NUMBER}" ]]; then - if gcloud storage cp "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" /tmp/build-cache.tar.zstd; then + rm -f /tmp/build-cache.tar.zstd + if ${use_sem_cache}; then + cache restore "build-cache-${SEMAPHORE_GIT_BRANCH}-${BUILD_CACHE_GROUP}" || true + else + gcloud storage cp "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" /tmp/build-cache.tar.zstd || true + fi + if [[ -f /tmp/build-cache.tar.zstd ]]; then echo "Found build cache for group ${BUILD_CACHE_GROUP}" downloaded_build_cache=true fi @@ -94,23 +111,31 @@ global_job_config: mkdir -p artifacts fi - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin - # Best-effort: load calico/go-build from the GCS cache populated by the - # "Pull: go-build image" prerequisite job. Falls back to a Docker Hub pull - # on cache miss. Skipped on non-x86_64 / cross-compile agents since the - # cache is amd64-only. + # Ensure calico/go-build is present, cheaply. Pull from Docker Hub first + # (free egress) and only fall back to the GCS cache — populated by the + # "Pull: go-build image" prerequisite job — if the pull fails. This keeps + # GCS egress near zero in the common case while keeping GCS as a robustness + # fallback for the occasional Docker Hub pull failure (which we still see + # even with an authenticated session). Skipped on non-x86_64 / cross-compile + # agents since the cache is amd64-only. - |- if [[ $(uname -m) == "x86_64" && -z "$ARCH" ]]; then GO_BUILD_VER=$(make --no-print-directory -f metadata.mk -f - <<<'print:; @echo $(GO_BUILD_VER)' print) image="calico/go-build:${GO_BUILD_VER}" if ! docker image inspect "$image" >/dev/null 2>&1; then - cache_path="gs://${GCS_BUILD_CACHE_BUCKET}/images/calico-go-build-${GO_BUILD_VER}.tar.zst" - if gcloud storage cp "$cache_path" /tmp/go-build.tar.zst 2>/dev/null; then - echo "Loading ${image} from GCS cache" - zstd -d --rm -o /tmp/go-build.tar /tmp/go-build.tar.zst - docker load -i /tmp/go-build.tar - rm -f /tmp/go-build.tar + if docker pull "$image"; then + echo "Pulled ${image} from Docker Hub" else - echo "No GCS cache for ${image}, docker will pull from Docker Hub if needed" + echo "Docker Hub pull failed for ${image}, falling back to GCS cache" + cache_path="gs://${GCS_BUILD_CACHE_BUCKET}/images/calico-go-build-${GO_BUILD_VER}.tar.zst" + if gcloud storage cp "$cache_path" /tmp/go-build.tar.zst 2>/dev/null; then + echo "Loading ${image} from GCS cache" + zstd -d --rm -o /tmp/go-build.tar /tmp/go-build.tar.zst + docker load -i /tmp/go-build.tar + rm -f /tmp/go-build.tar + else + echo "No GCS fallback for ${image}; later docker use will retry the pull" + fi fi fi fi @@ -121,8 +146,16 @@ global_job_config: if [[ -z "${SEMAPHORE_GIT_PR_NUMBER}" && -n "$BUILD_CACHE_GROUP" && "$STORE_BUILD_CACHE" = "true" ]]; then echo "On branch, storing build cache group ${BUILD_CACHE_GROUP}" tar --use-compress-program="zstd -3" -cf /tmp/build-cache.tar.zstd go ${CALICO_DIR_NAME}/.go-pkg-cache - gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - gcloud storage cp /tmp/build-cache.tar.zstd "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then + # master uses Semaphore's free cache; delete-then-store guarantees a + # refresh each master build regardless of `cache store` overwrite semantics. + key="build-cache-${SEMAPHORE_GIT_BRANCH}-${BUILD_CACHE_GROUP}" + cache delete "$key" || true + cache store "$key" /tmp/build-cache.tar.zstd + else + gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" + gcloud storage cp /tmp/build-cache.tar.zstd "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" + fi fi - cd "$CALICO_DIR" - .semaphore/publish-artifacts @@ -339,8 +372,18 @@ blocks: commands: - cd .. - tar --use-compress-program="zstd -3" -cf /tmp/working-copy.tar.zstd $CALICO_DIR_NAME - - gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - - gcloud storage cp /tmp/working-copy.tar.zstd "$gcs_branch_cache_dir/working-copy.tar.zstd" + - |- + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then + # master uses Semaphore's free cache (other branches use GCS). This + # block only runs on branch builds (its `when: "branch =~ '.*'"`), + # so PRs never store here — they only restore, in the prologue. + # Cache keys are not overwritten, so delete-then-store to refresh. + cache delete "working-copy-${SEMAPHORE_GIT_BRANCH}" || true + cache store "working-copy-${SEMAPHORE_GIT_BRANCH}" /tmp/working-copy.tar.zstd + else + gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" + gcloud storage cp /tmp/working-copy.tar.zstd "$gcs_branch_cache_dir/working-copy.tar.zstd" + fi - name: "Build: nftables RPMs" # Producer for the patched nftables + libnftnl RPMs that calico/node and the # istio CNI install image consume. The image tag is content-addressed (sha @@ -386,11 +429,13 @@ blocks: commands: - .semaphore/scripts/build-nft-rpms.sh "${ARCH}" - name: "Pull: go-build image" - # Populate the GCS cache of calico/go-build so that every downstream job can - # docker-load it from GCS (faster, and avoids Docker Hub rate limits) rather - # than pulling cold from Docker Hub. The cache is keyed by GO_BUILD_VER so - # the tarball survives across workflows — the common case is a no-op - # `gcloud storage ls` on a cache hit. + # Populate the GCS cache of calico/go-build so it can serve as a fallback when + # a Docker Hub pull fails (which we still see occasionally even with an + # authenticated session). Jobs pull from Docker Hub first and only fall back to + # this GCS copy on failure (see the global prologue), so this is just an + # insurance upload — uploads are free and the common case is a no-op + # `gcloud storage ls` cache hit, keyed by GO_BUILD_VER so the tarball survives + # across workflows. run: when: "true" dependencies: [] diff --git a/.semaphore/semaphore.yml b/.semaphore/semaphore.yml index 5db3c58a3a..eacdd38792 100644 --- a/.semaphore/semaphore.yml +++ b/.semaphore/semaphore.yml @@ -49,12 +49,29 @@ global_job_config: if [[ -n "${SEMAPHORE_GIT_BRANCH}" && -z "$SKIP_CACHE_RESTORE" ]]; then echo "Looking for cached working copy for branch ${SEMAPHORE_GIT_BRANCH}" gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - if gcloud storage cp "$gcs_branch_cache_dir/working-copy.tar.zstd" /tmp/working-copy.tar.zstd; then + # On master, use Semaphore's free built-in cache; other branches still + # use GCS. (For PRs, SEMAPHORE_GIT_BRANCH is the base branch, so PRs + # targeting master restore master's Semaphore cache.) Note: `cache + # restore` exits 0 even on a miss, so we test for the restored file. + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then use_sem_cache=true; else use_sem_cache=false; fi + rm -f /tmp/working-copy.tar.zstd + if ${use_sem_cache}; then + cache restore "working-copy-${SEMAPHORE_GIT_BRANCH}" || true + else + gcloud storage cp "$gcs_branch_cache_dir/working-copy.tar.zstd" /tmp/working-copy.tar.zstd || true + fi + if [[ -f /tmp/working-copy.tar.zstd ]]; then echo "Restoring cache for branch ${SEMAPHORE_GIT_BRANCH}" tar --use-compress-program=zstd -xf /tmp/working-copy.tar.zstd & tar_pid=$! downloaded_build_cache=false if [[ -n "$BUILD_CACHE_GROUP" && -n "${SEMAPHORE_GIT_PR_NUMBER}" ]]; then - if gcloud storage cp "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" /tmp/build-cache.tar.zstd; then + rm -f /tmp/build-cache.tar.zstd + if ${use_sem_cache}; then + cache restore "build-cache-${SEMAPHORE_GIT_BRANCH}-${BUILD_CACHE_GROUP}" || true + else + gcloud storage cp "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" /tmp/build-cache.tar.zstd || true + fi + if [[ -f /tmp/build-cache.tar.zstd ]]; then echo "Found build cache for group ${BUILD_CACHE_GROUP}" downloaded_build_cache=true fi @@ -94,23 +111,31 @@ global_job_config: mkdir -p artifacts fi - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin - # Best-effort: load calico/go-build from the GCS cache populated by the - # "Pull: go-build image" prerequisite job. Falls back to a Docker Hub pull - # on cache miss. Skipped on non-x86_64 / cross-compile agents since the - # cache is amd64-only. + # Ensure calico/go-build is present, cheaply. Pull from Docker Hub first + # (free egress) and only fall back to the GCS cache — populated by the + # "Pull: go-build image" prerequisite job — if the pull fails. This keeps + # GCS egress near zero in the common case while keeping GCS as a robustness + # fallback for the occasional Docker Hub pull failure (which we still see + # even with an authenticated session). Skipped on non-x86_64 / cross-compile + # agents since the cache is amd64-only. - |- if [[ $(uname -m) == "x86_64" && -z "$ARCH" ]]; then GO_BUILD_VER=$(make --no-print-directory -f metadata.mk -f - <<<'print:; @echo $(GO_BUILD_VER)' print) image="calico/go-build:${GO_BUILD_VER}" if ! docker image inspect "$image" >/dev/null 2>&1; then - cache_path="gs://${GCS_BUILD_CACHE_BUCKET}/images/calico-go-build-${GO_BUILD_VER}.tar.zst" - if gcloud storage cp "$cache_path" /tmp/go-build.tar.zst 2>/dev/null; then - echo "Loading ${image} from GCS cache" - zstd -d --rm -o /tmp/go-build.tar /tmp/go-build.tar.zst - docker load -i /tmp/go-build.tar - rm -f /tmp/go-build.tar + if docker pull "$image"; then + echo "Pulled ${image} from Docker Hub" else - echo "No GCS cache for ${image}, docker will pull from Docker Hub if needed" + echo "Docker Hub pull failed for ${image}, falling back to GCS cache" + cache_path="gs://${GCS_BUILD_CACHE_BUCKET}/images/calico-go-build-${GO_BUILD_VER}.tar.zst" + if gcloud storage cp "$cache_path" /tmp/go-build.tar.zst 2>/dev/null; then + echo "Loading ${image} from GCS cache" + zstd -d --rm -o /tmp/go-build.tar /tmp/go-build.tar.zst + docker load -i /tmp/go-build.tar + rm -f /tmp/go-build.tar + else + echo "No GCS fallback for ${image}; later docker use will retry the pull" + fi fi fi fi @@ -121,8 +146,16 @@ global_job_config: if [[ -z "${SEMAPHORE_GIT_PR_NUMBER}" && -n "$BUILD_CACHE_GROUP" && "$STORE_BUILD_CACHE" = "true" ]]; then echo "On branch, storing build cache group ${BUILD_CACHE_GROUP}" tar --use-compress-program="zstd -3" -cf /tmp/build-cache.tar.zstd go ${CALICO_DIR_NAME}/.go-pkg-cache - gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - gcloud storage cp /tmp/build-cache.tar.zstd "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then + # master uses Semaphore's free cache; delete-then-store guarantees a + # refresh each master build regardless of `cache store` overwrite semantics. + key="build-cache-${SEMAPHORE_GIT_BRANCH}-${BUILD_CACHE_GROUP}" + cache delete "$key" || true + cache store "$key" /tmp/build-cache.tar.zstd + else + gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" + gcloud storage cp /tmp/build-cache.tar.zstd "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" + fi fi - cd "$CALICO_DIR" - .semaphore/publish-artifacts @@ -339,8 +372,18 @@ blocks: commands: - cd .. - tar --use-compress-program="zstd -3" -cf /tmp/working-copy.tar.zstd $CALICO_DIR_NAME - - gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - - gcloud storage cp /tmp/working-copy.tar.zstd "$gcs_branch_cache_dir/working-copy.tar.zstd" + - |- + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then + # master uses Semaphore's free cache (other branches use GCS). This + # block only runs on branch builds (its `when: "branch =~ '.*'"`), + # so PRs never store here — they only restore, in the prologue. + # Cache keys are not overwritten, so delete-then-store to refresh. + cache delete "working-copy-${SEMAPHORE_GIT_BRANCH}" || true + cache store "working-copy-${SEMAPHORE_GIT_BRANCH}" /tmp/working-copy.tar.zstd + else + gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" + gcloud storage cp /tmp/working-copy.tar.zstd "$gcs_branch_cache_dir/working-copy.tar.zstd" + fi - name: "Build: nftables RPMs" # Producer for the patched nftables + libnftnl RPMs that calico/node and the # istio CNI install image consume. The image tag is content-addressed (sha @@ -655,11 +698,13 @@ blocks: commands: - .semaphore/scripts/build-nft-rpms.sh "${ARCH}" - name: "Pull: go-build image" - # Populate the GCS cache of calico/go-build so that every downstream job can - # docker-load it from GCS (faster, and avoids Docker Hub rate limits) rather - # than pulling cold from Docker Hub. The cache is keyed by GO_BUILD_VER so - # the tarball survives across workflows — the common case is a no-op - # `gcloud storage ls` on a cache hit. + # Populate the GCS cache of calico/go-build so it can serve as a fallback when + # a Docker Hub pull fails (which we still see occasionally even with an + # authenticated session). Jobs pull from Docker Hub first and only fall back to + # this GCS copy on failure (see the global prologue), so this is just an + # insurance upload — uploads are free and the common case is a no-op + # `gcloud storage ls` cache hit, keyed by GO_BUILD_VER so the tarball survives + # across workflows. run: when: "true" dependencies: [] diff --git a/.semaphore/semaphore.yml.d/02-global_job_config.yml b/.semaphore/semaphore.yml.d/02-global_job_config.yml index 997b56cf1a..0ede25b2ef 100644 --- a/.semaphore/semaphore.yml.d/02-global_job_config.yml +++ b/.semaphore/semaphore.yml.d/02-global_job_config.yml @@ -33,12 +33,29 @@ global_job_config: if [[ -n "${SEMAPHORE_GIT_BRANCH}" && -z "$SKIP_CACHE_RESTORE" ]]; then echo "Looking for cached working copy for branch ${SEMAPHORE_GIT_BRANCH}" gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - if gcloud storage cp "$gcs_branch_cache_dir/working-copy.tar.zstd" /tmp/working-copy.tar.zstd; then + # On master, use Semaphore's free built-in cache; other branches still + # use GCS. (For PRs, SEMAPHORE_GIT_BRANCH is the base branch, so PRs + # targeting master restore master's Semaphore cache.) Note: `cache + # restore` exits 0 even on a miss, so we test for the restored file. + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then use_sem_cache=true; else use_sem_cache=false; fi + rm -f /tmp/working-copy.tar.zstd + if ${use_sem_cache}; then + cache restore "working-copy-${SEMAPHORE_GIT_BRANCH}" || true + else + gcloud storage cp "$gcs_branch_cache_dir/working-copy.tar.zstd" /tmp/working-copy.tar.zstd || true + fi + if [[ -f /tmp/working-copy.tar.zstd ]]; then echo "Restoring cache for branch ${SEMAPHORE_GIT_BRANCH}" tar --use-compress-program=zstd -xf /tmp/working-copy.tar.zstd & tar_pid=$! downloaded_build_cache=false if [[ -n "$BUILD_CACHE_GROUP" && -n "${SEMAPHORE_GIT_PR_NUMBER}" ]]; then - if gcloud storage cp "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" /tmp/build-cache.tar.zstd; then + rm -f /tmp/build-cache.tar.zstd + if ${use_sem_cache}; then + cache restore "build-cache-${SEMAPHORE_GIT_BRANCH}-${BUILD_CACHE_GROUP}" || true + else + gcloud storage cp "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" /tmp/build-cache.tar.zstd || true + fi + if [[ -f /tmp/build-cache.tar.zstd ]]; then echo "Found build cache for group ${BUILD_CACHE_GROUP}" downloaded_build_cache=true fi @@ -78,23 +95,31 @@ global_job_config: mkdir -p artifacts fi - echo $DOCKERHUB_PASSWORD | docker login --username "$DOCKERHUB_USERNAME" --password-stdin - # Best-effort: load calico/go-build from the GCS cache populated by the - # "Pull: go-build image" prerequisite job. Falls back to a Docker Hub pull - # on cache miss. Skipped on non-x86_64 / cross-compile agents since the - # cache is amd64-only. + # Ensure calico/go-build is present, cheaply. Pull from Docker Hub first + # (free egress) and only fall back to the GCS cache — populated by the + # "Pull: go-build image" prerequisite job — if the pull fails. This keeps + # GCS egress near zero in the common case while keeping GCS as a robustness + # fallback for the occasional Docker Hub pull failure (which we still see + # even with an authenticated session). Skipped on non-x86_64 / cross-compile + # agents since the cache is amd64-only. - |- if [[ $(uname -m) == "x86_64" && -z "$ARCH" ]]; then GO_BUILD_VER=$(make --no-print-directory -f metadata.mk -f - <<<'print:; @echo $(GO_BUILD_VER)' print) image="calico/go-build:${GO_BUILD_VER}" if ! docker image inspect "$image" >/dev/null 2>&1; then - cache_path="gs://${GCS_BUILD_CACHE_BUCKET}/images/calico-go-build-${GO_BUILD_VER}.tar.zst" - if gcloud storage cp "$cache_path" /tmp/go-build.tar.zst 2>/dev/null; then - echo "Loading ${image} from GCS cache" - zstd -d --rm -o /tmp/go-build.tar /tmp/go-build.tar.zst - docker load -i /tmp/go-build.tar - rm -f /tmp/go-build.tar + if docker pull "$image"; then + echo "Pulled ${image} from Docker Hub" else - echo "No GCS cache for ${image}, docker will pull from Docker Hub if needed" + echo "Docker Hub pull failed for ${image}, falling back to GCS cache" + cache_path="gs://${GCS_BUILD_CACHE_BUCKET}/images/calico-go-build-${GO_BUILD_VER}.tar.zst" + if gcloud storage cp "$cache_path" /tmp/go-build.tar.zst 2>/dev/null; then + echo "Loading ${image} from GCS cache" + zstd -d --rm -o /tmp/go-build.tar /tmp/go-build.tar.zst + docker load -i /tmp/go-build.tar + rm -f /tmp/go-build.tar + else + echo "No GCS fallback for ${image}; later docker use will retry the pull" + fi fi fi fi @@ -105,8 +130,16 @@ global_job_config: if [[ -z "${SEMAPHORE_GIT_PR_NUMBER}" && -n "$BUILD_CACHE_GROUP" && "$STORE_BUILD_CACHE" = "true" ]]; then echo "On branch, storing build cache group ${BUILD_CACHE_GROUP}" tar --use-compress-program="zstd -3" -cf /tmp/build-cache.tar.zstd go ${CALICO_DIR_NAME}/.go-pkg-cache - gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - gcloud storage cp /tmp/build-cache.tar.zstd "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then + # master uses Semaphore's free cache; delete-then-store guarantees a + # refresh each master build regardless of `cache store` overwrite semantics. + key="build-cache-${SEMAPHORE_GIT_BRANCH}-${BUILD_CACHE_GROUP}" + cache delete "$key" || true + cache store "$key" /tmp/build-cache.tar.zstd + else + gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" + gcloud storage cp /tmp/build-cache.tar.zstd "$gcs_branch_cache_dir/build-cache-${BUILD_CACHE_GROUP}.tar.zstd" + fi fi - cd "$CALICO_DIR" - .semaphore/publish-artifacts diff --git a/.semaphore/semaphore.yml.d/blocks/10-prerequisites.yml b/.semaphore/semaphore.yml.d/blocks/10-prerequisites.yml index 94de2ec67b..2f514a8e55 100644 --- a/.semaphore/semaphore.yml.d/blocks/10-prerequisites.yml +++ b/.semaphore/semaphore.yml.d/blocks/10-prerequisites.yml @@ -124,8 +124,18 @@ commands: - cd .. - tar --use-compress-program="zstd -3" -cf /tmp/working-copy.tar.zstd $CALICO_DIR_NAME - - gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" - - gcloud storage cp /tmp/working-copy.tar.zstd "$gcs_branch_cache_dir/working-copy.tar.zstd" + - |- + if [[ "${SEMAPHORE_GIT_BRANCH}" == "master" ]]; then + # master uses Semaphore's free cache (other branches use GCS). This + # block only runs on branch builds (its `when: "branch =~ '.*'"`), + # so PRs never store here — they only restore, in the prologue. + # Cache keys are not overwritten, so delete-then-store to refresh. + cache delete "working-copy-${SEMAPHORE_GIT_BRANCH}" || true + cache store "working-copy-${SEMAPHORE_GIT_BRANCH}" /tmp/working-copy.tar.zstd + else + gcs_branch_cache_dir="${GCS_CACHE_DIR}/${SEMAPHORE_GIT_BRANCH}" + gcloud storage cp /tmp/working-copy.tar.zstd "$gcs_branch_cache_dir/working-copy.tar.zstd" + fi - name: "Build: nftables RPMs" # Producer for the patched nftables + libnftnl RPMs that calico/node and the @@ -173,11 +183,13 @@ - .semaphore/scripts/build-nft-rpms.sh "${ARCH}" - name: "Pull: go-build image" - # Populate the GCS cache of calico/go-build so that every downstream job can - # docker-load it from GCS (faster, and avoids Docker Hub rate limits) rather - # than pulling cold from Docker Hub. The cache is keyed by GO_BUILD_VER so - # the tarball survives across workflows — the common case is a no-op - # `gcloud storage ls` on a cache hit. + # Populate the GCS cache of calico/go-build so it can serve as a fallback when + # a Docker Hub pull fails (which we still see occasionally even with an + # authenticated session). Jobs pull from Docker Hub first and only fall back to + # this GCS copy on failure (see the global prologue), so this is just an + # insurance upload — uploads are free and the common case is a no-op + # `gcloud storage ls` cache hit, keyed by GO_BUILD_VER so the tarball survives + # across workflows. run: when: "true" dependencies: []