e2e improvements

.github/workflows/build-images.yaml

@@ -135,6 +135,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Build operator installer manifest
         if: ${{ matrix.generate_installer && steps.check.outputs.skip != 'true' }}

.github/workflows/controller-kind.yaml

@@ -15,9 +15,15 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Set up Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5
+        with:
+          go-version-file: controller/go.mod
+          cache-dependency-path: controller/go.sum
+
       - name: Run make deploy
         working-directory: controller
-        run: make deploy
+        run: make deploy-ci -j --output-sync=target
 
   e2e-test-operator:
     runs-on: ubuntu-latest
@@ -27,6 +33,12 @@ jobs:
         with:
           fetch-depth: 0
 
+      - name: Set up Go
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5
+        with:
+          go-version-file: controller/go.mod
+          cache-dependency-path: controller/go.sum
+
       - name: Run operator e2e test
         working-directory: controller
-        run: make test-operator-e2e
+        run: make test-operator-e2e-ci -j --output-sync=target

.github/workflows/e2e.yaml

@@ -10,6 +10,14 @@ permissions:
 
 env:
   CONTAINER_TOOL: docker
+  # Retry a failed spec once before failing the job. Retries are reported as
+  # flakes in the ginkgo summary, so this hides nothing; it only stops a single
+  # infrastructure hiccup from failing an unrelated PR.
+  E2E_FLAKE_ATTEMPTS: "2"
+  # Run the non-Serial containers two at a time. The runner has 4 CPUs and the
+  # suite spends most of its time waiting on the cluster, so a second process
+  # overlaps the two long containers (hooks, core) rather than competing for CPU.
+  E2E_PROCS: "2"
 
 jobs:
   changes:
@@ -68,6 +76,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Cache controller image
         id: cache
@@ -107,6 +116,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Cache operator artifacts
         id: cache
@@ -151,6 +161,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Cache exporterset-controller image
         id: cache
@@ -310,6 +321,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Load e2e artifacts
         uses: ./.github/actions/load-e2e-artifacts
@@ -327,6 +339,16 @@ jobs:
             sudo modprobe "$mod" 2>/dev/null || true
           done
 
+      - name: Cache controller tools
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: controller/bin
+          key: controller-tools-${{ matrix.arch }}-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }}
+        env:
+          KIND_VERSION: v0.27.0
+          KUSTOMIZE_VERSION: v5.4.1
+          GRPCURL_VERSION: v1.9.2
+
       - name: Setup e2e test environment
         run: make e2e-setup
         env:
@@ -374,6 +396,17 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
+
+      - name: Cache controller tools
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: controller/bin
+          key: controller-tools-amd64-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }}
+        env:
+          KIND_VERSION: v0.27.0
+          KUSTOMIZE_VERSION: v5.4.1
+          GRPCURL_VERSION: v1.9.2
 
       - name: Setup compat environment (old controller v0.8.1)
         run: make e2e-compat-setup COMPAT_SCENARIO=old-controller
@@ -416,12 +449,23 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Load e2e artifacts
         uses: ./.github/actions/load-e2e-artifacts
         with:
           arch: amd64
 
+      - name: Cache controller tools
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: controller/bin
+          key: controller-tools-amd64-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }}
+        env:
+          KIND_VERSION: v0.27.0
+          KUSTOMIZE_VERSION: v5.4.1
+          GRPCURL_VERSION: v1.9.2
+
       - name: Setup compat environment (old client v0.7.4)
         run: make e2e-compat-setup COMPAT_SCENARIO=old-client
         env:

.github/workflows/lint.yaml

@@ -55,6 +55,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Run go linter
         working-directory: controller

.github/workflows/python-tests.yaml

@@ -66,12 +66,6 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Free up runner disk space
-        uses: mathio/gha-cleanup@aca3d43a05bf564f5fc69acb929a65b01797f22b # v1.1.2
-        with:
-          remove-browsers: true
-          verbose: true
-
       - id: uv
         run: echo "version=$(cat .uv-version)" >> "$GITHUB_OUTPUT"
       - name: Install uv
@@ -96,19 +90,8 @@ jobs:
           sudo chmod 0666 /dev/kvm /dev/vhost-vsock /dev/vhost-net
 
           sudo apt-get update
-          sudo apt-get install -y qemu-system-arm qemu-system-x86 rpm2cpio cpio
-
-      - name: Install libgpiod-dev (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y libgpiod-dev liblgpio-dev
-
-      - name: Install nftables and dnsmasq (Linux)
-        if: runner.os == 'Linux'
-        run: |
-          sudo apt-get update
-          sudo apt-get install -y nftables dnsmasq-base isc-dhcp-client dnsutils
+          sudo apt-get install -y qemu-system-arm qemu-system-x86 rpm2cpio cpio \
+            libgpiod-dev liblgpio-dev nftables dnsmasq-base isc-dhcp-client dnsutils
 
       - name: Load kernel modules for DUT network tests (Linux)
         if: runner.os == 'Linux'
@@ -145,7 +128,7 @@ jobs:
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
         with:
           path: python/packages/jumpstarter-driver-qemu/images
-          key: fedora-cloud-43-1.6
+          key: fedora-cloud-43-1.6-preconfigured-v2
 
       - name: Download Fedora Cloud images
         if: steps.cache-fedora-cloud-images.outputs.cache-hit != 'true'
@@ -155,12 +138,39 @@ jobs:
               "https://iad.mirror.rackspace.com/fedora/releases/43/Cloud/${arch}/images/Fedora-Cloud-Base-Generic-43-1.6.${arch}.qcow2"
           done
 
+      - name: Pre-configure Fedora Cloud images for fast boot
+        if: runner.os == 'Linux' && steps.cache-fedora-cloud-images.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get install -y libguestfs-tools
+          arch=x86_64
+          img="python/packages/jumpstarter-driver-qemu/images/Fedora-Cloud-Base-Generic-43-1.6.${arch}.qcow2"
+          sudo virt-customize --no-network -a "$img" \
+            --hostname demo \
+            --run-command 'useradd -m -s /bin/bash jumpstarter' \
+            --password jumpstarter:password:password \
+            --run-command 'echo "jumpstarter ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/jumpstarter' \
+            --run-command 'systemctl disable cloud-init cloud-init-local cloud-config cloud-final' \
+            --run-command 'rm -rf /var/lib/cloud' \
+            --run-command 'ssh-keygen -A' \
+            --run-command 'sed -i "s/^#*PasswordAuthentication.*/PasswordAuthentication yes/" /etc/ssh/sshd_config' \
+            --run-command 'systemctl enable sshd' \
+            --selinux-relabel
+
       - name: Run pytest
         working-directory: python
         env:
-          PYTEST_ADDOPTS: "--cov-report=xml --log-level=CRITICAL --log-cli-level=CRITICAL"
+          PYTEST_ADDOPTS: "--log-level=CRITICAL --log-cli-level=CRITICAL"
         run: |
-          make test -j4 LOGS_DIR=${{ runner.temp }}/test-logs
+          changed_pkgs=$(git diff --name-only --relative origin/${{ github.base_ref || 'main' }}...HEAD -- packages/ \
+            | sed -n 's|^packages/\([^/]*\)/.*|\1|p' | sort -u | tr '\n' ' ')
+          echo "Coverage-enabled packages: ${changed_pkgs:-all (fallback)}"
+          if [ -z "$changed_pkgs" ]; then
+            # Fallback: workflow_dispatch or merge_group without a base ref
+            export PYTEST_ADDOPTS="--cov=. --cov-report=xml $PYTEST_ADDOPTS"
+            make test -j8 LOGS_DIR=${{ runner.temp }}/test-logs
+          else
+            make test -j8 LOGS_DIR=${{ runner.temp }}/test-logs COV_PACKAGES="$changed_pkgs"
+          fi
 
       - name: Upload test logs
         if: always()

.github/workflows/release-operator-installer.yaml

@@ -20,6 +20,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Build operator installer manifest
         env:

controller/Containerfile.ci

@@ -0,0 +1,5 @@
+FROM registry.access.redhat.com/ubi9/ubi-micro:9.8-1784702951@sha256:b1e86b97028b8fcfb6d85f997c39e6b6b67496163ef8d80d243220a4918e8bef
+WORKDIR /
+COPY manager router ./
+USER 65532:65532
+ENTRYPOINT ["/manager"]

controller/Containerfile.exporter-set-controller.ci

@@ -0,0 +1,5 @@
+FROM registry.access.redhat.com/ubi9/ubi-micro:9.8-1784702951@sha256:b1e86b97028b8fcfb6d85f997c39e6b6b67496163ef8d80d243220a4918e8bef
+WORKDIR /
+COPY exporter-set-controller .
+USER 65532:65532
+ENTRYPOINT ["/exporter-set-controller"]

controller/Containerfile.operator.ci

@@ -0,0 +1,5 @@
+FROM registry.access.redhat.com/ubi9/ubi-micro:9.8-1784702951@sha256:b1e86b97028b8fcfb6d85f997c39e6b6b67496163ef8d80d243220a4918e8bef
+WORKDIR /
+COPY manager .
+USER 65532:65532
+ENTRYPOINT ["/manager"]

controller/Makefile

@@ -110,6 +110,10 @@ lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes
 build-operator:
 	make -C deploy/operator build-installer docker-build
 
+.PHONY: build-operator-ci
+build-operator-ci:
+	make -C deploy/operator build-installer docker-build-ci
+
 .PHONY: build
 build: manifests generate fmt vet ## Build manager binary.
 	go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go
@@ -135,6 +139,14 @@ docker-build: ## Build docker image with the manager.
 		--build-arg BUILD_DATE=$(BUILD_DATE) \
 		-t ${IMG} -f Containerfile .
 
+.PHONY: docker-build-ci
+docker-build-ci: ## Build docker images from pre-compiled host binaries (fast CI path).
+	CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go
+	CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/router cmd/router/main.go
+	CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/exporter-set-controller cmd/exporter-set-controller/main.go
+	$(CONTAINER_TOOL) build -t ${IMG} -f Containerfile.ci bin/
+	$(CONTAINER_TOOL) build -t ${EXPORTER_SET_CONTROLLER_IMG} -f Containerfile.exporter-set-controller.ci bin/
+
 .PHONY: docker-build-exporter-set-controller
 docker-build-exporter-set-controller: ## Build docker image for the exporter-set-controller.
 	$(CONTAINER_TOOL) build \
@@ -206,14 +218,30 @@ ifeq ($(SKIP_BUILD),)
 endif
 	./hack/deploy_with_operator.sh
 
+.PHONY: deploy-ci
+deploy-ci: cluster grpcurl ## CI-optimized deploy: host-compiled binaries, no multi-stage container builds.
+ifeq ($(SKIP_BUILD),)
+	$(MAKE) docker-build-ci
+	$(MAKE) build-operator-ci
+endif
+	./hack/deploy_with_operator.sh
+
 
 .PHONY: deploy-operator
 deploy-operator: docker-build docker-build-exporter-set-controller build-operator cluster grpcurl ## Deploy only the operator (without Jumpstarter CR)
 	NETWORKING_MODE=ingress DEPLOY_JUMPSTARTER=false ./hack/deploy_with_operator.sh
 
+.PHONY: deploy-operator-ci
+deploy-operator-ci: docker-build-ci build-operator-ci cluster grpcurl ## CI-optimized: host-compiled binaries, no multi-stage container builds.
+	NETWORKING_MODE=ingress DEPLOY_JUMPSTARTER=false ./hack/deploy_with_operator.sh
+
 .PHONY: test-operator-e2e
 test-operator-e2e: grpcurl deploy-operator
 	make -C deploy/operator test-e2e
+
+.PHONY: test-operator-e2e-ci
+test-operator-e2e-ci: grpcurl deploy-operator-ci ## CI-optimized e2e test (host-compiled Go, cached modules).
+	make -C deploy/operator test-e2e
 .PHONY: operator-logs
 operator-logs:
 	kubectl logs -n jumpstarter-operator-system -l app.kubernetes.io/name=jumpstarter-operator -f

controller/deploy/operator/Makefile

@@ -177,6 +177,11 @@ docker-build: ## Build docker image with the manager.
 		--build-arg BUILD_DATE=$(BUILD_DATE) \
 		-t ${IMG} ../../ -f ../../Containerfile.operator
 
+.PHONY: docker-build-ci
+docker-build-ci: ## CI-optimized: host-compiled binary, no multi-stage build.
+	CGO_ENABLED=0 go build -ldflags "$(LDFLAGS)" -o bin/manager cmd/main.go
+	$(CONTAINER_TOOL) build -t ${IMG} -f ../../Containerfile.operator.ci bin/
+
 .PHONY: docker-push
 docker-push: ## Push docker image with the manager.
 	$(CONTAINER_TOOL) push ${IMG}

controller/hack/deploy_with_operator.sh

@@ -29,23 +29,40 @@ if [ "${USE_CERTMANAGER}" = "true" ]; then
     fi
 fi
 
-# load the container images into the cluster
-load_image "${IMG}"
-load_image "${OPERATOR_IMG}"
-load_image "${EXPORTER_SET_CONTROLLER_IMG}"
-# Exporter + QEMU runtime images are required for ExporterSet QEMU e2e / samples.
-# Missing images are skipped so plain controller deploys still work.
+# Load container images into the cluster in parallel. Each `kind load` is I/O
+# bound (piping a tarball into the node's containerd), so overlapping them cuts
+# wall-clock to roughly the cost of the single largest image.
+_load_pids=()
+_load_failed=0
+
+load_image "${IMG}" &
+_load_pids+=($!)
+load_image "${OPERATOR_IMG}" &
+_load_pids+=($!)
+load_image "${EXPORTER_SET_CONTROLLER_IMG}" &
+_load_pids+=($!)
+
 if container_image_exists "${EXPORTER_IMG}"; then
-  load_image "${EXPORTER_IMG}"
+  load_image "${EXPORTER_IMG}" &
+  _load_pids+=($!)
 else
   echo -e "${YELLOW}Skipping load of exporter image (not present locally): ${EXPORTER_IMG}${NC}"
 fi
 if container_image_exists "${QEMU_RUNTIME_IMG}"; then
-  load_image "${QEMU_RUNTIME_IMG}"
+  load_image "${QEMU_RUNTIME_IMG}" &
+  _load_pids+=($!)
 else
   echo -e "${YELLOW}Skipping load of qemu-runtime image (not present locally): ${QEMU_RUNTIME_IMG}${NC}"
 fi
 
+for pid in "${_load_pids[@]}"; do
+  wait "${pid}" || _load_failed=1
+done
+if [ "${_load_failed}" -eq 1 ]; then
+  echo -e "${RED}One or more images failed to load${NC}"
+  exit 1
+fi
+
 # Deploy the operator
 echo -e "${GREEN}Deploying Jumpstarter operator ...${NC}"
 kubectl apply -f deploy/operator/dist/install.yaml