e2e improvements

.github/workflows/build-images.yaml

@@ -135,6 +135,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Build operator installer manifest
         if: ${{ matrix.generate_installer && steps.check.outputs.skip != 'true' }}

.github/workflows/e2e.yaml

@@ -10,6 +10,14 @@ permissions:
 
 env:
   CONTAINER_TOOL: docker
+  # Retry a failed spec once before failing the job. Retries are reported as
+  # flakes in the ginkgo summary, so this hides nothing; it only stops a single
+  # infrastructure hiccup from failing an unrelated PR.
+  E2E_FLAKE_ATTEMPTS: "2"
+  # Run the non-Serial containers two at a time. The runner has 4 CPUs and the
+  # suite spends most of its time waiting on the cluster, so a second process
+  # overlaps the two long containers (hooks, core) rather than competing for CPU.
+  E2E_PROCS: "2"
 
 jobs:
   changes:
@@ -68,6 +76,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Cache controller image
         id: cache
@@ -107,6 +116,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Cache operator artifacts
         id: cache
@@ -151,6 +161,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Cache exporterset-controller image
         id: cache
@@ -310,6 +321,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Load e2e artifacts
         uses: ./.github/actions/load-e2e-artifacts
@@ -327,6 +339,16 @@ jobs:
             sudo modprobe "$mod" 2>/dev/null || true
           done
 
+      - name: Cache controller tools
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: controller/bin
+          key: controller-tools-${{ matrix.arch }}-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }}
+        env:
+          KIND_VERSION: v0.27.0
+          KUSTOMIZE_VERSION: v5.4.1
+          GRPCURL_VERSION: v1.9.2
+
       - name: Setup e2e test environment
         run: make e2e-setup
         env:
@@ -374,6 +396,17 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
+
+      - name: Cache controller tools
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: controller/bin
+          key: controller-tools-amd64-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }}
+        env:
+          KIND_VERSION: v0.27.0
+          KUSTOMIZE_VERSION: v5.4.1
+          GRPCURL_VERSION: v1.9.2
 
       - name: Setup compat environment (old controller v0.8.1)
         run: make e2e-compat-setup COMPAT_SCENARIO=old-controller
@@ -416,12 +449,23 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Load e2e artifacts
         uses: ./.github/actions/load-e2e-artifacts
         with:
           arch: amd64
 
+      - name: Cache controller tools
+        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
+        with:
+          path: controller/bin
+          key: controller-tools-amd64-kind${{ env.KIND_VERSION }}-kustomize${{ env.KUSTOMIZE_VERSION }}-grpcurl${{ env.GRPCURL_VERSION }}
+        env:
+          KIND_VERSION: v0.27.0
+          KUSTOMIZE_VERSION: v5.4.1
+          GRPCURL_VERSION: v1.9.2
+
       - name: Setup compat environment (old client v0.7.4)
         run: make e2e-compat-setup COMPAT_SCENARIO=old-client
         env:

.github/workflows/lint.yaml

@@ -55,6 +55,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Run go linter
         working-directory: controller

.github/workflows/python-tests.yaml

@@ -145,7 +145,7 @@ jobs:
         uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6
         with:
           path: python/packages/jumpstarter-driver-qemu/images
-          key: fedora-cloud-43-1.6
+          key: fedora-cloud-43-1.6-preconfigured-v1
 
       - name: Download Fedora Cloud images
         if: steps.cache-fedora-cloud-images.outputs.cache-hit != 'true'
@@ -155,12 +155,36 @@ jobs:
               "https://iad.mirror.rackspace.com/fedora/releases/43/Cloud/${arch}/images/Fedora-Cloud-Base-Generic-43-1.6.${arch}.qcow2"
           done
 
+      - name: Pre-configure Fedora Cloud images for fast boot
+        if: runner.os == 'Linux' && steps.cache-fedora-cloud-images.outputs.cache-hit != 'true'
+        run: |
+          sudo apt-get install -y libguestfs-tools
+          arch=x86_64
+          img="python/packages/jumpstarter-driver-qemu/images/Fedora-Cloud-Base-Generic-43-1.6.${arch}.qcow2"
+          sudo virt-customize -a "$img" \
+            --hostname demo \
+            --run-command 'useradd -m -s /bin/bash jumpstarter' \
+            --password jumpstarter:password:password \
+            --run-command 'echo "jumpstarter ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/jumpstarter' \
+            --run-command 'systemctl disable cloud-init cloud-init-local cloud-config cloud-final' \
+            --run-command 'rm -rf /var/lib/cloud' \
+            --selinux-relabel
+
       - name: Run pytest
         working-directory: python
         env:
-          PYTEST_ADDOPTS: "--cov-report=xml --log-level=CRITICAL --log-cli-level=CRITICAL"
+          PYTEST_ADDOPTS: "--log-level=CRITICAL --log-cli-level=CRITICAL"
         run: |
-          make test -j4 LOGS_DIR=${{ runner.temp }}/test-logs
+          changed_pkgs=$(git diff --name-only --relative origin/${{ github.base_ref || 'main' }}...HEAD -- packages/ \
+            | sed -n 's|^packages/\([^/]*\)/.*|\1|p' | sort -u | tr '\n' ' ')
+          echo "Coverage-enabled packages: ${changed_pkgs:-all (fallback)}"
+          if [ -z "$changed_pkgs" ]; then
+            # Fallback: workflow_dispatch or merge_group without a base ref
+            export PYTEST_ADDOPTS="--cov=. --cov-report=xml $PYTEST_ADDOPTS"
+            make test -j8 LOGS_DIR=${{ runner.temp }}/test-logs
+          else
+            make test -j8 LOGS_DIR=${{ runner.temp }}/test-logs COV_PACKAGES="$changed_pkgs"
+          fi
 
       - name: Upload test logs
         if: always()

.github/workflows/release-operator-installer.yaml

@@ -20,6 +20,7 @@ jobs:
         uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7
         with:
           go-version-file: .go-version
+          cache-dependency-path: "**/go.sum"
 
       - name: Build operator installer manifest
         env:

controller/hack/deploy_with_operator.sh

@@ -29,23 +29,40 @@ if [ "${USE_CERTMANAGER}" = "true" ]; then
     fi
 fi
 
-# load the container images into the cluster
-load_image "${IMG}"
-load_image "${OPERATOR_IMG}"
-load_image "${EXPORTER_SET_CONTROLLER_IMG}"
-# Exporter + QEMU runtime images are required for ExporterSet QEMU e2e / samples.
-# Missing images are skipped so plain controller deploys still work.
+# Load container images into the cluster in parallel. Each `kind load` is I/O
+# bound (piping a tarball into the node's containerd), so overlapping them cuts
+# wall-clock to roughly the cost of the single largest image.
+_load_pids=()
+_load_failed=0
+
+load_image "${IMG}" &
+_load_pids+=($!)
+load_image "${OPERATOR_IMG}" &
+_load_pids+=($!)
+load_image "${EXPORTER_SET_CONTROLLER_IMG}" &
+_load_pids+=($!)
+
 if container_image_exists "${EXPORTER_IMG}"; then
-  load_image "${EXPORTER_IMG}"
+  load_image "${EXPORTER_IMG}" &
+  _load_pids+=($!)
 else
   echo -e "${YELLOW}Skipping load of exporter image (not present locally): ${EXPORTER_IMG}${NC}"
 fi
 if container_image_exists "${QEMU_RUNTIME_IMG}"; then
-  load_image "${QEMU_RUNTIME_IMG}"
+  load_image "${QEMU_RUNTIME_IMG}" &
+  _load_pids+=($!)
 else
   echo -e "${YELLOW}Skipping load of qemu-runtime image (not present locally): ${QEMU_RUNTIME_IMG}${NC}"
 fi
 
+for pid in "${_load_pids[@]}"; do
+  wait "${pid}" || _load_failed=1
+done
+if [ "${_load_failed}" -eq 1 ]; then
+  echo -e "${RED}One or more images failed to load${NC}"
+  exit 1
+fi
+
 # Deploy the operator
 echo -e "${GREEN}Deploying Jumpstarter operator ...${NC}"
 kubectl apply -f deploy/operator/dist/install.yaml

e2e/lib/common.sh

@@ -79,7 +79,23 @@ run_ginkgo() {
         timeout="60m"
     fi
 
-    local flags=(-v --show-node-events --trace --timeout "${timeout}")
+    # Retry a failed spec instead of failing the whole suite. The e2e suite talks
+    # to a real cluster over the network, so a spec can fail for reasons that have
+    # nothing to do with the code under test (a slow DNS answer, a pod scheduled
+    # late, a router connection dropped). A retried spec is still reported as
+    # flaky in the summary, so genuine instability stays visible.
+    local flake_attempts="${E2E_FLAKE_ATTEMPTS:-1}"
+
+    # Run top-level containers concurrently when asked. Off by default: the
+    # suite shares one cluster and one runner, so more processes is not free.
+    # Containers that touch host-global state or the shared client config are
+    # marked Serial and still run one at a time, after the parallel ones.
+    local procs="${E2E_PROCS:-1}"
+
+    local flags=(-v --show-node-events --trace --timeout "${timeout}" --flake-attempts "${flake_attempts}")
+    if [ "${procs}" -gt 1 ]; then
+        flags+=(--procs "${procs}")
+    fi
     if [ -n "$label_filter" ]; then
         flags+=(--label-filter "$label_filter")
     fi

e2e/setup-e2e.sh

@@ -275,6 +275,35 @@ deploy_controller() {
 # shellcheck source=lib/install.sh
 source "$SCRIPT_DIR/lib/install.sh"
 
+# Pin the ingress hostnames in /etc/hosts so that host-side clients (jmp, curl)
+# never depend on public DNS.
+#
+# baseDomain is a nip.io wildcard name of the form jumpstarter.<IP>.nip.io, so
+# every jmp invocation would otherwise resolve <prefix>.jumpstarter.<IP>.nip.io
+# against a public resolver. A slow or rate-limited lookup burns the client's
+# whole connect budget and surfaces as "Timeout connecting to grpc....:8082".
+# The IP is already embedded in the name, so we can serve the same answer
+# locally. Non-nip.io base domains are left alone.
+pin_basedomain_hosts_entries() {
+    local basedomain="$1"
+    local ip
+
+    ip=$(echo "${basedomain}" | sed -nE 's/^.*\.([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)\.nip\.io$/\1/p')
+    if [ -z "${ip}" ]; then
+        log_info "baseDomain ${basedomain} is not a nip.io name, leaving DNS resolution alone"
+        return 0
+    fi
+
+    if grep -q "grpc.${basedomain}" /etc/hosts 2>/dev/null; then
+        log_info "✓ ${basedomain} entries already in /etc/hosts"
+        return 0
+    fi
+
+    log_warn "About to add ${basedomain} entries to /etc/hosts (requires sudo)"
+    echo "${ip} ${basedomain} grpc.${basedomain} router.${basedomain} login.${basedomain}" | sudo tee -a /etc/hosts
+    log_info "✓ Pinned ${basedomain} to ${ip} in /etc/hosts"
+}
+
 # Step 6: Setup test environment
 setup_test_environment() {
     log_info "Setting up test environment..."
@@ -290,6 +319,7 @@ setup_test_environment() {
         log_error "Failed to get baseDomain from Jumpstarter CR in namespace ${JS_NAMESPACE}"
         exit 1
     fi
+    pin_basedomain_hosts_entries "${BASEDOMAIN}"
     export ENDPOINT="grpc.${BASEDOMAIN}:8082"
     export LOGIN_ENDPOINT="login.${BASEDOMAIN}:8086"
     log_info "Controller endpoint: $ENDPOINT"

e2e/test/auth_logging_test.go

@@ -35,7 +35,7 @@ import (
 // with legacy (controller-issued) tokens so the token lives in a local config
 // file where the test can corrupt it. It is intentionally NOT labelled for the
 // compat suites — old controller images do not have auth-failure logging.
-var _ = Describe("Auth Failure Logging E2E Tests", Label("auth-logging"), Ordered, func() {
+var _ = Describe("Auth Failure Logging E2E Tests", Label("auth-logging"), Ordered, ContinueOnFailure, func() {
 	const (
 		clientName   = "test-client-authlog"
 		exporterName = "test-exporter-authlog"

e2e/test/compat_old_client_test.go

@@ -26,7 +26,8 @@ import (
 	. "github.com/onsi/gomega"    //nolint:revive
 )
 
-var _ = Describe("Compat: Old Client E2E Tests", Label("compat", "old-client"), Ordered, func() {
+// Serial: installs an older client into the shared environment.
+var _ = Describe("Compat: Old Client E2E Tests", Label("compat", "old-client"), Ordered, Serial, func() {
 	var (
 		tracker    *ProcessTracker
 		ns         string

e2e/test/compat_old_controller_test.go

@@ -26,7 +26,8 @@ import (
 	. "github.com/onsi/gomega"    //nolint:revive
 )
 
-var _ = Describe("Compat: Old Controller E2E Tests", Label("compat", "old-controller"), Ordered, func() {
+// Serial: replaces the running controller and switches the active client.
+var _ = Describe("Compat: Old Controller E2E Tests", Label("compat", "old-controller"), Ordered, Serial, func() {
 	var (
 		tracker *ProcessTracker
 		ns      string

e2e/test/direct_listener_test.go

@@ -26,7 +26,7 @@ import (
 	. "github.com/onsi/gomega"    //nolint:revive
 )
 
-var _ = Describe("Direct Listener E2E Tests", Label("direct-listener"), Ordered, func() {
+var _ = Describe("Direct Listener E2E Tests", Label("direct-listener"), Ordered, ContinueOnFailure, func() {
 	var (
 		tracker      *ProcessTracker
 		listenerPort = 19090

e2e/test/dut_network_test.go

@@ -51,7 +51,9 @@ func sudoArgs(args ...string) (string, []string) {
 	return args[0], args[1:]
 }
 
-var _ = Describe("DUT Network E2E Tests", Label("dut-network"), Ordered, func() {
+// Serial: builds veth pairs, bridges and nftables rules in the host network
+// namespace, and drives dnsmasq. There is only one host to share.
+var _ = Describe("DUT Network E2E Tests", Label("dut-network"), Ordered, ContinueOnFailure, Serial, func() {
 	var (
 		tracker      *ProcessTracker
 		listenerPort = 19091