ci: make Nethermind E2E failures actionable

.github/scripts/capture-nethermind-e2e-diagnostics.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+readonly output_dir="${DIAGNOSTICS_OUTPUT_DIR:-../e2e_tests}"
+readonly diagnostics_log="${output_dir}/${DIAGNOSTICS_LOG_NAME:-nethermind-e2e-diagnostics.log}"
+readonly tail_lines="${DIAGNOSTICS_TAIL_LINES:-2000}"
+readonly default_services=(
+  shasta-deployer
+  catalyst-node-1
+  catalyst-node-2
+  taiko-client-go-1
+  taiko-client-go-2
+  taiko-nethermind-1
+  taiko-nethermind-2
+  web3signer_l1
+  web3signer_l2
+  transfer-funds
+  p2p-bootnode
+)
+
+if (( $# > 0 )); then
+  services=("$@")
+else
+  services=("${default_services[@]}")
+fi
+readonly -a services
+
+mkdir -p "$output_dir"
+: > "$diagnostics_log"
+
+capture() {
+  local command_status
+
+  echo "+ $*" | tee -a "$diagnostics_log"
+  "$@" 2>&1 | tee -a "$diagnostics_log"
+  command_status=${PIPESTATUS[0]}
+  if (( command_status != 0 )); then
+    echo "Diagnostic command exited with ${command_status}; continuing" \
+      | tee -a "$diagnostics_log"
+  fi
+}
+
+capture docker ps -a
+capture docker compose ps -a
+for service in "${services[@]}"; do
+  capture docker compose logs --no-color --timestamps --tail "$tail_lines" "$service"
+done
+
+exit 0

.github/scripts/install-actionlint.sh

@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+readonly actionlint_version="1.7.12"
+readonly install_dir="${RUNNER_TEMP:?RUNNER_TEMP must be set}/actionlint"
+
+if [[ "${RUNNER_OS:?RUNNER_OS must be set}" != "Linux" ]]; then
+  echo "actionlint installer only supports Linux runners" >&2
+  exit 1
+fi
+
+case "${RUNNER_ARCH:?RUNNER_ARCH must be set}" in
+  # Refresh these digests with:
+  # gh api repos/rhysd/actionlint/releases/tags/v1.7.12 \
+  #   --jq '.assets[] | select(.name | test("linux_(386|amd64|arm64|armv6)")) | [.name, .digest] | @tsv'
+  X64)
+    actionlint_arch="amd64"
+    expected_sha256="8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8"
+    ;;
+  ARM64)
+    actionlint_arch="arm64"
+    expected_sha256="325e971b6ba9bfa504672e29be93c24981eeb1c07576d730e9f7c8805afff0c6"
+    ;;
+  X86)
+    actionlint_arch="386"
+    expected_sha256="72a44b32c2d032700e6d0c23ca2f540b67519ec68db098ddfcfa96059e61f723"
+    ;;
+  ARM)
+    actionlint_arch="armv6"
+    expected_sha256="ae4a0a5227578e66f5d00ee02788d5c64fdae1fa6484ab88ceaeee9359c28fa4"
+    ;;
+  *)
+    echo "unsupported runner architecture: ${RUNNER_ARCH}" >&2
+    exit 1
+    ;;
+esac
+
+readonly actionlint_arch expected_sha256
+readonly archive="actionlint_${actionlint_version}_linux_${actionlint_arch}.tar.gz"
+readonly archive_path="${RUNNER_TEMP}/${archive}"
+readonly download_url="https://github.com/rhysd/actionlint/releases/download/v${actionlint_version}/${archive}"
+readonly executable="${install_dir}/actionlint"
+
+mkdir -p "$install_dir"
+trap 'rm -f "$archive_path"' EXIT
+
+curl --fail --silent --show-error --location \
+  --retry 3 --retry-delay 2 --retry-max-time 300 \
+  --connect-timeout 10 --max-time 120 \
+  --output "$archive_path" "$download_url"
+printf '%s  %s\n' "$expected_sha256" "$archive_path" | sha256sum --check -
+tar -xzf "$archive_path" -C "$install_dir" actionlint
+
+if [[ ! -x "$executable" ]]; then
+  echo "actionlint executable was not extracted" >&2
+  exit 1
+fi
+
+printf 'executable=%s\n' "$executable" >> "${GITHUB_OUTPUT:?GITHUB_OUTPUT must be set}"

.github/scripts/run-shasta-e2e.sh

@@ -0,0 +1,62 @@
+#!/usr/bin/env bash
+
+set -uo pipefail
+
+readonly pytest_log_file="${PYTEST_LOG_FILE:-pytest-shasta.log}"
+readonly smoke_timeout="${SHASTA_SMOKE_TIMEOUT:-20m}"
+readonly suite_timeout="${SHASTA_SUITE_TIMEOUT:-90m}"
+readonly pytest_options=(
+  -v
+  --capture=tee-sys
+  -r a
+  --tb=long
+)
+readonly smoke_tests=(
+  test_avs_node.py::test_rpcs
+  test_avs_node.py::test_preconfirm_transaction
+  test_avs_node.py::test_p2p_preconfirmation
+)
+
+: > "$pytest_log_file"
+
+run_and_log() {
+  "$@" 2>&1 | tee -a "$pytest_log_file"
+  local -a pipeline_status=("${PIPESTATUS[@]}")
+  if (( pipeline_status[0] != 0 )); then
+    return "${pipeline_status[0]}"
+  fi
+  return "${pipeline_status[1]}"
+}
+
+echo "Running Shasta block-production smoke gate" | tee -a "$pytest_log_file"
+run_and_log \
+  timeout --signal=TERM --kill-after=30s "$smoke_timeout" \
+  pytest "${pytest_options[@]}" --maxfail=1 "${smoke_tests[@]}"
+smoke_status=$?
+
+if (( smoke_status != 0 )); then
+  if (( smoke_status == 124 )); then
+    echo "Shasta smoke gate timed out after ${smoke_timeout}" | tee -a "$pytest_log_file"
+  else
+    echo "Shasta smoke gate failed with exit code ${smoke_status}" | tee -a "$pytest_log_file"
+  fi
+  exit "$smoke_status"
+fi
+
+echo "Running remaining Shasta E2E tests" | tee -a "$pytest_log_file"
+remaining_test_options=()
+for test_name in "${smoke_tests[@]}"; do
+  remaining_test_options+=("--deselect=${test_name}")
+done
+
+run_and_log \
+  timeout --signal=TERM --kill-after=30s "$suite_timeout" \
+  pytest "${pytest_options[@]}" "${remaining_test_options[@]}"
+suite_status=$?
+
+if (( suite_status == 124 )); then
+  echo "Remaining Shasta E2E tests timed out after ${suite_timeout}" \
+    | tee -a "$pytest_log_file"
+fi
+
+exit "$suite_status"