diff --git a/.github/workflows/build-and-deploy-job.yml b/.github/workflows/build-and-deploy-job.yml index bad60f9..56fad67 100644 --- a/.github/workflows/build-and-deploy-job.yml +++ b/.github/workflows/build-and-deploy-job.yml @@ -28,7 +28,7 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} steps: - - name: 'Generate/build derived environment variables' + - name: "Generate/build derived environment variables" run: | echo "TARGET_ENVIRONMENT_UPPER=${TARGET_ENVIRONMENT^^}" >> ${GITHUB_ENV} echo "CONTAINER_INSTANCE_BASE_NAME=aci-${APP_NAME}" >> ${GITHUB_ENV} @@ -37,7 +37,7 @@ jobs: echo "APP_NAME=${APP_NAME}" >> ${GITHUB_ENV} echo "AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}" >> ${GITHUB_ENV} - - name: 'Print calculated environment variables' + - name: "Print calculated environment variables" run: | echo $TARGET_ENVIRONMENT_UPPER echo $CONTAINER_INSTANCE_BASE_NAME @@ -45,45 +45,46 @@ jobs: echo $STORAGE_ACCOUNT_NAME echo $APP_NAME echo $AZURE_SUBSCRIPTION_ID - - name: 'Checkout GitHub Action' + + - name: "Checkout GitHub Action" uses: actions/checkout@v4 - - name: 'Login via Azure CLI' + - name: "Login via Azure CLI" uses: azure/login@v2 with: creds: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_CREDENTIALS')] }} - - name: 'Login to Docker Hub' + - name: "Login to Docker Hub" uses: docker/login-action@v3.2.0 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: 'Login to Azure Container Registry' + - name: "Login to Azure Container Registry" uses: azure/docker-login@v2 with: login-server: ${{ env.ACR_LOGIN_SERVER }} username: ${{ env.ACR_USERNAME }} password: ${{ env.ACR_PASSWORD }} - - name: 'Build and push image' + - name: "Build and push image" run: | IMAGE_NAME=$ACR_LOGIN_SERVER/$APP_NAME-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV docker build . -f Dockerfile -t $IMAGE_NAME docker push $IMAGE_NAME - - name: 'Print IMAGE_NAME for Bulk Data Service image' + - name: "Print IMAGE_NAME for Bulk Data Service image" run: echo $IMAGE_NAME - - name: 'Create htpasswd file for nginx reverse proxy' + - name: "Create htpasswd file for nginx reverse proxy" run: | htpasswd -BC 10 -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom "${{ secrets.PROM_NGINX_REVERSE_PROXY_PASSWORD }}" docker build ./azure-deployment/nginx-reverse-proxy -t criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG docker push criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG - - name: 'Delete existing container group' - uses: 'azure/CLI@v2' + - name: "Delete existing container group" + uses: "azure/CLI@v2" with: inlineScript: | az -v @@ -91,7 +92,7 @@ jobs: --name "${{ env.CONTAINER_INSTANCE_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \ --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" - - name: 'Replace Env Vars and Secrets in ARM Yaml template' + - name: "Replace Env Vars and Secrets in ARM Yaml template" env: # Secrets / credentials for the app's resources AZURE_SERVICE_BUS_CONNECTION_STRING: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_SERVICE_BUS_CONNECTION_STRING')] }} @@ -136,8 +137,8 @@ jobs: run: | ./azure-deployment/generate-manifest-from-template.sh - - name: 'Deploy group to Azure Container Instances' - uses: 'azure/CLI@v2' + - name: "Deploy group to Azure Container Instances" + uses: "azure/CLI@v2" with: inlineScript: | az -v @@ -145,15 +146,15 @@ jobs: --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \ --file ./azure-deployment/azure-resource-manager-deployment-manifest.yml - - name: 'Re-generate the website links' + - name: "Re-generate the website links" env: WEB_BASE_URL: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'WEB_BASE_URL')] }} run: | - sed -e "s#{{WEB_BASE_URL}}#$WEB_BASE_URL#" web/index-template.html > web/index.html + ./azure-deployment/generate-index-from-template.sh - - name: 'Update the website' - uses: 'azure/CLI@v2' + - name: "Update the website" + uses: "azure/CLI@v2" with: inlineScript: | az storage blob upload-batch -s web -d '$web' --account-name $STORAGE_ACCOUNT_NAME --overwrite diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97113bf..d8ee58b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,5 +27,15 @@ jobs: - name: Run docker-compose run: cd ./tests-local-environment; docker compose up -d + # The Service Bus emulator has no docker compose healthcheck (its image is + # distroless), so `docker compose up -d` returns before it is accepting AMQP + # connections. Without this gate the MQ integration tests race its startup. + - name: Wait for Service Bus emulator + run: ./tests-local-environment/wait-for-mq-emulator.sh + - name: Run automated tests run: pytest + + - name: Docker compose logs (on failure) + if: failure() + run: cd ./tests-local-environment; docker compose logs --no-color diff --git a/CHANGELOG.md b/CHANGELOG.md index f8019a8..cc54ca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Removed +## [1.4.10] - 2026-07-27 + +### Changed + + - Expanded explanatory text, added 'Last updated: ' for the indices, and + brought IATI footer up to date. + - Pinned the Azure Service Bus emulator to version 1.1.2 in the local + development and test docker compose setups, rather than tracking `latest`. + +### Fixed + + - Fixed intermittent CI test failures caused by a race condition due to the + tests sometimes starting before the Azure Service Bus emulator was ready: CI + now waits for the emulator's health API before running the tests, and dumps + the docker compose logs if the tests fail. + ## [1.4.9] - 2026-05-25 ### Fixed diff --git a/README.md b/README.md index fedeecb..995c6bd 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,19 @@ docker compose up --remove-orphans _Note: the `--remove-orphans` just helps keep things clean as you develop, and alter the setup._ +The Azure Service Bus emulator takes a while to start, and `docker compose` +cannot wait for it because the emulator image is distroless. Running the tests +before it is ready causes the MQ integration tests to fail with connection +errors, so wait for it with: + +``` +./tests-local-environment/wait-for-mq-emulator.sh +``` + +(For the local development docker compose setup, rather than the test setup, +pass the dev environment's health URL: +`./tests-local-environment/wait-for-mq-emulator.sh http://localhost:5300/health`) + Once this is running, run the tests with: ``` diff --git a/azure-deployment/generate-index-from-template.sh b/azure-deployment/generate-index-from-template.sh new file mode 100755 index 0000000..5be532a --- /dev/null +++ b/azure-deployment/generate-index-from-template.sh @@ -0,0 +1,17 @@ +#!/bin/env bash + +# This script is not intended to be run directly. +# It is run by the generic 'build-and-deploy' Github action, and by +# 'azure-provision/azure-create-resources.sh', so that both paths generate +# an identical web/index.html. +# +# Expects WEB_BASE_URL in the environment. BULK_DATA_SERVICE_VERSION is read +# from pyproject.toml unless already set. Must be run from the repo root. + +BULK_DATA_SERVICE_VERSION="${BULK_DATA_SERVICE_VERSION:-$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")}" + +echo "Generating web/index.html (version ${BULK_DATA_SERVICE_VERSION}, base URL ${WEB_BASE_URL})" + +sed -e "s#{{WEB_BASE_URL}}#${WEB_BASE_URL}#" \ + -e "s#{{BULK_DATA_SERVICE_VERSION}}#${BULK_DATA_SERVICE_VERSION}#" \ + ./web/index-template.html > ./web/index.html diff --git a/azure-provision/azure-create-resources.sh b/azure-provision/azure-create-resources.sh index 5e55a8c..61edce9 100755 --- a/azure-provision/azure-create-resources.sh +++ b/azure-provision/azure-create-resources.sh @@ -178,7 +178,7 @@ AZURE_BASE_URL=$(az storage account show -n "$STORAGE_ACCOUNT_NAME" -g "$RESOURC AZURE_BASE_HOSTNAME="$(sed "s#https://\(.*\)/#\1#" <<< $AZURE_BASE_URL)" # WEB_BASE_URL is calculated above from TARGET_ENVIRONMENT, bearing in mind 'prod' doesn't have prefix -sed -e "s#{{WEB_BASE_URL}}#$WEB_BASE_URL#" web/index-template.html > web/index.html +./azure-deployment/generate-index-from-template.sh echo "Uploading index and 404 pages to storage account..." az storage blob upload-batch -s web -d '$web' --account-name $STORAGE_ACCOUNT_NAME --overwrite diff --git a/docker-compose.yml b/docker-compose.yml index 20a0192..c47fdf0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,8 +52,9 @@ services: bds-local-dev-net: bds-local-dev-mq-emulator: - image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest - pull_policy: always + # NOTE: The emulator version is pinned deliberately; see the note in + # tests-local-environment/docker-compose.yml for why. + image: mcr.microsoft.com/azure-messaging/servicebus-emulator:1.1.2 volumes: # NOTE: The local dev docker compose environment and the local test docker # compose environment use the same Azure Service Bus configuration diff --git a/pyproject.toml b/pyproject.toml index a6b6bc4..db44f72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bulk-data-service" -version = "1.4.9" +version = "1.4.10" requires-python = ">= 3.12.6" readme = "README.md" dependencies = [ diff --git a/tests-local-environment/docker-compose.yml b/tests-local-environment/docker-compose.yml index 95ea3d9..867ee91 100644 --- a/tests-local-environment/docker-compose.yml +++ b/tests-local-environment/docker-compose.yml @@ -60,8 +60,12 @@ services: bds-local-test-net: bds-test-env-mq-emulator: - image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest - pull_policy: always + # NOTE: The emulator version is pinned deliberately. The `latest` tag moved + # to 2.0.1 in July 2026, which broke CI (the emulator takes long + # enough to start that the tests raced it), and 2.x carries an open + # bug where sends hang despite the emulator reporting healthy: + # https://github.com/Azure/azure-service-bus-emulator-installer/issues/135 + image: mcr.microsoft.com/azure-messaging/servicebus-emulator:1.1.2 volumes: # NOTE: The local dev docker compose environment and the local test docker # compose environment use the same Azure Service Bus configuration diff --git a/tests-local-environment/wait-for-mq-emulator.sh b/tests-local-environment/wait-for-mq-emulator.sh new file mode 100755 index 0000000..bb32455 --- /dev/null +++ b/tests-local-environment/wait-for-mq-emulator.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Waits for the Azure Service Bus emulator to report itself healthy. +# +# The emulator exposes a health API (GET /health, which returns 503 while the +# emulator is starting up and 200 when ready). This can't be a docker compose +# healthcheck because the emulator image is distroless. See: +# https://github.com/Azure/azure-service-bus-emulator-installer/issues/88 + +# So we poll the health API from the host instead. +# +# Usage: wait-for-mq-emulator.sh [health-url] [timeout-seconds] +# +# The default URL is the local test environment (host port 5301). For the local +# development docker compose environment, pass http://localhost:5300/health + +set -euo pipefail + +HEALTH_URL="${1:-http://localhost:5301/health}" +TIMEOUT_SECONDS="${2:-120}" + +deadline=$((SECONDS + TIMEOUT_SECONDS)) + +until curl --fail --silent --show-error "${HEALTH_URL}" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "Service Bus emulator did not become healthy within ${TIMEOUT_SECONDS}s: ${HEALTH_URL}" >&2 + exit 1 + fi + sleep 2 +done + +echo "Service Bus emulator is healthy (${HEALTH_URL})" diff --git a/web/404.html b/web/404.html index d1f2bba..1093c2f 100644 --- a/web/404.html +++ b/web/404.html @@ -66,24 +66,52 @@

404 - Page or file not found