Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 19 additions & 18 deletions .github/workflows/build-and-deploy-job.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,7 @@ jobs:
AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }}

steps:
- name: 'Generate/build derived environment variables'
- name: "Generate/build derived environment variables"
run: |
echo "TARGET_ENVIRONMENT_UPPER=${TARGET_ENVIRONMENT^^}" >> ${GITHUB_ENV}
echo "CONTAINER_INSTANCE_BASE_NAME=aci-${APP_NAME}" >> ${GITHUB_ENV}
Expand All @@ -37,61 +37,62 @@ jobs:
echo "APP_NAME=${APP_NAME}" >> ${GITHUB_ENV}
echo "AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}" >> ${GITHUB_ENV}

- name: 'Print calculated environment variables'
- name: "Print calculated environment variables"
run: |
echo $TARGET_ENVIRONMENT_UPPER
echo $CONTAINER_INSTANCE_BASE_NAME
echo $RESOURCE_GROUP_BASE_NAME
echo $STORAGE_ACCOUNT_NAME
echo $APP_NAME
echo $AZURE_SUBSCRIPTION_ID
- name: 'Checkout GitHub Action'

- name: "Checkout GitHub Action"
uses: actions/checkout@v4

- name: 'Login via Azure CLI'
- name: "Login via Azure CLI"
uses: azure/login@v2
with:
creds: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_CREDENTIALS')] }}

- name: 'Login to Docker Hub'
- name: "Login to Docker Hub"
uses: docker/login-action@v3.2.0
with:
username: ${{ secrets.DOCKER_HUB_USERNAME }}
password: ${{ secrets.DOCKER_HUB_TOKEN }}

- name: 'Login to Azure Container Registry'
- name: "Login to Azure Container Registry"
uses: azure/docker-login@v2
with:
login-server: ${{ env.ACR_LOGIN_SERVER }}
username: ${{ env.ACR_USERNAME }}
password: ${{ env.ACR_PASSWORD }}

- name: 'Build and push image'
- name: "Build and push image"
run: |
IMAGE_NAME=$ACR_LOGIN_SERVER/$APP_NAME-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG
echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV
docker build . -f Dockerfile -t $IMAGE_NAME
docker push $IMAGE_NAME

- name: 'Print IMAGE_NAME for Bulk Data Service image'
- name: "Print IMAGE_NAME for Bulk Data Service image"
run: echo $IMAGE_NAME

- name: 'Create htpasswd file for nginx reverse proxy'
- name: "Create htpasswd file for nginx reverse proxy"
run: |
htpasswd -BC 10 -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom "${{ secrets.PROM_NGINX_REVERSE_PROXY_PASSWORD }}"
docker build ./azure-deployment/nginx-reverse-proxy -t criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG
docker push criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG

- name: 'Delete existing container group'
uses: 'azure/CLI@v2'
- name: "Delete existing container group"
uses: "azure/CLI@v2"
with:
inlineScript: |
az -v
az container delete -y \
--name "${{ env.CONTAINER_INSTANCE_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \
--resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}"

- name: 'Replace Env Vars and Secrets in ARM Yaml template'
- name: "Replace Env Vars and Secrets in ARM Yaml template"
env:
# Secrets / credentials for the app's resources
AZURE_SERVICE_BUS_CONNECTION_STRING: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_SERVICE_BUS_CONNECTION_STRING')] }}
Expand Down Expand Up @@ -136,24 +137,24 @@ jobs:
run: |
./azure-deployment/generate-manifest-from-template.sh

- name: 'Deploy group to Azure Container Instances'
uses: 'azure/CLI@v2'
- name: "Deploy group to Azure Container Instances"
uses: "azure/CLI@v2"
with:
inlineScript: |
az -v
az container create --debug \
--resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \
--file ./azure-deployment/azure-resource-manager-deployment-manifest.yml

- name: 'Re-generate the website links'
- name: "Re-generate the website links"
env:
WEB_BASE_URL: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'WEB_BASE_URL')] }}

run: |
sed -e "s#{{WEB_BASE_URL}}#$WEB_BASE_URL#" web/index-template.html > web/index.html
./azure-deployment/generate-index-from-template.sh

- name: 'Update the website'
uses: 'azure/CLI@v2'
- name: "Update the website"
uses: "azure/CLI@v2"
with:
inlineScript: |
az storage blob upload-batch -s web -d '$web' --account-name $STORAGE_ACCOUNT_NAME --overwrite
10 changes: 10 additions & 0 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -27,5 +27,15 @@ jobs:
- name: Run docker-compose
run: cd ./tests-local-environment; docker compose up -d

# The Service Bus emulator has no docker compose healthcheck (its image is
# distroless), so `docker compose up -d` returns before it is accepting AMQP
# connections. Without this gate the MQ integration tests race its startup.
- name: Wait for Service Bus emulator
run: ./tests-local-environment/wait-for-mq-emulator.sh

- name: Run automated tests
run: pytest

- name: Docker compose logs (on failure)
if: failure()
run: cd ./tests-local-environment; docker compose logs --no-color
16 changes: 16 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,22 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/).

### Removed

## [1.4.10] - 2026-07-27

### Changed

- Expanded explanatory text, added 'Last updated: ' for the indices, and
brought IATI footer up to date.
- Pinned the Azure Service Bus emulator to version 1.1.2 in the local
development and test docker compose setups, rather than tracking `latest`.

### Fixed

- Fixed intermittent CI test failures caused by a race condition due to the
tests sometimes starting before the Azure Service Bus emulator was ready: CI
now waits for the emulator's health API before running the tests, and dumps
the docker compose logs if the tests fail.

## [1.4.9] - 2026-05-25

### Fixed
Expand Down
13 changes: 13 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,19 @@ docker compose up --remove-orphans

_Note: the `--remove-orphans` just helps keep things clean as you develop, and alter the setup._

The Azure Service Bus emulator takes a while to start, and `docker compose`
cannot wait for it because the emulator image is distroless. Running the tests
before it is ready causes the MQ integration tests to fail with connection
errors, so wait for it with:

```
./tests-local-environment/wait-for-mq-emulator.sh
```

(For the local development docker compose setup, rather than the test setup,
pass the dev environment's health URL:
`./tests-local-environment/wait-for-mq-emulator.sh http://localhost:5300/health`)

Once this is running, run the tests with:

```
Expand Down
17 changes: 17 additions & 0 deletions azure-deployment/generate-index-from-template.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
#!/bin/env bash

# This script is not intended to be run directly.
# It is run by the generic 'build-and-deploy' Github action, and by
# 'azure-provision/azure-create-resources.sh', so that both paths generate
# an identical web/index.html.
#
# Expects WEB_BASE_URL in the environment. BULK_DATA_SERVICE_VERSION is read
# from pyproject.toml unless already set. Must be run from the repo root.

BULK_DATA_SERVICE_VERSION="${BULK_DATA_SERVICE_VERSION:-$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")}"

echo "Generating web/index.html (version ${BULK_DATA_SERVICE_VERSION}, base URL ${WEB_BASE_URL})"

sed -e "s#{{WEB_BASE_URL}}#${WEB_BASE_URL}#" \
-e "s#{{BULK_DATA_SERVICE_VERSION}}#${BULK_DATA_SERVICE_VERSION}#" \
./web/index-template.html > ./web/index.html
2 changes: 1 addition & 1 deletion azure-provision/azure-create-resources.sh
Original file line number Diff line number Diff line change
Expand Up @@ -178,7 +178,7 @@ AZURE_BASE_URL=$(az storage account show -n "$STORAGE_ACCOUNT_NAME" -g "$RESOURC
AZURE_BASE_HOSTNAME="$(sed "s#https://\(.*\)/#\1#" <<< $AZURE_BASE_URL)"

# WEB_BASE_URL is calculated above from TARGET_ENVIRONMENT, bearing in mind 'prod' doesn't have prefix
sed -e "s#{{WEB_BASE_URL}}#$WEB_BASE_URL#" web/index-template.html > web/index.html
./azure-deployment/generate-index-from-template.sh

echo "Uploading index and 404 pages to storage account..."
az storage blob upload-batch -s web -d '$web' --account-name $STORAGE_ACCOUNT_NAME --overwrite
Expand Down
5 changes: 3 additions & 2 deletions docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ services:
bds-local-dev-net:

bds-local-dev-mq-emulator:
image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest
pull_policy: always
# NOTE: The emulator version is pinned deliberately; see the note in
# tests-local-environment/docker-compose.yml for why.
image: mcr.microsoft.com/azure-messaging/servicebus-emulator:1.1.2
volumes:
# NOTE: The local dev docker compose environment and the local test docker
# compose environment use the same Azure Service Bus configuration
Expand Down
2 changes: 1 addition & 1 deletion pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
[project]
name = "bulk-data-service"
version = "1.4.9"
version = "1.4.10"
requires-python = ">= 3.12.6"
readme = "README.md"
dependencies = [
Expand Down
8 changes: 6 additions & 2 deletions tests-local-environment/docker-compose.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,12 @@ services:
bds-local-test-net:

bds-test-env-mq-emulator:
image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest
pull_policy: always
# NOTE: The emulator version is pinned deliberately. The `latest` tag moved
# to 2.0.1 in July 2026, which broke CI (the emulator takes long
# enough to start that the tests raced it), and 2.x carries an open
# bug where sends hang despite the emulator reporting healthy:
# https://github.com/Azure/azure-service-bus-emulator-installer/issues/135
image: mcr.microsoft.com/azure-messaging/servicebus-emulator:1.1.2
volumes:
# NOTE: The local dev docker compose environment and the local test docker
# compose environment use the same Azure Service Bus configuration
Expand Down
32 changes: 32 additions & 0 deletions tests-local-environment/wait-for-mq-emulator.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
#!/usr/bin/env bash

# Waits for the Azure Service Bus emulator to report itself healthy.
#
# The emulator exposes a health API (GET /health, which returns 503 while the
# emulator is starting up and 200 when ready). This can't be a docker compose
# healthcheck because the emulator image is distroless. See:
# https://github.com/Azure/azure-service-bus-emulator-installer/issues/88

# So we poll the health API from the host instead.
#
# Usage: wait-for-mq-emulator.sh [health-url] [timeout-seconds]
#
# The default URL is the local test environment (host port 5301). For the local
# development docker compose environment, pass http://localhost:5300/health

set -euo pipefail

HEALTH_URL="${1:-http://localhost:5301/health}"
TIMEOUT_SECONDS="${2:-120}"

deadline=$((SECONDS + TIMEOUT_SECONDS))

until curl --fail --silent --show-error "${HEALTH_URL}" >/dev/null 2>&1; do
if ((SECONDS >= deadline)); then
echo "Service Bus emulator did not become healthy within ${TIMEOUT_SECONDS}s: ${HEALTH_URL}" >&2
exit 1
fi
sleep 2
done

echo "Service Bus emulator is healthy (${HEALTH_URL})"
48 changes: 38 additions & 10 deletions web/404.html
Original file line number Diff line number Diff line change
Expand Up @@ -66,24 +66,52 @@ <h1>404 - Page or file not found</h1>

<footer class="iati-footer iati-brand-background">
<div class="iati-brand-background__content">
<div class="iati-footer__section">
<div class="iati-footer__container">
<div class="iati-footer-block">
<h2 class="iati-footer-block__title">Additional Information</h2>
<div class="iati-footer-block__content iati-footer-block__content--columns">
<div>
<p>Part of the IATI Unified Platform.</p>
<p>
Code licensed under
<a href="https://www.gnu.org/licenses/agpl-3.0.en.html">GNU AGPL</a>.
</p>
<p>
Documentation licensed under
<a href="https://creativecommons.org/licenses/by/4.0/">CC BY 3.0</a>.
</p>
</div>
<div>
<ul>
<li><a href="https://github.com/IATI/bulk-data-service">Bulk Data Service Code</a></li>
</ul>
</div>
</div>
Please report bugs and request features using <a
href="https://github.com/IATI/bulk-data-service/issues">GitHub issues</a> or via <a
href="https://iatistandard.org/en/guidance/get-support/">IATI Support</a>.
</div>
</div>
</div>

<div class="iati-footer__section iati-footer__section--last">
<div class="iati-footer__container">
<nav>
<ul class="iati-piped-list iati-footer__legal-nav">
<li><a href="https://iatistandard.org/en/privacy-policy/">Privacy</a></li>
<li><a href="https://iatistandard.org/en/data-removal/">Data removal</a></li>
<li><span>&copy; Copyright IATI 2025</span></li>
<li><a href="https://web-terms.iatistandard.org/en/latest/privacy-policy/">Privacy</a></li>
<li><a href="https://web-terms.iatistandard.org/en/latest/cookie-policy/">Cookies</a></li>
<li><a href="https://web-terms.iatistandard.org/en/latest/copyright/">Copyright</a></li>
<li><a href="https://web-terms.iatistandard.org/en/latest/data-removal/">Data Removal</a></li>
</ul>
</nav>
<div class="iati-footer__actions">

<div class="iati-country-switcher">
<label for="iati-country-switcher-footer" class="iati-country-switcher__label">Choose your language</label>
<select id="iati-country-switcher-footer" class="iati-country-switcher__control">
<option>English</option>
</select>
<a href="#" class="iati-button iati-button--light hide--mobile-nav">
<span>Feedback</span>
<i class="iati-icon iati-icon--feedback"></i>
</a>
</div>

<div class="iati-footer__social">
<a href="https://www.linkedin.com/company/international-aid-transparency-initiative/" aria-label="LinkedIn">
<i class="iati-icon iati-icon--linkedin"></i>
Expand All @@ -94,7 +122,7 @@ <h1>404 - Page or file not found</h1>
<a href="https://www.youtube.com/channel/UCAVH1gcgJXElsj8ENC-bDQQ" aria-label="YouTube">
<i class="iati-icon iati-icon--youtube"></i>
</a>
<a href="https://www.facebook.com/IATIaid" aria-label="Facebook">
<a href="https://www.facebook.com/IATIaid/" aria-label="Facebook">
<i class="iati-icon iati-icon--facebook"></i>
</a>
</div>
Expand Down
Loading
Loading