diff --git a/.github/workflows/build-and-deploy-job.yml b/.github/workflows/build-and-deploy-job.yml index bad60f9..56fad67 100644 --- a/.github/workflows/build-and-deploy-job.yml +++ b/.github/workflows/build-and-deploy-job.yml @@ -28,7 +28,7 @@ jobs: AZURE_SUBSCRIPTION_ID: ${{ secrets.AZURE_SUBSCRIPTION_ID }} steps: - - name: 'Generate/build derived environment variables' + - name: "Generate/build derived environment variables" run: | echo "TARGET_ENVIRONMENT_UPPER=${TARGET_ENVIRONMENT^^}" >> ${GITHUB_ENV} echo "CONTAINER_INSTANCE_BASE_NAME=aci-${APP_NAME}" >> ${GITHUB_ENV} @@ -37,7 +37,7 @@ jobs: echo "APP_NAME=${APP_NAME}" >> ${GITHUB_ENV} echo "AZURE_SUBSCRIPTION_ID=${AZURE_SUBSCRIPTION_ID}" >> ${GITHUB_ENV} - - name: 'Print calculated environment variables' + - name: "Print calculated environment variables" run: | echo $TARGET_ENVIRONMENT_UPPER echo $CONTAINER_INSTANCE_BASE_NAME @@ -45,45 +45,46 @@ jobs: echo $STORAGE_ACCOUNT_NAME echo $APP_NAME echo $AZURE_SUBSCRIPTION_ID - - name: 'Checkout GitHub Action' + + - name: "Checkout GitHub Action" uses: actions/checkout@v4 - - name: 'Login via Azure CLI' + - name: "Login via Azure CLI" uses: azure/login@v2 with: creds: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_CREDENTIALS')] }} - - name: 'Login to Docker Hub' + - name: "Login to Docker Hub" uses: docker/login-action@v3.2.0 with: username: ${{ secrets.DOCKER_HUB_USERNAME }} password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: 'Login to Azure Container Registry' + - name: "Login to Azure Container Registry" uses: azure/docker-login@v2 with: login-server: ${{ env.ACR_LOGIN_SERVER }} username: ${{ env.ACR_USERNAME }} password: ${{ env.ACR_PASSWORD }} - - name: 'Build and push image' + - name: "Build and push image" run: | IMAGE_NAME=$ACR_LOGIN_SERVER/$APP_NAME-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG echo "IMAGE_NAME=$IMAGE_NAME" >> $GITHUB_ENV docker build . -f Dockerfile -t $IMAGE_NAME docker push $IMAGE_NAME - - name: 'Print IMAGE_NAME for Bulk Data Service image' + - name: "Print IMAGE_NAME for Bulk Data Service image" run: echo $IMAGE_NAME - - name: 'Create htpasswd file for nginx reverse proxy' + - name: "Create htpasswd file for nginx reverse proxy" run: | htpasswd -BC 10 -c -b ./azure-deployment/nginx-reverse-proxy/htpasswd prom "${{ secrets.PROM_NGINX_REVERSE_PROXY_PASSWORD }}" docker build ./azure-deployment/nginx-reverse-proxy -t criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG docker push criati.azurecr.io/bds-prom-nginx-reverse-proxy-$TARGET_ENVIRONMENT:$DOCKER_IMAGE_TAG - - name: 'Delete existing container group' - uses: 'azure/CLI@v2' + - name: "Delete existing container group" + uses: "azure/CLI@v2" with: inlineScript: | az -v @@ -91,7 +92,7 @@ jobs: --name "${{ env.CONTAINER_INSTANCE_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \ --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" - - name: 'Replace Env Vars and Secrets in ARM Yaml template' + - name: "Replace Env Vars and Secrets in ARM Yaml template" env: # Secrets / credentials for the app's resources AZURE_SERVICE_BUS_CONNECTION_STRING: ${{ secrets[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'AZURE_SERVICE_BUS_CONNECTION_STRING')] }} @@ -136,8 +137,8 @@ jobs: run: | ./azure-deployment/generate-manifest-from-template.sh - - name: 'Deploy group to Azure Container Instances' - uses: 'azure/CLI@v2' + - name: "Deploy group to Azure Container Instances" + uses: "azure/CLI@v2" with: inlineScript: | az -v @@ -145,15 +146,15 @@ jobs: --resource-group "${{ env.RESOURCE_GROUP_BASE_NAME }}-${{ env.TARGET_ENVIRONMENT }}" \ --file ./azure-deployment/azure-resource-manager-deployment-manifest.yml - - name: 'Re-generate the website links' + - name: "Re-generate the website links" env: WEB_BASE_URL: ${{ vars[format('{0}_{1}', env.TARGET_ENVIRONMENT_UPPER, 'WEB_BASE_URL')] }} run: | - sed -e "s#{{WEB_BASE_URL}}#$WEB_BASE_URL#" web/index-template.html > web/index.html + ./azure-deployment/generate-index-from-template.sh - - name: 'Update the website' - uses: 'azure/CLI@v2' + - name: "Update the website" + uses: "azure/CLI@v2" with: inlineScript: | az storage blob upload-batch -s web -d '$web' --account-name $STORAGE_ACCOUNT_NAME --overwrite diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 97113bf..d8ee58b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -27,5 +27,15 @@ jobs: - name: Run docker-compose run: cd ./tests-local-environment; docker compose up -d + # The Service Bus emulator has no docker compose healthcheck (its image is + # distroless), so `docker compose up -d` returns before it is accepting AMQP + # connections. Without this gate the MQ integration tests race its startup. + - name: Wait for Service Bus emulator + run: ./tests-local-environment/wait-for-mq-emulator.sh + - name: Run automated tests run: pytest + + - name: Docker compose logs (on failure) + if: failure() + run: cd ./tests-local-environment; docker compose logs --no-color diff --git a/CHANGELOG.md b/CHANGELOG.md index f8019a8..ebf50d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,35 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/). ### Removed +## [1.4.12] - 2026-08-03 + +### Changed + + - Added a small meta JSON file for use by the landing page so that it can show + the last updated timestamps without needing to access the full indices. + +## [1.4.11] - 2026-07-29 + +### Changed + + - Updated licence codelist list with new licence code 'other'. + +## [1.4.10] - 2026-07-27 + +### Changed + + - Expanded explanatory text, added 'Last updated: ' for the indices, and + brought IATI footer up to date. + - Pinned the Azure Service Bus emulator to version 1.1.2 in the local + development and test docker compose setups, rather than tracking `latest`. + +### Fixed + + - Fixed intermittent CI test failures caused by a race condition due to the + tests sometimes starting before the Azure Service Bus emulator was ready: CI + now waits for the emulator's health API before running the tests, and dumps + the docker compose logs if the tests fail. + ## [1.4.9] - 2026-05-25 ### Fixed diff --git a/README.md b/README.md index fedeecb..995c6bd 100644 --- a/README.md +++ b/README.md @@ -209,6 +209,19 @@ docker compose up --remove-orphans _Note: the `--remove-orphans` just helps keep things clean as you develop, and alter the setup._ +The Azure Service Bus emulator takes a while to start, and `docker compose` +cannot wait for it because the emulator image is distroless. Running the tests +before it is ready causes the MQ integration tests to fail with connection +errors, so wait for it with: + +``` +./tests-local-environment/wait-for-mq-emulator.sh +``` + +(For the local development docker compose setup, rather than the test setup, +pass the dev environment's health URL: +`./tests-local-environment/wait-for-mq-emulator.sh http://localhost:5300/health`) + Once this is running, run the tests with: ``` diff --git a/azure-deployment/generate-index-from-template.sh b/azure-deployment/generate-index-from-template.sh new file mode 100755 index 0000000..5be532a --- /dev/null +++ b/azure-deployment/generate-index-from-template.sh @@ -0,0 +1,17 @@ +#!/bin/env bash + +# This script is not intended to be run directly. +# It is run by the generic 'build-and-deploy' Github action, and by +# 'azure-provision/azure-create-resources.sh', so that both paths generate +# an identical web/index.html. +# +# Expects WEB_BASE_URL in the environment. BULK_DATA_SERVICE_VERSION is read +# from pyproject.toml unless already set. Must be run from the repo root. + +BULK_DATA_SERVICE_VERSION="${BULK_DATA_SERVICE_VERSION:-$(python3 -c "import tomllib; print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])")}" + +echo "Generating web/index.html (version ${BULK_DATA_SERVICE_VERSION}, base URL ${WEB_BASE_URL})" + +sed -e "s#{{WEB_BASE_URL}}#${WEB_BASE_URL}#" \ + -e "s#{{BULK_DATA_SERVICE_VERSION}}#${BULK_DATA_SERVICE_VERSION}#" \ + ./web/index-template.html > ./web/index.html diff --git a/azure-provision/azure-create-resources.sh b/azure-provision/azure-create-resources.sh index 5e55a8c..61edce9 100755 --- a/azure-provision/azure-create-resources.sh +++ b/azure-provision/azure-create-resources.sh @@ -178,7 +178,7 @@ AZURE_BASE_URL=$(az storage account show -n "$STORAGE_ACCOUNT_NAME" -g "$RESOURC AZURE_BASE_HOSTNAME="$(sed "s#https://\(.*\)/#\1#" <<< $AZURE_BASE_URL)" # WEB_BASE_URL is calculated above from TARGET_ENVIRONMENT, bearing in mind 'prod' doesn't have prefix -sed -e "s#{{WEB_BASE_URL}}#$WEB_BASE_URL#" web/index-template.html > web/index.html +./azure-deployment/generate-index-from-template.sh echo "Uploading index and 404 pages to storage account..." az storage blob upload-batch -s web -d '$web' --account-name $STORAGE_ACCOUNT_NAME --overwrite diff --git a/docker-compose.yml b/docker-compose.yml index 20a0192..35fc00b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,8 +52,9 @@ services: bds-local-dev-net: bds-local-dev-mq-emulator: - image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest - pull_policy: always + # NOTE: The emulator version is pinned deliberately; see the note in + # tests-local-environment/docker-compose.yml for why. + image: mcr.microsoft.com/azure-messaging/servicebus-emulator:1.1.2 volumes: # NOTE: The local dev docker compose environment and the local test docker # compose environment use the same Azure Service Bus configuration @@ -79,6 +80,7 @@ services: bds-local-dev-mq-db: image: "mcr.microsoft.com/azure-sql-edge:latest" + platform: linux/amd64 ports: - "1433:1433" environment: diff --git a/pyproject.toml b/pyproject.toml index a6b6bc4..1dd80bb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "bulk-data-service" -version = "1.4.9" +version = "1.4.12" requires-python = ">= 3.12.6" readme = "README.md" dependencies = [ @@ -76,4 +76,4 @@ max_complexity = 7 [tool.black] line-length = 119 target-version = ['py312'] -include='''src/.*/*.py$''' +include='''(src|tests)/.*\.py$''' diff --git a/src/bulk_data_service/data_validation_values.py b/src/bulk_data_service/data_validation_values.py index 1945105..5ca5d8d 100644 --- a/src/bulk_data_service/data_validation_values.py +++ b/src/bulk_data_service/data_validation_values.py @@ -255,6 +255,7 @@ LICENCE_LIST = [ "", + "other", "notspecified", "odc-pddl", "odc-odbl", diff --git a/src/bulk_data_service/dataset_indexing.py b/src/bulk_data_service/dataset_indexing.py index f2dcdcf..43a4733 100644 --- a/src/bulk_data_service/dataset_indexing.py +++ b/src/bulk_data_service/dataset_indexing.py @@ -34,6 +34,10 @@ def create_and_upload_indices( upload_index_json_to_azure(context, get_reporting_org_index_name(context), reporting_org_index) + upload_index_json_to_azure( + context, get_indices_meta_json_name(context), create_meta_json(context, index_creation_time) + ) + context.logger.info("Creation of indices finished") @@ -76,6 +80,37 @@ def create_reporting_org_index_json( return json.dumps(index, default=str, sort_keys=True, indent=True) +def create_meta_json( + context: BDSContext, + created_time: datetime, +) -> str: + """Generate meta data for the indices (currently just the timestamps) + + Args: + context (BDSContext): App context + created_time (datetime): Time the indices were created + + Returns: + str: JSON of indices meta data + """ + + index_created_entries = create_index_created_entries(created_time) + + return json.dumps( + { + "meta": { + "datasets_minimal": index_created_entries, + "datasets_full": index_created_entries, + "reporting_orgs": index_created_entries, + "zip": index_created_entries, + } + }, + default=str, + sort_keys=True, + indent=True, + ) + + def create_index_created_entries(created_time: datetime) -> dict[str, Any]: return {"index_created": created_time, "index_created_unix_timestamp": int(created_time.timestamp())} @@ -117,3 +152,7 @@ def get_dataset_index_name(context: BDSContext, index_type: str) -> str: def get_reporting_org_index_name(context: BDSContext) -> str: return "reporting-orgs" + + +def get_indices_meta_json_name(context: BDSContext) -> str: + return "indices-meta" diff --git a/tests-local-environment/docker-compose.yml b/tests-local-environment/docker-compose.yml index 95ea3d9..56f68b0 100644 --- a/tests-local-environment/docker-compose.yml +++ b/tests-local-environment/docker-compose.yml @@ -60,8 +60,12 @@ services: bds-local-test-net: bds-test-env-mq-emulator: - image: mcr.microsoft.com/azure-messaging/servicebus-emulator:latest - pull_policy: always + # NOTE: The emulator version is pinned deliberately. The `latest` tag moved + # to 2.0.1 in July 2026, which broke CI (the emulator takes long + # enough to start that the tests raced it), and 2.x carries an open + # bug where sends hang despite the emulator reporting healthy: + # https://github.com/Azure/azure-service-bus-emulator-installer/issues/135 + image: mcr.microsoft.com/azure-messaging/servicebus-emulator:1.1.2 volumes: # NOTE: The local dev docker compose environment and the local test docker # compose environment use the same Azure Service Bus configuration @@ -87,6 +91,7 @@ services: bds-test-env-mq-db: image: "mcr.microsoft.com/azure-sql-edge:latest" + platform: linux/amd64 ports: - "1434:1433" environment: diff --git a/tests-local-environment/wait-for-mq-emulator.sh b/tests-local-environment/wait-for-mq-emulator.sh new file mode 100755 index 0000000..bb32455 --- /dev/null +++ b/tests-local-environment/wait-for-mq-emulator.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash + +# Waits for the Azure Service Bus emulator to report itself healthy. +# +# The emulator exposes a health API (GET /health, which returns 503 while the +# emulator is starting up and 200 when ready). This can't be a docker compose +# healthcheck because the emulator image is distroless. See: +# https://github.com/Azure/azure-service-bus-emulator-installer/issues/88 + +# So we poll the health API from the host instead. +# +# Usage: wait-for-mq-emulator.sh [health-url] [timeout-seconds] +# +# The default URL is the local test environment (host port 5301). For the local +# development docker compose environment, pass http://localhost:5300/health + +set -euo pipefail + +HEALTH_URL="${1:-http://localhost:5301/health}" +TIMEOUT_SECONDS="${2:-120}" + +deadline=$((SECONDS + TIMEOUT_SECONDS)) + +until curl --fail --silent --show-error "${HEALTH_URL}" >/dev/null 2>&1; do + if ((SECONDS >= deadline)); then + echo "Service Bus emulator did not become healthy within ${TIMEOUT_SECONDS}s: ${HEALTH_URL}" >&2 + exit 1 + fi + sleep 2 +done + +echo "Service Bus emulator is healthy (${HEALTH_URL})" diff --git a/tests/integration/test_dataset_unregister.py b/tests/integration/test_dataset_unregister.py index 3e48b65..95c7c52 100644 --- a/tests/integration/test_dataset_unregister.py +++ b/tests/integration/test_dataset_unregister.py @@ -86,8 +86,7 @@ def test_remove_unregistered_dataset_from_azure_blob(get_and_clear_up_context): context["DATA_REGISTRY_BASE_URL"] = "http://localhost:3000/ckan-registration/datasets-01-1-dataset" checker_run(context, datasets_in_bds) - blob_service_client = \ - BlobServiceClient.from_connection_string(context["AZURE_STORAGE_CONNECTION_STRING"]) + blob_service_client = BlobServiceClient.from_connection_string(context["AZURE_STORAGE_CONNECTION_STRING"]) xml_container_name = get_azure_container_name(context, "xml") zip_container_name = get_azure_container_name(context, "zip") diff --git a/tests/integration/test_dataset_upload_to_azure.py b/tests/integration/test_dataset_upload_to_azure.py index f22e53f..1ddf57c 100644 --- a/tests/integration/test_dataset_upload_to_azure.py +++ b/tests/integration/test_dataset_upload_to_azure.py @@ -13,22 +13,26 @@ from utilities.azure import get_azure_blob_name, get_azure_container_name -@pytest.mark.parametrize("artifact_filename", [ - ("test_foundation_a-dataset-001.xml"), - ("test_foundation_a-dataset-001-utf-8-with-bom"), - ("test_foundation_a-dataset-001-utf-16-le"), - ("test_foundation_a-dataset-001-utf-16-be"), - ("test_foundation_a-dataset-001-utf-32-le"), - ("test_foundation_a-dataset-001-utf-32-be"), - ("test_foundation_a-dataset-001-iso-8859-1"), -]) +@pytest.mark.parametrize( + "artifact_filename", + [ + ("test_foundation_a-dataset-001.xml"), + ("test_foundation_a-dataset-001-utf-8-with-bom"), + ("test_foundation_a-dataset-001-utf-16-le"), + ("test_foundation_a-dataset-001-utf-16-be"), + ("test_foundation_a-dataset-001-utf-32-le"), + ("test_foundation_a-dataset-001-utf-32-be"), + ("test_foundation_a-dataset-001-iso-8859-1"), + ], +) def test_valid_dataset_azure_xml_upload(get_and_clear_up_context, artifact_filename): # noqa: F811 context = get_and_clear_up_context dataset_id = uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159") - context["DATA_REGISTRY_BASE_URL"] = ("http://localhost:3000/ckan-registration/datasets-01-1-dataset/" - "http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}").format(artifact_filename) + context["DATA_REGISTRY_BASE_URL"] = ( + "http://localhost:3000/ckan-registration/datasets-01-1-dataset/http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}" + ).format(artifact_filename) datasets_in_bds = {} checker_run(context, datasets_in_bds) @@ -39,22 +43,26 @@ def test_valid_dataset_azure_xml_upload(get_and_clear_up_context, artifact_filen assert dataset_contents_from_disk == dataset_contents_from_azure -@pytest.mark.parametrize("artifact_filename", [ - ("test_foundation_a-dataset-001.xml"), - ("test_foundation_a-dataset-001-utf-8-with-bom"), - ("test_foundation_a-dataset-001-utf-16-le"), - ("test_foundation_a-dataset-001-utf-16-be"), - ("test_foundation_a-dataset-001-utf-32-le"), - ("test_foundation_a-dataset-001-utf-32-be"), - ("test_foundation_a-dataset-001-iso-8859-1"), -]) +@pytest.mark.parametrize( + "artifact_filename", + [ + ("test_foundation_a-dataset-001.xml"), + ("test_foundation_a-dataset-001-utf-8-with-bom"), + ("test_foundation_a-dataset-001-utf-16-le"), + ("test_foundation_a-dataset-001-utf-16-be"), + ("test_foundation_a-dataset-001-utf-32-le"), + ("test_foundation_a-dataset-001-utf-32-be"), + ("test_foundation_a-dataset-001-iso-8859-1"), + ], +) def test_valid_dataset_azure_zip_upload(get_and_clear_up_context, artifact_filename): # noqa: F811 context = get_and_clear_up_context dataset_id = uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159") - context["DATA_REGISTRY_BASE_URL"] = ("http://localhost:3000/ckan-registration/datasets-01-1-dataset/" - "http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}").format(artifact_filename) + context["DATA_REGISTRY_BASE_URL"] = ( + "http://localhost:3000/ckan-registration/datasets-01-1-dataset/http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}" + ).format(artifact_filename) datasets_in_bds = {} checker_run(context, datasets_in_bds) @@ -62,24 +70,29 @@ def test_valid_dataset_azure_zip_upload(get_and_clear_up_context, artifact_filen dataset_contents_from_azure_zipped = download_dataset_from_azure(context, datasets_in_bds[dataset_id], "zip") - dataset_contents_from_azure = unzip_from_buffer("{}.xml".format(datasets_in_bds[dataset_id]["short_name"]), - dataset_contents_from_azure_zipped) + dataset_contents_from_azure = unzip_from_buffer( + "{}.xml".format(datasets_in_bds[dataset_id]["short_name"]), dataset_contents_from_azure_zipped + ) assert dataset_contents_from_disk == dataset_contents_from_azure -@pytest.mark.parametrize("artifact_filename", [ - ("test_foundation_a-dataset-empty.xml"), - ("test_foundation_a-dataset-html.xml"), - ("test_foundation_a-dataset.pdf"), -]) +@pytest.mark.parametrize( + "artifact_filename", + [ + ("test_foundation_a-dataset-empty.xml"), + ("test_foundation_a-dataset-html.xml"), + ("test_foundation_a-dataset.pdf"), + ], +) def test_invalid_dataset_no_azure_xml_upload(get_and_clear_up_context, artifact_filename): # noqa: F811 context = get_and_clear_up_context dataset_id = uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159") - context["DATA_REGISTRY_BASE_URL"] = ("http://localhost:3000/ckan-registration/datasets-01-1-dataset/" - "http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}").format(artifact_filename) + context["DATA_REGISTRY_BASE_URL"] = ( + "http://localhost:3000/ckan-registration/datasets-01-1-dataset/http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}" + ).format(artifact_filename) datasets_in_bds = {} checker_run(context, datasets_in_bds) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 8a08a31..f211845 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -20,7 +20,7 @@ def test_metrics_after_simple_add(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 0) + ("datasets_downloads_non_200", 0), ] for expected_result in expected_results: @@ -48,7 +48,7 @@ def test_metrics_after_new_registration(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 0) + ("datasets_downloads_non_200", 0), ] for expected_result in expected_results: @@ -76,7 +76,7 @@ def test_metrics_after_unregistration(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 1), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 0) + ("datasets_downloads_non_200", 0), ] for expected_result in expected_results: @@ -101,14 +101,14 @@ def test_metrics_with_success_then_immediate_404(get_and_clear_up_context): # n ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 1) + ("datasets_downloads_non_200", 1), ] for expected in expected_results: - assert context["prom_metrics"][expected[0]].set.called, \ - "metric name: {}".format(expected[0]) - assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), \ - "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.called, "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), "metric name: {}".format( + expected[0] + ) def test_metrics_with_success_then_delay_404(get_and_clear_up_context): # noqa: F811 @@ -133,14 +133,14 @@ def test_metrics_with_success_then_delay_404(get_and_clear_up_context): # noqa: ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 1) + ("datasets_downloads_non_200", 1), ] for expected in expected_results: - assert context["prom_metrics"][expected[0]].set.called, \ - "metric name: {}".format(expected[0]) - assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), \ - "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.called, "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), "metric name: {}".format( + expected[0] + ) def test_metrics_with_only_404(get_and_clear_up_context): # noqa: F811 @@ -157,11 +157,11 @@ def test_metrics_with_only_404(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), # 0 b/c HEAD reqs only after successful dl - ("datasets_downloads_non_200", 1) + ("datasets_downloads_non_200", 1), ] for expected in expected_results: - assert context["prom_metrics"][expected[0]].set.called, \ - "metric name: {}".format(expected[0]) - assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), \ - "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.called, "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), "metric name: {}".format( + expected[0] + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index ec9ca6d..b5687b2 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -12,7 +12,7 @@ def test_config_blob_storage_base_url_has_no_trailing_slash_1(): config = get_basic_config() - assert config["WEB_BASE_URL"] == 'http://127.0.0.1:10000/devstoreaccount1' + assert config["WEB_BASE_URL"] == "http://127.0.0.1:10000/devstoreaccount1" def test_config_blob_storage_base_url_has_no_trailing_slash_2(): @@ -21,7 +21,7 @@ def test_config_blob_storage_base_url_has_no_trailing_slash_2(): config = get_basic_config() - assert config["WEB_BASE_URL"] == 'http://127.0.0.1:10000/devstoreaccount1' + assert config["WEB_BASE_URL"] == "http://127.0.0.1:10000/devstoreaccount1" def test_config_dataset_timeouts_loaded(): diff --git a/tests/unit/test_dataset_registration.py b/tests/unit/test_dataset_registration.py index 00c9d77..a574d73 100644 --- a/tests/unit/test_dataset_registration.py +++ b/tests/unit/test_dataset_registration.py @@ -12,23 +12,28 @@ def get_level1_field_blanker(key): def field_blanker(dict, attribute_value, key=key): dict[key] = attribute_value + return partial(field_blanker, key=key) def get_level2_field_blanker(key1, key2): def field_blanker(dict, attribute_value, key1=key1, key2=key2): dict[key1][key2] = attribute_value + return partial(field_blanker, key1=key1, key2=key2) -@pytest.mark.parametrize("field_blanker", [ - get_level1_field_blanker("id"), - get_level1_field_blanker("name"), - get_level1_field_blanker("organization"), - get_level1_field_blanker("extras"), - get_level2_field_blanker("organization", "id"), - get_level2_field_blanker("organization", "name") - ]) +@pytest.mark.parametrize( + "field_blanker", + [ + get_level1_field_blanker("id"), + get_level1_field_blanker("name"), + get_level1_field_blanker("organization"), + get_level1_field_blanker("extras"), + get_level2_field_blanker("organization", "id"), + get_level2_field_blanker("organization", "name"), + ], +) @pytest.mark.parametrize("attribute_value", [None, "None", ""]) def test_incomplete_necessary_data_from_ckan(field_blanker, attribute_value): @@ -40,7 +45,7 @@ def test_incomplete_necessary_data_from_ckan(field_blanker, attribute_value): ckan_datasets = clean_datasets_metadata(logger, ckan_datasets) - assert(len(ckan_datasets) == 0) + assert len(ckan_datasets) == 0 def test_create_empty_dataset_error_occurred_defaults_to_false(): @@ -62,5 +67,5 @@ def test_missing_url_from_ckan(resources_value): registered_datasets_dtos = convert_datasets_metadata(ckan_datasets) - assert(len(registered_datasets_dtos) == 1) - assert(registered_datasets_dtos[uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159")]["source_url"] == "") + assert len(registered_datasets_dtos) == 1 + assert registered_datasets_dtos[uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159")]["source_url"] == "" diff --git a/tests/unit/test_utilities_http.py b/tests/unit/test_utilities_http.py index 5630145..16e253d 100644 --- a/tests/unit/test_utilities_http.py +++ b/tests/unit/test_utilities_http.py @@ -5,21 +5,27 @@ from utilities.http import add_qs_params_to_url, parse_last_modified_header -@pytest.mark.parametrize("input,expected", [ - ("Fri, 06 Sep 2024 13:08:28 GMT", datetime.datetime(2024, 9, 6, 13, 8, 28, 0, datetime.timezone.utc)), - ("Wed, 21 Oct 2015 07:28:00 GMT", datetime.datetime(2015, 10, 21, 7, 28, 0, 0, datetime.timezone.utc)), - ("Wed, 21 Oct 2015 07:28:00", None), - ("Wed, 21 October 2015 07:28:00 +00:00", None), -]) +@pytest.mark.parametrize( + "input,expected", + [ + ("Fri, 06 Sep 2024 13:08:28 GMT", datetime.datetime(2024, 9, 6, 13, 8, 28, 0, datetime.timezone.utc)), + ("Wed, 21 Oct 2015 07:28:00 GMT", datetime.datetime(2015, 10, 21, 7, 28, 0, 0, datetime.timezone.utc)), + ("Wed, 21 Oct 2015 07:28:00", None), + ("Wed, 21 October 2015 07:28:00 +00:00", None), + ], +) def test_parse_http_last_modified_header(input, expected): assert parse_last_modified_header(input) == expected -@pytest.mark.parametrize("input,params,expected", [ - ("http://www.a.com", {}, "http://www.a.com"), - ("http://www.a.com", {"one": 1}, "http://www.a.com?one=1"), - ("http://www.a.com", {"one": 1, "two": 2}, "http://www.a.com?one=1&two=2"), - ("http://www.a.com?one=1", {"one": "updated"}, "http://www.a.com?one=updated"), -]) +@pytest.mark.parametrize( + "input,params,expected", + [ + ("http://www.a.com", {}, "http://www.a.com"), + ("http://www.a.com", {"one": 1}, "http://www.a.com?one=1"), + ("http://www.a.com", {"one": 1, "two": 2}, "http://www.a.com?one=1&two=2"), + ("http://www.a.com?one=1", {"one": "updated"}, "http://www.a.com?one=updated"), + ], +) def test_add_qs_params_to_url(input, params, expected): assert add_qs_params_to_url(input, params) == expected diff --git a/tests/unit/test_utilities_misc.py b/tests/unit/test_utilities_misc.py index 2efe9e2..7979209 100644 --- a/tests/unit/test_utilities_misc.py +++ b/tests/unit/test_utilities_misc.py @@ -32,7 +32,7 @@ def test_get_hash(): hash = get_hash(yiplActivitiesXmlFile, "utf-8") - assert(hash == "3486d4cee556d2584020bed2c86305465b8b3880") + assert hash == "3486d4cee556d2584020bed2c86305465b8b3880" def test_get_hash_excluding_generated_timestamp(): @@ -58,173 +58,151 @@ def test_get_hash_excluding_generated_timestamp(): hash = get_hash_excluding_generated_timestamp(yiplActivitiesXmlFile, "utf-8") - assert(hash == "759eaa39276381f3fc146232cefd2111a2abc199") + assert hash == "759eaa39276381f3fc146232cefd2111a2abc199" -@pytest.mark.parametrize("str_to_test,expected_result", [ - ("None", False), - ("not a uuid", False), - ("1a3e3f42-6704-4adf-897a-9bdf5b854a00", True) -]) +@pytest.mark.parametrize( + "str_to_test,expected_result", + [("None", False), ("not a uuid", False), ("1a3e3f42-6704-4adf-897a-9bdf5b854a00", True)], +) def test_is_str_valid_uuid(str_to_test, expected_result): assert is_str_valid_uuid(str_to_test) == expected_result -@pytest.mark.parametrize("input,structure,expected", [ - ({"a": 10}, {"a" : None}, {"a": 10}), - ({"a": None}, {"a" : None}, {"a": None}), - ({"a": 10, "b": "should be filtered"}, {"a" : None}, {"a": 10}), - ({"a": 10, "b": "should be filtered", "c" : "also filtered"}, {"a" : None}, {"a": 10}), - ({"a": 10, "b": "included"}, {"a" : None, "b": None}, {"a": 10, "b": "included"}), - ({"b": "included"}, {"a" : None, "b": None}, {"b": "included"}), - ({"c": 10}, {"a" : None, "b": None}, {}), - ({}, {"a" : None, "b": None}, {}), - ({}, {}, {}), -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({"a": 10}, {"a": None}, {"a": 10}), + ({"a": None}, {"a": None}, {"a": None}), + ({"a": 10, "b": "should be filtered"}, {"a": None}, {"a": 10}), + ({"a": 10, "b": "should be filtered", "c": "also filtered"}, {"a": None}, {"a": 10}), + ({"a": 10, "b": "included"}, {"a": None, "b": None}, {"a": 10, "b": "included"}), + ({"b": "included"}, {"a": None, "b": None}, {"b": "included"}), + ({"c": 10}, {"a": None, "b": None}, {}), + ({}, {"a": None, "b": None}, {}), + ({}, {}, {}), + ], +) def test_filter_dict_atomic_value(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,structure,expected", [ - ({}, - {"a": {"include": None}}, - {}), - - ({"a": {}}, - {"a": {"include": None}}, - {"a": {}}), - - ({"a": {"include": 10, "filter out": 20}}, - {"a": {"include": None}}, - {"a": {"include": 10}}), - - ({"a": {"filter out": 20}}, - {"a": {"include": None}}, - {"a": {}}), - - ({"a": {"filter out": 20}}, - {"a": {}}, - {"a": {}}), - - ({"a": None}, - {"a": {}}, - {"a": None}), - - ({"a": None}, - {"a": {"b": None}}, - {"a": None}), - - ({"a": 10}, - {"a": {"b": {}}}, - {"a": 10}), - - ({"a": ["test"]}, - {"a": {"b": {}}}, - {"a": ["test"]}), - - -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({}, {"a": {"include": None}}, {}), + ({"a": {}}, {"a": {"include": None}}, {"a": {}}), + ({"a": {"include": 10, "filter out": 20}}, {"a": {"include": None}}, {"a": {"include": 10}}), + ({"a": {"filter out": 20}}, {"a": {"include": None}}, {"a": {}}), + ({"a": {"filter out": 20}}, {"a": {}}, {"a": {}}), + ({"a": None}, {"a": {}}, {"a": None}), + ({"a": None}, {"a": {"b": None}}, {"a": None}), + ({"a": 10}, {"a": {"b": {}}}, {"a": 10}), + ({"a": ["test"]}, {"a": {"b": {}}}, {"a": ["test"]}), + ], +) def test_filter_dict_nested_dict(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,structure,expected", [ - ({}, - {"a": []}, - {}), - - ({"a": []}, - {"a": None}, - {"a": []}), - - ({"a": ["one", "two"]}, - {"a": None}, - {"a": ["one", "two"]}), - -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({}, {"a": []}, {}), + ({"a": []}, {"a": None}, {"a": []}), + ({"a": ["one", "two"]}, {"a": None}, {"a": ["one", "two"]}), + ], +) def test_filter_dict_with_list_no_dict_items(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,structure,expected", [ - ({}, - {"a": [{}]}, - {}), - - ({"a": []}, - {"a": [{}]}, - {"a": []}), - - ({"a": ["one", "two"]}, - {"a": [{}]}, - {"a": [{}, {}]}), - - ({"a": [{"include": 10, "filter": 20}]}, - {"a": [{"include": None}]}, - {"a": [{"include": 10}]}), - -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({}, {"a": [{}]}, {}), + ({"a": []}, {"a": [{}]}, {"a": []}), + ({"a": ["one", "two"]}, {"a": [{}]}, {"a": [{}, {}]}), + ({"a": [{"include": 10, "filter": 20}]}, {"a": [{"include": None}]}, {"a": [{"include": 10}]}), + ], +) def test_filter_dict_with_list_with_dict_items(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,expected", - [ - ("", False), - ("String without an opening element", False), - ("Contents with something before correct element ", False), - ("Contents with something before correct element ", False), - ('String without an opening element', False), - ('String without an opening element', False), - ("iati-activities more here", False), - ("iati-activities> and more content", False), - (" and more content", False), - ("", False), + ("Contents with something before correct element ", False), + ('String without an opening element', False), + ('String without an opening element', False), + ("iati-activities more here", False), + ("iati-activities> and more content", False), + (" and more content", False), + (" + --> - + - 404 - Page or file not found