diff --git a/pyproject.toml b/pyproject.toml index c858573..6f06f01 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -76,4 +76,4 @@ max_complexity = 7 [tool.black] line-length = 119 target-version = ['py312'] -include='''src/.*/*.py$''' +include='''(src|tests)/.*\.py$''' diff --git a/tests/integration/test_dataset_unregister.py b/tests/integration/test_dataset_unregister.py index 3e48b65..95c7c52 100644 --- a/tests/integration/test_dataset_unregister.py +++ b/tests/integration/test_dataset_unregister.py @@ -86,8 +86,7 @@ def test_remove_unregistered_dataset_from_azure_blob(get_and_clear_up_context): context["DATA_REGISTRY_BASE_URL"] = "http://localhost:3000/ckan-registration/datasets-01-1-dataset" checker_run(context, datasets_in_bds) - blob_service_client = \ - BlobServiceClient.from_connection_string(context["AZURE_STORAGE_CONNECTION_STRING"]) + blob_service_client = BlobServiceClient.from_connection_string(context["AZURE_STORAGE_CONNECTION_STRING"]) xml_container_name = get_azure_container_name(context, "xml") zip_container_name = get_azure_container_name(context, "zip") diff --git a/tests/integration/test_dataset_upload_to_azure.py b/tests/integration/test_dataset_upload_to_azure.py index f22e53f..1ddf57c 100644 --- a/tests/integration/test_dataset_upload_to_azure.py +++ b/tests/integration/test_dataset_upload_to_azure.py @@ -13,22 +13,26 @@ from utilities.azure import get_azure_blob_name, get_azure_container_name -@pytest.mark.parametrize("artifact_filename", [ - ("test_foundation_a-dataset-001.xml"), - ("test_foundation_a-dataset-001-utf-8-with-bom"), - ("test_foundation_a-dataset-001-utf-16-le"), - ("test_foundation_a-dataset-001-utf-16-be"), - ("test_foundation_a-dataset-001-utf-32-le"), - ("test_foundation_a-dataset-001-utf-32-be"), - ("test_foundation_a-dataset-001-iso-8859-1"), -]) +@pytest.mark.parametrize( + "artifact_filename", + [ + ("test_foundation_a-dataset-001.xml"), + ("test_foundation_a-dataset-001-utf-8-with-bom"), + ("test_foundation_a-dataset-001-utf-16-le"), + ("test_foundation_a-dataset-001-utf-16-be"), + ("test_foundation_a-dataset-001-utf-32-le"), + ("test_foundation_a-dataset-001-utf-32-be"), + ("test_foundation_a-dataset-001-iso-8859-1"), + ], +) def test_valid_dataset_azure_xml_upload(get_and_clear_up_context, artifact_filename): # noqa: F811 context = get_and_clear_up_context dataset_id = uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159") - context["DATA_REGISTRY_BASE_URL"] = ("http://localhost:3000/ckan-registration/datasets-01-1-dataset/" - "http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}").format(artifact_filename) + context["DATA_REGISTRY_BASE_URL"] = ( + "http://localhost:3000/ckan-registration/datasets-01-1-dataset/http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}" + ).format(artifact_filename) datasets_in_bds = {} checker_run(context, datasets_in_bds) @@ -39,22 +43,26 @@ def test_valid_dataset_azure_xml_upload(get_and_clear_up_context, artifact_filen assert dataset_contents_from_disk == dataset_contents_from_azure -@pytest.mark.parametrize("artifact_filename", [ - ("test_foundation_a-dataset-001.xml"), - ("test_foundation_a-dataset-001-utf-8-with-bom"), - ("test_foundation_a-dataset-001-utf-16-le"), - ("test_foundation_a-dataset-001-utf-16-be"), - ("test_foundation_a-dataset-001-utf-32-le"), - ("test_foundation_a-dataset-001-utf-32-be"), - ("test_foundation_a-dataset-001-iso-8859-1"), -]) +@pytest.mark.parametrize( + "artifact_filename", + [ + ("test_foundation_a-dataset-001.xml"), + ("test_foundation_a-dataset-001-utf-8-with-bom"), + ("test_foundation_a-dataset-001-utf-16-le"), + ("test_foundation_a-dataset-001-utf-16-be"), + ("test_foundation_a-dataset-001-utf-32-le"), + ("test_foundation_a-dataset-001-utf-32-be"), + ("test_foundation_a-dataset-001-iso-8859-1"), + ], +) def test_valid_dataset_azure_zip_upload(get_and_clear_up_context, artifact_filename): # noqa: F811 context = get_and_clear_up_context dataset_id = uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159") - context["DATA_REGISTRY_BASE_URL"] = ("http://localhost:3000/ckan-registration/datasets-01-1-dataset/" - "http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}").format(artifact_filename) + context["DATA_REGISTRY_BASE_URL"] = ( + "http://localhost:3000/ckan-registration/datasets-01-1-dataset/http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}" + ).format(artifact_filename) datasets_in_bds = {} checker_run(context, datasets_in_bds) @@ -62,24 +70,29 @@ def test_valid_dataset_azure_zip_upload(get_and_clear_up_context, artifact_filen dataset_contents_from_azure_zipped = download_dataset_from_azure(context, datasets_in_bds[dataset_id], "zip") - dataset_contents_from_azure = unzip_from_buffer("{}.xml".format(datasets_in_bds[dataset_id]["short_name"]), - dataset_contents_from_azure_zipped) + dataset_contents_from_azure = unzip_from_buffer( + "{}.xml".format(datasets_in_bds[dataset_id]["short_name"]), dataset_contents_from_azure_zipped + ) assert dataset_contents_from_disk == dataset_contents_from_azure -@pytest.mark.parametrize("artifact_filename", [ - ("test_foundation_a-dataset-empty.xml"), - ("test_foundation_a-dataset-html.xml"), - ("test_foundation_a-dataset.pdf"), -]) +@pytest.mark.parametrize( + "artifact_filename", + [ + ("test_foundation_a-dataset-empty.xml"), + ("test_foundation_a-dataset-html.xml"), + ("test_foundation_a-dataset.pdf"), + ], +) def test_invalid_dataset_no_azure_xml_upload(get_and_clear_up_context, artifact_filename): # noqa: F811 context = get_and_clear_up_context dataset_id = uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159") - context["DATA_REGISTRY_BASE_URL"] = ("http://localhost:3000/ckan-registration/datasets-01-1-dataset/" - "http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}").format(artifact_filename) + context["DATA_REGISTRY_BASE_URL"] = ( + "http://localhost:3000/ckan-registration/datasets-01-1-dataset/http%3A%2F%2Flocalhost%3A3000%2Fdata%2F{}" + ).format(artifact_filename) datasets_in_bds = {} checker_run(context, datasets_in_bds) diff --git a/tests/integration/test_prometheus_metrics.py b/tests/integration/test_prometheus_metrics.py index 8a08a31..f211845 100644 --- a/tests/integration/test_prometheus_metrics.py +++ b/tests/integration/test_prometheus_metrics.py @@ -20,7 +20,7 @@ def test_metrics_after_simple_add(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 0) + ("datasets_downloads_non_200", 0), ] for expected_result in expected_results: @@ -48,7 +48,7 @@ def test_metrics_after_new_registration(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 0) + ("datasets_downloads_non_200", 0), ] for expected_result in expected_results: @@ -76,7 +76,7 @@ def test_metrics_after_unregistration(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 1), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 0) + ("datasets_downloads_non_200", 0), ] for expected_result in expected_results: @@ -101,14 +101,14 @@ def test_metrics_with_success_then_immediate_404(get_and_clear_up_context): # n ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 1) + ("datasets_downloads_non_200", 1), ] for expected in expected_results: - assert context["prom_metrics"][expected[0]].set.called, \ - "metric name: {}".format(expected[0]) - assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), \ - "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.called, "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), "metric name: {}".format( + expected[0] + ) def test_metrics_with_success_then_delay_404(get_and_clear_up_context): # noqa: F811 @@ -133,14 +133,14 @@ def test_metrics_with_success_then_delay_404(get_and_clear_up_context): # noqa: ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), - ("datasets_downloads_non_200", 1) + ("datasets_downloads_non_200", 1), ] for expected in expected_results: - assert context["prom_metrics"][expected[0]].set.called, \ - "metric name: {}".format(expected[0]) - assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), \ - "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.called, "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), "metric name: {}".format( + expected[0] + ) def test_metrics_with_only_404(get_and_clear_up_context): # noqa: F811 @@ -157,11 +157,11 @@ def test_metrics_with_only_404(get_and_clear_up_context): # noqa: F811 ("datasets_unregistered", 0), ("datasets_expired", 0), ("datasets_head_request_non_200", 0), # 0 b/c HEAD reqs only after successful dl - ("datasets_downloads_non_200", 1) + ("datasets_downloads_non_200", 1), ] for expected in expected_results: - assert context["prom_metrics"][expected[0]].set.called, \ - "metric name: {}".format(expected[0]) - assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), \ - "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.called, "metric name: {}".format(expected[0]) + assert context["prom_metrics"][expected[0]].set.call_args.args == (expected[1],), "metric name: {}".format( + expected[0] + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py index ec9ca6d..b5687b2 100644 --- a/tests/unit/test_config.py +++ b/tests/unit/test_config.py @@ -12,7 +12,7 @@ def test_config_blob_storage_base_url_has_no_trailing_slash_1(): config = get_basic_config() - assert config["WEB_BASE_URL"] == 'http://127.0.0.1:10000/devstoreaccount1' + assert config["WEB_BASE_URL"] == "http://127.0.0.1:10000/devstoreaccount1" def test_config_blob_storage_base_url_has_no_trailing_slash_2(): @@ -21,7 +21,7 @@ def test_config_blob_storage_base_url_has_no_trailing_slash_2(): config = get_basic_config() - assert config["WEB_BASE_URL"] == 'http://127.0.0.1:10000/devstoreaccount1' + assert config["WEB_BASE_URL"] == "http://127.0.0.1:10000/devstoreaccount1" def test_config_dataset_timeouts_loaded(): diff --git a/tests/unit/test_dataset_registration.py b/tests/unit/test_dataset_registration.py index 00c9d77..a574d73 100644 --- a/tests/unit/test_dataset_registration.py +++ b/tests/unit/test_dataset_registration.py @@ -12,23 +12,28 @@ def get_level1_field_blanker(key): def field_blanker(dict, attribute_value, key=key): dict[key] = attribute_value + return partial(field_blanker, key=key) def get_level2_field_blanker(key1, key2): def field_blanker(dict, attribute_value, key1=key1, key2=key2): dict[key1][key2] = attribute_value + return partial(field_blanker, key1=key1, key2=key2) -@pytest.mark.parametrize("field_blanker", [ - get_level1_field_blanker("id"), - get_level1_field_blanker("name"), - get_level1_field_blanker("organization"), - get_level1_field_blanker("extras"), - get_level2_field_blanker("organization", "id"), - get_level2_field_blanker("organization", "name") - ]) +@pytest.mark.parametrize( + "field_blanker", + [ + get_level1_field_blanker("id"), + get_level1_field_blanker("name"), + get_level1_field_blanker("organization"), + get_level1_field_blanker("extras"), + get_level2_field_blanker("organization", "id"), + get_level2_field_blanker("organization", "name"), + ], +) @pytest.mark.parametrize("attribute_value", [None, "None", ""]) def test_incomplete_necessary_data_from_ckan(field_blanker, attribute_value): @@ -40,7 +45,7 @@ def test_incomplete_necessary_data_from_ckan(field_blanker, attribute_value): ckan_datasets = clean_datasets_metadata(logger, ckan_datasets) - assert(len(ckan_datasets) == 0) + assert len(ckan_datasets) == 0 def test_create_empty_dataset_error_occurred_defaults_to_false(): @@ -62,5 +67,5 @@ def test_missing_url_from_ckan(resources_value): registered_datasets_dtos = convert_datasets_metadata(ckan_datasets) - assert(len(registered_datasets_dtos) == 1) - assert(registered_datasets_dtos[uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159")]["source_url"] == "") + assert len(registered_datasets_dtos) == 1 + assert registered_datasets_dtos[uuid.UUID("c8a40aa5-9f31-4bcf-a36f-51c1fc2cc159")]["source_url"] == "" diff --git a/tests/unit/test_utilities_http.py b/tests/unit/test_utilities_http.py index 5630145..16e253d 100644 --- a/tests/unit/test_utilities_http.py +++ b/tests/unit/test_utilities_http.py @@ -5,21 +5,27 @@ from utilities.http import add_qs_params_to_url, parse_last_modified_header -@pytest.mark.parametrize("input,expected", [ - ("Fri, 06 Sep 2024 13:08:28 GMT", datetime.datetime(2024, 9, 6, 13, 8, 28, 0, datetime.timezone.utc)), - ("Wed, 21 Oct 2015 07:28:00 GMT", datetime.datetime(2015, 10, 21, 7, 28, 0, 0, datetime.timezone.utc)), - ("Wed, 21 Oct 2015 07:28:00", None), - ("Wed, 21 October 2015 07:28:00 +00:00", None), -]) +@pytest.mark.parametrize( + "input,expected", + [ + ("Fri, 06 Sep 2024 13:08:28 GMT", datetime.datetime(2024, 9, 6, 13, 8, 28, 0, datetime.timezone.utc)), + ("Wed, 21 Oct 2015 07:28:00 GMT", datetime.datetime(2015, 10, 21, 7, 28, 0, 0, datetime.timezone.utc)), + ("Wed, 21 Oct 2015 07:28:00", None), + ("Wed, 21 October 2015 07:28:00 +00:00", None), + ], +) def test_parse_http_last_modified_header(input, expected): assert parse_last_modified_header(input) == expected -@pytest.mark.parametrize("input,params,expected", [ - ("http://www.a.com", {}, "http://www.a.com"), - ("http://www.a.com", {"one": 1}, "http://www.a.com?one=1"), - ("http://www.a.com", {"one": 1, "two": 2}, "http://www.a.com?one=1&two=2"), - ("http://www.a.com?one=1", {"one": "updated"}, "http://www.a.com?one=updated"), -]) +@pytest.mark.parametrize( + "input,params,expected", + [ + ("http://www.a.com", {}, "http://www.a.com"), + ("http://www.a.com", {"one": 1}, "http://www.a.com?one=1"), + ("http://www.a.com", {"one": 1, "two": 2}, "http://www.a.com?one=1&two=2"), + ("http://www.a.com?one=1", {"one": "updated"}, "http://www.a.com?one=updated"), + ], +) def test_add_qs_params_to_url(input, params, expected): assert add_qs_params_to_url(input, params) == expected diff --git a/tests/unit/test_utilities_misc.py b/tests/unit/test_utilities_misc.py index 2efe9e2..7979209 100644 --- a/tests/unit/test_utilities_misc.py +++ b/tests/unit/test_utilities_misc.py @@ -32,7 +32,7 @@ def test_get_hash(): hash = get_hash(yiplActivitiesXmlFile, "utf-8") - assert(hash == "3486d4cee556d2584020bed2c86305465b8b3880") + assert hash == "3486d4cee556d2584020bed2c86305465b8b3880" def test_get_hash_excluding_generated_timestamp(): @@ -58,173 +58,151 @@ def test_get_hash_excluding_generated_timestamp(): hash = get_hash_excluding_generated_timestamp(yiplActivitiesXmlFile, "utf-8") - assert(hash == "759eaa39276381f3fc146232cefd2111a2abc199") + assert hash == "759eaa39276381f3fc146232cefd2111a2abc199" -@pytest.mark.parametrize("str_to_test,expected_result", [ - ("None", False), - ("not a uuid", False), - ("1a3e3f42-6704-4adf-897a-9bdf5b854a00", True) -]) +@pytest.mark.parametrize( + "str_to_test,expected_result", + [("None", False), ("not a uuid", False), ("1a3e3f42-6704-4adf-897a-9bdf5b854a00", True)], +) def test_is_str_valid_uuid(str_to_test, expected_result): assert is_str_valid_uuid(str_to_test) == expected_result -@pytest.mark.parametrize("input,structure,expected", [ - ({"a": 10}, {"a" : None}, {"a": 10}), - ({"a": None}, {"a" : None}, {"a": None}), - ({"a": 10, "b": "should be filtered"}, {"a" : None}, {"a": 10}), - ({"a": 10, "b": "should be filtered", "c" : "also filtered"}, {"a" : None}, {"a": 10}), - ({"a": 10, "b": "included"}, {"a" : None, "b": None}, {"a": 10, "b": "included"}), - ({"b": "included"}, {"a" : None, "b": None}, {"b": "included"}), - ({"c": 10}, {"a" : None, "b": None}, {}), - ({}, {"a" : None, "b": None}, {}), - ({}, {}, {}), -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({"a": 10}, {"a": None}, {"a": 10}), + ({"a": None}, {"a": None}, {"a": None}), + ({"a": 10, "b": "should be filtered"}, {"a": None}, {"a": 10}), + ({"a": 10, "b": "should be filtered", "c": "also filtered"}, {"a": None}, {"a": 10}), + ({"a": 10, "b": "included"}, {"a": None, "b": None}, {"a": 10, "b": "included"}), + ({"b": "included"}, {"a": None, "b": None}, {"b": "included"}), + ({"c": 10}, {"a": None, "b": None}, {}), + ({}, {"a": None, "b": None}, {}), + ({}, {}, {}), + ], +) def test_filter_dict_atomic_value(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,structure,expected", [ - ({}, - {"a": {"include": None}}, - {}), - - ({"a": {}}, - {"a": {"include": None}}, - {"a": {}}), - - ({"a": {"include": 10, "filter out": 20}}, - {"a": {"include": None}}, - {"a": {"include": 10}}), - - ({"a": {"filter out": 20}}, - {"a": {"include": None}}, - {"a": {}}), - - ({"a": {"filter out": 20}}, - {"a": {}}, - {"a": {}}), - - ({"a": None}, - {"a": {}}, - {"a": None}), - - ({"a": None}, - {"a": {"b": None}}, - {"a": None}), - - ({"a": 10}, - {"a": {"b": {}}}, - {"a": 10}), - - ({"a": ["test"]}, - {"a": {"b": {}}}, - {"a": ["test"]}), - - -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({}, {"a": {"include": None}}, {}), + ({"a": {}}, {"a": {"include": None}}, {"a": {}}), + ({"a": {"include": 10, "filter out": 20}}, {"a": {"include": None}}, {"a": {"include": 10}}), + ({"a": {"filter out": 20}}, {"a": {"include": None}}, {"a": {}}), + ({"a": {"filter out": 20}}, {"a": {}}, {"a": {}}), + ({"a": None}, {"a": {}}, {"a": None}), + ({"a": None}, {"a": {"b": None}}, {"a": None}), + ({"a": 10}, {"a": {"b": {}}}, {"a": 10}), + ({"a": ["test"]}, {"a": {"b": {}}}, {"a": ["test"]}), + ], +) def test_filter_dict_nested_dict(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,structure,expected", [ - ({}, - {"a": []}, - {}), - - ({"a": []}, - {"a": None}, - {"a": []}), - - ({"a": ["one", "two"]}, - {"a": None}, - {"a": ["one", "two"]}), - -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({}, {"a": []}, {}), + ({"a": []}, {"a": None}, {"a": []}), + ({"a": ["one", "two"]}, {"a": None}, {"a": ["one", "two"]}), + ], +) def test_filter_dict_with_list_no_dict_items(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,structure,expected", [ - ({}, - {"a": [{}]}, - {}), - - ({"a": []}, - {"a": [{}]}, - {"a": []}), - - ({"a": ["one", "two"]}, - {"a": [{}]}, - {"a": [{}, {}]}), - - ({"a": [{"include": 10, "filter": 20}]}, - {"a": [{"include": None}]}, - {"a": [{"include": 10}]}), - -]) +@pytest.mark.parametrize( + "input,structure,expected", + [ + ({}, {"a": [{}]}, {}), + ({"a": []}, {"a": [{}]}, {"a": []}), + ({"a": ["one", "two"]}, {"a": [{}]}, {"a": [{}, {}]}), + ({"a": [{"include": 10, "filter": 20}]}, {"a": [{"include": None}]}, {"a": [{"include": 10}]}), + ], +) def test_filter_dict_with_list_with_dict_items(input, structure, expected): assert filter_dict_by_structure(input, structure) == expected -@pytest.mark.parametrize("input,expected", - [ - ("", False), - ("String without an opening element", False), - ("Contents with something before correct element ", False), - ("Contents with something before correct element ", False), - ('String without an opening element', False), - ('String without an opening element', False), - ("iati-activities more here", False), - ("iati-activities> and more content", False), - (" and more content", False), - ("", False), + ("Contents with something before correct element ", False), + ('String without an opening element', False), + ('String without an opening element', False), + ("iati-activities more here", False), + ("iati-activities> and more content", False), + (" and more content", False), + (" + --> - + -