diff --git a/import-automation/executor/requirements.txt b/import-automation/executor/requirements.txt index 7ee91417a2..f052145a68 100644 --- a/import-automation/executor/requirements.txt +++ b/import-automation/executor/requirements.txt @@ -1,6 +1,7 @@ # Requirements for Python scripts in this repo that have automation enabled! absl-py +aiolimiter arcgis2geojson beautifulsoup4 chardet diff --git a/scripts/us_census/acs5yr/subject_tables/common/make_column_map.log b/scripts/us_census/acs5yr/subject_tables/common/make_column_map.log deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/scripts/us_census/acs5yr/subject_tables/s1201/generate_col_map.py b/scripts/us_census/acs5yr/subject_tables/s1201/generate_col_map.py index 8bc25480ad..fb04b5dc26 100644 --- a/scripts/us_census/acs5yr/subject_tables/s1201/generate_col_map.py +++ b/scripts/us_census/acs5yr/subject_tables/s1201/generate_col_map.py @@ -25,11 +25,17 @@ from collections import OrderedDict import pandas as pd -# Allows the following module imports to work when running as a script +# Allows the sibling/parent module imports to work when running as a script or module _SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(_SCRIPT_PATH, - '../common')) # for statvar_dcid_generator -from generate_col_map import generate_stat_var_map +_curr = _SCRIPT_PATH +while _curr and _curr != os.path.dirname(_curr): + if os.path.exists(os.path.join(_curr, '.git')) or os.path.exists( + os.path.join(_curr, 'WORKSPACE')): + sys.path.append(_curr) + break + _curr = os.path.dirname(_curr) + +from scripts.us_census.acs5yr.subject_tables.common.generate_col_map import generate_stat_var_map def process_zip_file(zip_file_path, diff --git a/scripts/us_census/acs5yr/subject_tables/s1201/process.py b/scripts/us_census/acs5yr/subject_tables/s1201/process.py index 716b0964dc..06de4675f1 100644 --- a/scripts/us_census/acs5yr/subject_tables/s1201/process.py +++ b/scripts/us_census/acs5yr/subject_tables/s1201/process.py @@ -12,9 +12,20 @@ # See the License for the specific language governing permissions and # limitations under the License. """Generic proces module to generate the csv/tmcf and csv""" -# TODO: Add unit tests +# Allows the sibling/parent module imports to work when running as a script or module import os import sys + +_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) +_curr = _SCRIPT_PATH +while _curr and _curr != os.path.dirname(_curr): + if os.path.exists(os.path.join(_curr, '.git')) or os.path.exists( + os.path.join(_curr, 'WORKSPACE')): + sys.path.append(_curr) + break + _curr = os.path.dirname(_curr) + +# TODO: Add unit tests import json from zipfile import ZipFile import pandas as pd @@ -22,13 +33,8 @@ from absl import app, flags # TODO: logs from the column map step is empty when invoked from here, needs to be checked -from .generate_col_map import generate_stat_var_map, process_zip_file - -# Allows the following module imports to work when running as a script -_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(_SCRIPT_PATH, - '../common')) # for col_map_generator, data_loader -from data_loader import process_subject_tables +from scripts.us_census.acs5yr.subject_tables.s1201.generate_col_map import process_zip_file +from scripts.us_census.acs5yr.subject_tables.common.data_loader import process_subject_tables FLAGS = flags.FLAGS flags.DEFINE_string( @@ -56,9 +62,8 @@ def set_column_map(input_path, spec_path, output_dir): generated_col_map = process_zip_file(input_path, spec_path, write_output=False) - f = open(os.path.join(output_dir, 'column_map.json'), 'w') - json.dump(generated_col_map, f, indent=4) - f.close() + with open(os.path.join(output_dir, 'column_map.json'), 'w') as f: + json.dump(generated_col_map, f, indent=4) def main(argv): @@ -71,6 +76,8 @@ def main(argv): has_percent = FLAGS.has_percent debug = FLAGS.debug + os.makedirs(output_dir, exist_ok=True) + # TODO: remove the constraint of inputs being only zip file # context: the current implementation of the column map generator accepts # only zip files as input and we will need to add new methods to handle inputs diff --git a/scripts/us_census/api_utils/census_api_config_fetcher.py b/scripts/us_census/api_utils/census_api_config_fetcher.py index 0d14367047..8cc5583238 100644 --- a/scripts/us_census/api_utils/census_api_config_fetcher.py +++ b/scripts/us_census/api_utils/census_api_config_fetcher.py @@ -29,9 +29,9 @@ CONFIG_PATH_ = os.path.join(module_dir_, 'config_files') path.insert(1, os.path.join(module_dir_, '../../../')) -from .download_utils import download_url_list_iterations, async_save_resp_json +from scripts.us_census.api_utils.download_utils import download_url_list_iterations, async_save_resp_json from tools.download_utils.requests_wrappers import request_url_json -from .status_file_utils import get_pending_or_fail_url_list, sync_status_list +from scripts.us_census.api_utils.status_file_utils import get_pending_or_fail_url_list, sync_status_list FLAGS = flags.FLAGS diff --git a/scripts/us_census/api_utils/census_api_data_downloader.py b/scripts/us_census/api_utils/census_api_data_downloader.py index 7a4f812f0d..0ceb1ee086 100644 --- a/scripts/us_census/api_utils/census_api_data_downloader.py +++ b/scripts/us_census/api_utils/census_api_data_downloader.py @@ -35,9 +35,9 @@ module_dir_ = os.path.dirname(os.path.realpath(__file__)) path.insert(1, os.path.join(module_dir_, '../../../')) -from .download_utils import download_url_list_iterations +from scripts.us_census.api_utils.download_utils import download_url_list_iterations from tools.download_utils.requests_wrappers import request_url_json -from .status_file_utils import sync_status_list +from scripts.us_census.api_utils.status_file_utils import sync_status_list FLAGS = flags.FLAGS @@ -300,6 +300,7 @@ def consolidate_files(dataset: str, # substitute annotations if table_id in column_name and column_name[-1] != 'A': if replace_annotations: + df2[column_name] = df2[column_name].astype(object) df2.loc[df2[column_name + 'A'].notna(), column_name] = df2[column_name + 'A'] if drop_annotations: diff --git a/scripts/us_census/api_utils/download_utils.py b/scripts/us_census/api_utils/download_utils.py index 4cd58930f1..b09e3b17cc 100644 --- a/scripts/us_census/api_utils/download_utils.py +++ b/scripts/us_census/api_utils/download_utils.py @@ -24,7 +24,7 @@ from typing import Any, Callable, Union from aiolimiter import AsyncLimiter -from .status_file_utils import get_pending_or_fail_url_list, url_to_download +from scripts.us_census.api_utils.status_file_utils import get_pending_or_fail_url_list, url_to_download async def async_save_resp_json(response: Any, filename: str): diff --git a/scripts/us_census/api_utils/download_utils_test.py b/scripts/us_census/api_utils/download_utils_test.py index ed4e0651a4..68ac396647 100644 --- a/scripts/us_census/api_utils/download_utils_test.py +++ b/scripts/us_census/api_utils/download_utils_test.py @@ -12,9 +12,11 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import unittest -from .download_utils import * + +from scripts.us_census.api_utils.download_utils import async_save_resp_json, download_url_list class TestCommonUtil(unittest.TestCase): @@ -50,4 +52,4 @@ def test_download_url_list(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/scripts/us_census/api_utils/status_file_utils_test.py b/scripts/us_census/api_utils/status_file_utils_test.py index c5b6cc2770..72a5e34df6 100644 --- a/scripts/us_census/api_utils/status_file_utils_test.py +++ b/scripts/us_census/api_utils/status_file_utils_test.py @@ -12,10 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os import unittest -from .status_file_utils import * +from scripts.us_census.api_utils.status_file_utils import ( + get_failed_http_url_list, + get_failed_url_list, + get_pending_or_fail_url_list, + get_pending_url_list, + url_to_download, +) class TestCommonUtil(unittest.TestCase): @@ -108,4 +115,4 @@ def test_get_pending_or_fail_url_list(self): if __name__ == '__main__': - unittest.main() \ No newline at end of file + unittest.main() diff --git a/scripts/us_census/api_utils/url_list_compiler.py b/scripts/us_census/api_utils/url_list_compiler.py index 8e59ffd2f0..8766c24f89 100644 --- a/scripts/us_census/api_utils/url_list_compiler.py +++ b/scripts/us_census/api_utils/url_list_compiler.py @@ -26,7 +26,7 @@ from typing import Any, Union from census_api_helpers import * -from .status_file_utils import sync_status_list +from scripts.us_census.api_utils.status_file_utils import sync_status_list FLAGS = flags.FLAGS diff --git a/util/dc_api_wrapper_test.py b/util/dc_api_wrapper_test.py index 4b0601158e..4afcb2c9ab 100644 --- a/util/dc_api_wrapper_test.py +++ b/util/dc_api_wrapper_test.py @@ -244,8 +244,19 @@ def test_dc_api_resolve_placeid(self): self.assertEqual(response['ChIJT3IGqvxznW4Rqgw7pv9zYz8'], 'wikidataId/Q9727') - def test_dc_api_resolve_latlng(self): + @mock.patch('dc_api_wrapper.request_url') + def test_dc_api_resolve_latlng(self, mock_request_url): """Test API wrapper for latlng resolution.""" + mock_request_url.return_value = { + "entities": [{ + "node": + "37.42#-122.08", + "candidates": [{ + "dcid": "geoId/0649670", + "dominantType": "City" + }] + }] + } latlngs = [{'latitude': 37.42, 'longitude': -122.08}] response = dc_api.dc_api_resolve_latlng(latlngs) diff --git a/util/latlng_recon_service_test.py b/util/latlng_recon_service_test.py index a7c858d9a7..7f2a24bf11 100644 --- a/util/latlng_recon_service_test.py +++ b/util/latlng_recon_service_test.py @@ -34,7 +34,36 @@ def assert_list_contains(self, superset_list, subset_list): f"The following items were expected but not found: {sorted(list(missing_items))}" ) - def test_basic(self): + @mock.patch('util.latlng_recon_service.dc_api_resolve_latlng') + def test_basic(self, mock_dc_api_resolve_latlng): + mock_dc_api_resolve_latlng.return_value = { + 'placeCoordinates': [{ + 'latitude': + 37.391, + 'longitude': + -122.081, + 'placeDcids': [ + 'zip/94041', 'ipcc_50/37.25_-122.25_USA', + 'geoId/sch0626280', 'geoId/0649670', 'geoId/0618', + 'geoId/0608592830', 'geoId/060855096001', + 'geoId/06085509600', 'geoId/06085', 'geoId/06', + 'country/USA' + ] + }, { + 'latitude': + 12.998, + 'longitude': + 80.272, + 'placeDcids': [ + 'wikidataId/Q15116', 'wikidataId/Q1445', + 'ipcc_50/12.75_80.25_IND', 'country/IND' + ] + }, { + 'latitude': 37.700, + 'longitude': -123.015, + 'placeDcids': [] + }] + } idmap_in = { 'cascal_mtv': (37.391, -122.081), 'besant_beach_chennai': (12.998, 80.272), @@ -53,7 +82,27 @@ def test_basic(self): ]) self.assertEqual(idmap_out['farallon_islands'], []) - def test_filter(self): + @mock.patch('util.latlng_recon_service.dc_api_resolve_latlng') + def test_filter(self, mock_dc_api_resolve_latlng): + mock_dc_api_resolve_latlng.return_value = { + 'placeCoordinates': [{ + 'latitude': + 37.391, + 'longitude': + -122.081, + 'placeDcids': [ + 'zip/94041', 'ipcc_50/37.25_-122.25_USA', + 'geoId/sch0626280', 'geoId/0649670', 'geoId/0618', + 'geoId/0608592830', 'geoId/060855096001', + 'geoId/06085509600', 'geoId/06085', 'geoId/06', + 'country/USA' + ] + }, { + 'latitude': 37.700, + 'longitude': -123.015, + 'placeDcids': [] + }] + } idmap_in = { 'cascal_mtv': (37.391, -122.081), 'farallon_islands': (37.700, -123.015)