S1201 schema fix and data refresh

import-automation/executor/requirements.txt

@@ -1,6 +1,7 @@
 # Requirements for Python scripts in this repo that have automation enabled!
 
 absl-py
+aiolimiter
 arcgis2geojson
 beautifulsoup4
 chardet

scripts/us_census/acs5yr/subject_tables/s1201/generate_col_map.py

@@ -25,11 +25,17 @@
 from collections import OrderedDict
 import pandas as pd
 
-# Allows the following module imports to work when running as a script
+# Allows the sibling/parent module imports to work when running as a script or module
 _SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(_SCRIPT_PATH,
-                             '../common'))  # for statvar_dcid_generator
-from generate_col_map import generate_stat_var_map
+_curr = _SCRIPT_PATH
+while _curr and _curr != os.path.dirname(_curr):
+    if os.path.exists(os.path.join(_curr, '.git')) or os.path.exists(
+            os.path.join(_curr, 'WORKSPACE')):
+        sys.path.append(_curr)
+        break
+    _curr = os.path.dirname(_curr)
+
+from scripts.us_census.acs5yr.subject_tables.common.generate_col_map import generate_stat_var_map
 
 
 def process_zip_file(zip_file_path,

scripts/us_census/acs5yr/subject_tables/s1201/process.py

@@ -12,23 +12,29 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 """Generic proces module to generate the csv/tmcf and csv"""
-# TODO: Add unit tests
+# Allows the sibling/parent module imports to work when running as a script or module
 import os
 import sys
+
+_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
+_curr = _SCRIPT_PATH
+while _curr and _curr != os.path.dirname(_curr):
+    if os.path.exists(os.path.join(_curr, '.git')) or os.path.exists(
+            os.path.join(_curr, 'WORKSPACE')):
+        sys.path.append(_curr)
+        break
+    _curr = os.path.dirname(_curr)
+
+# TODO: Add unit tests
 import json
 from zipfile import ZipFile
 import pandas as pd
 
 from absl import app, flags
 # TODO: logs from the column map step is empty when invoked from here, needs to be checked
 
-from .generate_col_map import generate_stat_var_map, process_zip_file
-
-# Allows the following module imports to work when running as a script
-_SCRIPT_PATH = os.path.dirname(os.path.abspath(__file__))
-sys.path.append(os.path.join(_SCRIPT_PATH,
-                             '../common'))  # for col_map_generator, data_loader
-from data_loader import process_subject_tables
+from scripts.us_census.acs5yr.subject_tables.s1201.generate_col_map import process_zip_file
+from scripts.us_census.acs5yr.subject_tables.common.data_loader import process_subject_tables
 
 FLAGS = flags.FLAGS
 flags.DEFINE_string(
@@ -56,9 +62,8 @@ def set_column_map(input_path, spec_path, output_dir):
     generated_col_map = process_zip_file(input_path,
                                          spec_path,
                                          write_output=False)
-    f = open(os.path.join(output_dir, 'column_map.json'), 'w')
-    json.dump(generated_col_map, f, indent=4)
-    f.close()
+    with open(os.path.join(output_dir, 'column_map.json'), 'w') as f:
+        json.dump(generated_col_map, f, indent=4)
 
 
 def main(argv):
@@ -71,6 +76,8 @@ def main(argv):
     has_percent = FLAGS.has_percent
     debug = FLAGS.debug
 
+    os.makedirs(output_dir, exist_ok=True)
+
     # TODO: remove the constraint of inputs being only zip file
     # context: the current implementation of the column map generator accepts
     # only zip files as input and we will need to add new methods to handle inputs

scripts/us_census/api_utils/census_api_config_fetcher.py

@@ -29,9 +29,9 @@
 CONFIG_PATH_ = os.path.join(module_dir_, 'config_files')
 path.insert(1, os.path.join(module_dir_, '../../../'))
 
-from .download_utils import download_url_list_iterations, async_save_resp_json
+from scripts.us_census.api_utils.download_utils import download_url_list_iterations, async_save_resp_json
 from tools.download_utils.requests_wrappers import request_url_json
-from .status_file_utils import get_pending_or_fail_url_list, sync_status_list
+from scripts.us_census.api_utils.status_file_utils import get_pending_or_fail_url_list, sync_status_list
 
 FLAGS = flags.FLAGS
 

scripts/us_census/api_utils/census_api_data_downloader.py

@@ -35,9 +35,9 @@
 module_dir_ = os.path.dirname(os.path.realpath(__file__))
 path.insert(1, os.path.join(module_dir_, '../../../'))
 
-from .download_utils import download_url_list_iterations
+from scripts.us_census.api_utils.download_utils import download_url_list_iterations
 from tools.download_utils.requests_wrappers import request_url_json
-from .status_file_utils import sync_status_list
+from scripts.us_census.api_utils.status_file_utils import sync_status_list
 
 FLAGS = flags.FLAGS
 
@@ -300,6 +300,7 @@ def consolidate_files(dataset: str,
                 # substitute annotations
                 if table_id in column_name and column_name[-1] != 'A':
                     if replace_annotations:
+                        df2[column_name] = df2[column_name].astype(object)
                         df2.loc[df2[column_name + 'A'].notna(),
                                 column_name] = df2[column_name + 'A']
                     if drop_annotations:

scripts/us_census/api_utils/download_utils.py

@@ -24,7 +24,7 @@
 from typing import Any, Callable, Union
 from aiolimiter import AsyncLimiter
 
-from .status_file_utils import get_pending_or_fail_url_list, url_to_download
+from scripts.us_census.api_utils.status_file_utils import get_pending_or_fail_url_list, url_to_download
 
 
 async def async_save_resp_json(response: Any, filename: str):

scripts/us_census/api_utils/download_utils_test.py

@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import unittest
-from .download_utils import *
+
+from scripts.us_census.api_utils.download_utils import async_save_resp_json, download_url_list
 
 
 class TestCommonUtil(unittest.TestCase):
@@ -50,4 +52,4 @@ def test_download_url_list(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()

scripts/us_census/api_utils/status_file_utils_test.py

@@ -12,10 +12,17 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import unittest
 
-from .status_file_utils import *
+from scripts.us_census.api_utils.status_file_utils import (
+    get_failed_http_url_list,
+    get_failed_url_list,
+    get_pending_or_fail_url_list,
+    get_pending_url_list,
+    url_to_download,
+)
 
 
 class TestCommonUtil(unittest.TestCase):
@@ -108,4 +115,4 @@ def test_get_pending_or_fail_url_list(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
+    unittest.main()

scripts/us_census/api_utils/url_list_compiler.py

@@ -26,7 +26,7 @@
 from typing import Any, Union
 
 from census_api_helpers import *
-from .status_file_utils import sync_status_list
+from scripts.us_census.api_utils.status_file_utils import sync_status_list
 
 FLAGS = flags.FLAGS