IPEDS_StudentToFacultyRatio: Adding provisional data mapping, updated download/preprocess scripts, and golden validation

statvar_imports/ipeds/student_to_faculty_ratio/download.py

@@ -24,11 +24,24 @@
 START_YEAR = 2009
 # Set END_YEAR dynamically to the current calendar year
 END_YEAR = date.today().year
-BASE_URL = "https://nces.ed.gov/ipeds/datacenter/data/EF{}D.zip"
+
+BASE_URL_LEGACY = "https://nces.ed.gov/ipeds/datacenter/data/EF{year}D.zip"
+BASE_URL_COMPLETE = "https://nces.ed.gov/ipeds/complete-data-files/EF{year}D.zip"
+BASE_URL_RV = "https://nces.ed.gov/ipeds/data-generator?year={year}&tableName=EF{year}D&HasRV=1&type=csv"
+BASE_URL_PROV = "https://nces.ed.gov/ipeds/data-generator?year={year}&tableName=EF{year}D&HasRV=0&type=csv"
+
+BASE_URL_TEMPLATES = [
+    BASE_URL_LEGACY,
+    BASE_URL_COMPLETE,
+    BASE_URL_RV,
+    BASE_URL_PROV,
+]
+
 DOWNLOAD_DIR = "input_files"
 # Pattern to match files ending in '_rv' followed by a file extension
-# The pattern should match '_rv.txt', '_rv.csv', etc.
 RV_PATTERN = re.compile(r'_rv\.[a-z0-9]+$', re.IGNORECASE)
+# Pattern to match provisional files (e.g. ef2023d.csv, ef2024d.csv)
+PROVISIONAL_PATTERN = re.compile(r'^ef\d{4}d\.[a-z0-9]+$', re.IGNORECASE)
 # ---------------------
 
 # --- Path Adjustment for Utility Import ---
@@ -45,117 +58,134 @@
     raise RuntimeError(f"FATAL: Missing utility script dependency: {e}")
 
 
-def process_and_filter_zip(zip_path: str, output_dir: str, filter_pattern: re.Pattern) -> bool:
+def process_and_filter_zip(zip_path: str, output_dir: str, require_rv: bool) -> bool:
     """
-    Handles the custom unzipping, RV-pattern filtering, and cleanup.
-    If extraction fails unexpectedly, it is treated as a fatal error.
+    Unzips and filters contents of zip_path.
+    If require_rv is True, extracts only files matching RV_PATTERN.
+    If require_rv is False, extracts files matching PROVISIONAL_PATTERN.
+    Returns True if matching file(s) were found and extracted, False otherwise.
     """
     zip_filename = os.path.basename(zip_path)
-    logging.info("  Unzipping and filtering contents (keeping only files matching pattern: %s)...", filter_pattern.pattern)
-
-    extraction_successful = False
 
     try:
-        # 1. Unzip and filter
         with zipfile.ZipFile(zip_path, 'r') as zip_ref:
             all_files = zip_ref.namelist()
-            files_to_extract = []
+            target_pattern = RV_PATTERN if require_rv else PROVISIONAL_PATTERN
 
-            for file_name in all_files:
-                base_name = os.path.basename(file_name)
-                # Check if the file is at the root or within a single folder.
-                if filter_pattern.search(base_name):
-                    files_to_extract.append(file_name)
+            files_to_extract = [
+                f for f in all_files if target_pattern.search(os.path.basename(f))
+            ]
 
             if not files_to_extract:
-                # This is a warning, not a failure, as the process was clean.
-                logging.fatal("  Warning: No files matching the pattern found in %s. Skipping extraction.", zip_filename)
-                extraction_successful = True
+                return False
 
-            # Extract the filtered files
             for file_name in files_to_extract:
                 zip_ref.extract(file_name, output_dir)
-                logging.info("    Extracted: %s", file_name)
+                logging.info("    Extracted (%s): %s", "REVISED" if require_rv else "PROVISIONAL", file_name)
 
-            if files_to_extract:
-                logging.info("  Extraction successful.")
-            extraction_successful = True
+            return True
 
     except zipfile.BadZipFile:
-        # FATAL: Corrupted zip file means data is unavailable, and we must stop processing this file.
         logging.fatal("  FATAL ERROR: %s is a corrupted or empty zip file. Cannot proceed.", zip_filename)
         raise RuntimeError(f"Corrupted or empty zip file encountered: {zip_filename}")
     except Exception as e:
-        # FATAL: Any unexpected extraction error means partial data, which must be avoided.
         logging.fatal("  FATAL ERROR: An unexpected error occurred during unzipping/extraction of %s: %s", zip_filename, e)
-        # Raise an error to stop the script from proceeding with potentially partial unzipped files
         raise RuntimeError(f"Extraction failed for {zip_filename}: {e}")
     finally:
-        # 2. Clean up the downloaded zip file manually, regardless of success or failure
         if os.path.exists(zip_path):
             try:
                 os.remove(zip_path)
-                logging.info("  Removed zip file: %s", zip_path)
-            except OSError as e:
-                # Use info for non-critical file removal errors
-                logging.info("  Warning: Failed to remove zip file %s: %s", zip_path, e)
-
-    return extraction_successful
+            except OSError:
+                pass
 
 
-def main():
+def download_for_year(year: int) -> bool:
     """
-    Downloads IPEDS zip files using the utility, then handles custom unzipping/filtering locally.
-    Download failure is logged as a warning, allowing the script to proceed to the next year.
-    Extraction failure is logged as FATAL, which will stop the script for that year's file.
+    Downloads data for a given year following strict precedence:
+    1. Checks candidate URLs to see if a REVISED dataset (_rv) is available.
+    2. ONLY IF no revised dataset is available across any candidate URL,
+       checks candidate URLs to download a PROVISIONAL dataset.
     """
+    logging.info("\nProcessing year %d...", year)
 
-    # 1. Create the target directory if it doesn't exist
-    if not os.path.exists(DOWNLOAD_DIR):
-        try:
-            os.makedirs(DOWNLOAD_DIR)
-            logging.info("Created directory: %s", DOWNLOAD_DIR)
-        except OSError as e:
-            # Use logging.fatal for critical directory creation errors
-            logging.fatal("FATAL ERROR: Could not create directory %s: %s", DOWNLOAD_DIR, e)
-            raise RuntimeError(f"FATAL: Directory creation failed for {DOWNLOAD_DIR}: {e}")
+    # --- Phase 1: Try to download REVISED dataset across candidate URLs ---
+    for url_template in BASE_URL_TEMPLATES:
+        if "{year}" in url_template:
+            url = url_template.format(year=year)
+        else:
+            url = url_template.format(year)
 
-    # 2. Iterate through the required year range
-    for year in range(START_YEAR, END_YEAR + 1):
-        url = BASE_URL.format(year)
         zip_filename = f"EF{year}D.zip"
         download_path = os.path.join(DOWNLOAD_DIR, zip_filename)
 
-        logging.info("\nProcessing year %d...", year)
-
         try:
-            # 3. Call the utility function to DOWNLOAD ONLY (unzip=False)
             download_success = download_file(
                 url=url,
                 output_folder=DOWNLOAD_DIR,
-                unzip=False, # <-- CRITICAL: Do not let the utility unzip the file
-                tries=3,
-                delay=5,
-                backoff=2
+                unzip=False, # <-- CRITICAL: Do not let utility unzip file
+                tries=1,
+                delay=1,
+                backoff=1
             )
 
-            if download_success:
-                # 4. Handle custom processing (unzip, filter, and cleanup) locally
-                # If process_and_filter_zip encounters a fatal error, it will raise an exception
-                process_and_filter_zip(download_path, DOWNLOAD_DIR, RV_PATTERN)
+            if download_success and os.path.exists(download_path) and zipfile.is_zipfile(download_path):
+                if process_and_filter_zip(download_path, DOWNLOAD_DIR, require_rv=True):
+                    logging.info("  Successfully fetched REVISED dataset for year %d.", year)
+                    return True
+        except Exception as e:
+            logging.info("  Candidate URL %s (REVISED check) failed: %s", url, e)
+
+    logging.info("  No REVISED dataset available for year %d. Checking for PROVISIONAL dataset...", year)
 
-            else:
-                # Not available/download failed: Log as a warning and skip, as requested
-                logging.info("Warning: Download failed for year %d. Skipping processing for this year.", year)
+    # --- Phase 2: ONLY if REVISED is not available, try to download PROVISIONAL dataset ---
+    for url_template in BASE_URL_TEMPLATES:
+        if "{year}" in url_template:
+            url = url_template.format(year=year)
+        else:
+            url = url_template.format(year)
 
+        zip_filename = f"EF{year}D.zip"
+        download_path = os.path.join(DOWNLOAD_DIR, zip_filename)
+
+        try:
+            download_success = download_file(
+                url=url,
+                output_folder=DOWNLOAD_DIR,
+                unzip=False,
+                tries=1,
+                delay=1,
+                backoff=1
+            )
+
+            if download_success and os.path.exists(download_path) and zipfile.is_zipfile(download_path):
+                if process_and_filter_zip(download_path, DOWNLOAD_DIR, require_rv=False):
+                    logging.info("  Successfully fetched PROVISIONAL dataset for year %d.", year)
+                    return True
         except Exception as e:
-            # Catch errors that process_and_filter_zip explicitly raises (BadZipFile, Extraction Error)
-            # or any other unexpected error during the loop. The error is already logged as FATAL.
-            logging.info("Execution failed for year %d, continuing to next year (if possible). Error: %s", year, e)
-            # The script will now proceed to the next iteration unless the error is outside the loop
+            logging.info("  Candidate URL %s (PROVISIONAL check) failed: %s", url, e)
+
+    logging.info("Warning: Neither REVISED nor PROVISIONAL dataset found for year %d across candidate URLs.", year)
+    return False
+
+
+def main():
+    """
+    Iterates through year range downloading datasets (preferring revised, falling back to provisional).
+    """
+
+    # 1. Create target directory if it doesn't exist
+    if not os.path.exists(DOWNLOAD_DIR):
+        try:
+            os.makedirs(DOWNLOAD_DIR)
+            logging.info("Created directory: %s", DOWNLOAD_DIR)
+        except OSError as e:
+            logging.fatal("FATAL ERROR: Could not create directory %s: %s", DOWNLOAD_DIR, e)
+            raise RuntimeError(f"FATAL: Directory creation failed for {DOWNLOAD_DIR}: {e}")
 
-        # Optional: Add a pause between years to respect NCES server requests
-        time.sleep(5)
+    # 2. Iterate through required year range
+    for year in range(START_YEAR, END_YEAR + 1):
+        download_for_year(year)
+        time.sleep(2)
 
 
 if __name__ == "__main__":