feat: Input Schema

README.md

@@ -249,6 +249,7 @@ uv run pytest -m "not e2e"
 - Use data extracts from [Panorama PEAR](https://accessonehealth.ca/)
 - Place input files in the `input/` subfolder (not tracked by Git)
 - Files must be `.xlsx` format with a **single worksheet** per file
+  - Column names must match the required schema exactly - see [Getting Started](docs/user_guide/getting_started.md#preparing-input-data) for the full column list
 
 ## Preprocessing
 

config/input_schema.json

@@ -0,0 +1,73 @@
+{
+  "$schema": "https://specs.frictionlessdata.io/schemas/table-schema.json",
+  "fields": [
+    {
+      "name": "School Type",
+      "type": "string"
+    },
+    {
+      "name": "School Name",
+      "type": "string"
+    },
+    {
+      "name": "Client Id",
+      "type": "string",
+      "constraints": {
+        "pattern": "^\\d{10}$"
+      }
+    },
+    {
+      "name": "First Name",
+      "type": "string"
+    },
+    {
+      "name": "Last Name",
+      "type": "string"
+    },
+    {
+      "name": "Age",
+      "type": "integer"
+    },
+    {
+      "name": "Date of Birth",
+      "type": "date"
+    },
+    {
+      "name": "Street Address Line 1",
+      "type": "string"
+    },
+    {
+      "name": "Street Address Line 2",
+      "type": "string"
+    },
+    {
+      "name": "City",
+      "type": "string"
+    },
+    {
+      "name": "Province/Territory",
+      "type": "string"
+    },
+    {
+      "name": "Postal Code",
+      "type": "string"
+    },
+    {
+      "name": "Overdue Disease",
+      "type": "string"
+    },
+    {
+      "name": "Overdue Agent",
+      "type": "string"
+    },
+    {
+      "name": "Imms Given",
+      "type": "string"
+    },
+    {
+      "name": "Birth Year",
+      "type": "string"
+    }
+  ],
+  "missingValues": [""]
+}

config/input_schema.yaml

@@ -0,0 +1,37 @@
+fields:
+  - name: School Type
+    type: string
+  - name: School Name
+    type: string
+  - name: Client Id
+    type: string
+    constraints:
+      pattern: '^\d{10}$'
+  - name: First Name
+    type: string
+  - name: Last Name
+    type: string
+  - name: Age
+    type: integer
+  - name: Date of Birth
+    type: date
+  - name: Street Address Line 1
+    type: string
+  - name: Street Address Line 2
+    type: string
+  - name: City
+    type: string
+  - name: Province/Territory
+    type: string
+  - name: Postal Code
+    type: string
+  - name: Overdue Disease
+    type: string
+  - name: Overdue Agent
+    type: string
+  - name: Imms Given
+    type: string
+  - name: Birth Year
+    type: string
+missingValues:
+  - ''

docs/user_guide/getting_started.md

@@ -27,6 +27,38 @@ uv run pre-commit install
 
 Input files must be `.xlsx` format with a single worksheet, extracted from [Panorama PEAR](https://accessonehealth.ca/).
 
+The pipeline enforces a strict column schema — column names must match exactly (no fuzzy matching). The following columns are **required**:
+
+| Column name | Notes |
+|---|---|
+| `School Type` | |
+| `School Name` | |
+| `Client Id` | 10-digit numeric string |
+| `First Name` | |
+| `Last Name` | |
+| `Age` | Integer |
+| `Date of Birth` | ISO 8601 date (`YYYY-MM-DD`) |
+| `Street Address Line 1` | |
+| `Street Address Line 2` | May be blank |
+| `City` | |
+| `Province/Territory` | |
+| `Postal Code` | |
+| `Overdue Disease` | May be blank |
+| `Overdue Agent` | May be blank |
+| `Imms Given` | May be blank |
+| `Birth Year` | |
+
+The following columns are **optional** and will be used when present:
+
+| Column name |
+|---|
+| `Board Name` |
+| `Board Id` |
+| `School Id` |
+| `Unique Id` |
+
+The full schema is defined in `config/input_schema.yaml`. If the file is missing any required column, the pipeline will stop immediately with a clear error message listing the missing columns.
+
 Place input files in the `input/` subdirectory (not tracked by Git):
 
 ```

pipeline/generate_notices.py

@@ -293,6 +293,9 @@ def read_artifact(path: Path) -> ArtifactPayload:
     )
 
 
+_TYP_IDENT_RE = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_-]*$")
+
+
 def escape_string(value: str) -> str:
     """Escape special characters in a string for Typst template output.
 
@@ -359,7 +362,9 @@ def to_typ_value(value) -> str:
             inner = ", ".join(items)
         return f"({inner})"
     if isinstance(value, Mapping):
-        items = ", ".join(f"{key}: {to_typ_value(val)}" for key, val in value.items())
+        def _typ_key(k: str) -> str:
+            return k if _TYP_IDENT_RE.match(k) else f'"{escape_string(k)}"'
+        items = ", ".join(f"{_typ_key(key)}: {to_typ_value(val)}" for key, val in value.items())
         return f"({items})"
     raise TypeError(f"Unsupported value type for Typst conversion: {type(value)!r}")
 

pipeline/orchestrator.py

@@ -225,9 +225,8 @@ def run_step_2_preprocess(
     # Load and process input data
     input_path = input_dir / input_file
     df_raw = preprocess.read_input(input_path)
-    mapped_df, column_mapping = preprocess.map_columns(df_raw)
-    df_filtered = preprocess.filter_columns(mapped_df)
-    df = preprocess.normalize_dataframe(df_filtered)
+    preprocess.validate_input(input_path)
+    df = preprocess.normalize_dataframe(preprocess.map_columns(df_raw))
 
     # Check that addresses are complete, return only complete rows
     df = preprocess.check_addresses_complete(df)