feat: Input Schema

.github/workflows/docs.yml

@@ -21,6 +21,7 @@ on:
       - '**.png'
       - '**.svg'
       - '**.jpeg'
+      - 'config/input_schema.json'
   pull_request:
     branches:
       - main
@@ -38,6 +39,7 @@ on:
       - '**.png'
       - '**.svg'
       - '**.jpeg'
+      - 'config/input_schema.json'
 
 jobs:
   build:
@@ -59,6 +61,9 @@ jobs:
       - name: Install the project with docs dependencies
         run: uv sync --group docs
 
+      - name: Generate schema docs
+        run: uv run python docs/generate_schema_docs.py
+
       - name: Build docs
         run: uv run mkdocs build --strict
 

.gitignore

@@ -10,11 +10,12 @@ __pycache__/
 build/
 dist/
 site/
+docs/user_guide/input_schema.md
 .coverage
 htmlcov/
 coverage.xml
 coverage.json
-!input/rodent_dataset.xlsx
+!input/rodent_dataset.csv
 input/*
 phu_templates/*
 !phu_templates/README.md

CHANGELOG.md

@@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
 <!-- insertion marker -->
+## [v1.1.0](https://github.com/WDGPH/ImmuKnow/releases/tag/v1.1.0) - 2026-08-27
+
+<small>[Compare with v1.0.0](https://github.com/WDGPH/ImmuKnow/compare/v1.0.0...v1.1.0)</small>
+
+### Changed
+
+- Switch input schema from fuzzy matching / data normalization to enforced Frictionless schema validation. Add schema page to mkdocs.
+- Minimize required columns by deriving where possible.
+- Adopt `lower_snake_case` column naming.
+- Check client information completeness during preprocessing: records missing `first_name`, `last_name`, `date_of_birth`, `client_id`, `school_name`, `overdue_disease`, or `imms_given` are logged and written to `output/incomplete_clients.csv`; like the address completeness check, by default incomplete records are dropped before further processing.
+- Address and client info completeness checks able to be toggled to drop or retain incomplete records (e.g. students with incomplete address may still receive their notice through email).
+
 ## [v1.0.0](https://github.com/WDGPH/ImmuKnow/releases/tag/v1.0.0) - 2026-08-13
 
 <small>[Compare with v0.3.0](https://github.com/WDGPH/ImmuKnow/compare/v0.3.0...v1.0.0)</small>

README.md

@@ -249,6 +249,7 @@ uv run pytest -m "not e2e"
 - Use data extracts from [Panorama PEAR](https://accessonehealth.ca/)
 - Place input files in the `input/` subfolder (not tracked by Git)
 - Files must be `.xlsx` format with a **single worksheet** per file
+  - Column names must match the required schema exactly - see [Getting Started](docs/user_guide/getting_started.md#preparing-input-data) for the full column list
 
 ## Preprocessing
 
@@ -281,13 +282,21 @@ The preprocessed artifact contains:
   "warnings": [],
   "clients": [
     {
-      "sequence": 1,
+      "sequence": "00001",
       "client_id": "1009876545",
-      "person": {"first_name": "...", "last_name": "...", "date_of_birth": "..."},
-      "school": {"name": "...", "board": "..."},
-      "contact": {"street_address": "...", "city": "...", "postal_code": "...", "province": "..."},
-      "vaccines": {"due": "...", "received": [...]},
-      "metadata": {"recipient": "...", "over_16": false}
+      "language": "en",
+      "person": {
+        "first_name": "...", "last_name": "...",
+        "date_of_birth": "...", "date_of_birth_display": "...", "date_of_birth_iso": "...",
+        "age": "...", "over_16": true
+      },
+      "school": {"name": "...", "id": "..."},
+      "board": {"name": "...", "id": "..."},
+      "contact": {"street": "...", "city": "...", "province": "...", "postal_code": "..."},
+      "vaccines_due": "...",
+      "vaccines_due_list": ["..."],
+      "received": [...],
+      "metadata": {"version_id": null}
     },
     ...
   ]

config/input_schema.json

@@ -0,0 +1,137 @@
+{
+  "$schema": "https://datapackage.org/profiles/2.0/tableschema.json",
+  "title": "input_schema",
+  "description": "Schema for input file for automated notice generation.",
+  "fieldsMatch": ["subset"],
+  "fields": [
+    {
+      "name": "school_name",
+      "type": "string",
+      "description": "School name with PHIX ID (if available) in format 'SCHOOL NAME - PHIX ID'",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "client_id",
+      "type": "string",
+      "description": "10-digit Panorama client identifier",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true,
+        "pattern": "^\\d{10}$"
+      }
+    },
+    {
+      "name": "first_name",
+      "description": "Client first name",
+      "type": "string",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "last_name",
+      "description": "Client last name",
+      "type": "string",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "date_of_birth",
+      "description": "Client date of birth",
+      "type": "date",
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "street_address_line_1",
+      "description": "Client street address (line 1)",
+      "type": "string",
+      "stripWhitespace": true
+    },
+    {
+      "name": "street_address_line_2",
+      "description": "Client street address (line 2)",
+      "type": "string",
+      "stripWhitespace": true
+    },
+    {
+      "name": "city",
+      "description": "City of client address",
+      "type": "string",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "province",
+      "description": "Province / territory of client address",
+      "type": "string",
+      "stripWhitespace": true
+    },
+    {
+      "name": "postal_code",
+      "description": "Postal code of client address",
+      "type": "string",
+      "stripWhitespace": true
+    },
+    {
+      "name": "overdue_disease",
+      "description": "Comma-separated list of overdue diseases for client",
+      "type": "string",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "overdue_agent",
+      "description": "Comma-separated list of overdue agents for client",
+      "type": "string",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "imms_given",
+      "description": "List of immunizations given to client, separated by ';'. Each list entry is in the format 'Mon Day, YYYY - <received vaccine>'. Optionally, each entry may also include validity status of the dose appended as '- <dose validity>'",
+      "type": "string",
+      "stripWhitespace": true,
+      "constraints": {
+        "required": true
+      }
+    },
+    {
+      "name": "board_name",
+      "description": "School board name",
+      "type": "string",
+      "stripWhitespace": true
+    },
+    {
+      "name": "board_id",
+      "description": "School board identifier",
+      "type": "string",
+      "stripWhitespace": true
+    },
+    {
+      "name": "school_id",
+      "description": "School identifier",
+      "type": "string"
+    },
+    {
+      "name": "version_id",
+      "description": "Version identifier",
+      "type": "string",
+      "stripWhitespace": true
+    }
+  ],
+  "missingValues": [""]
+}

docs/generate_schema_docs.py

@@ -0,0 +1,35 @@
+from pathlib import Path
+import json
+
+SCHEMA_FILE = Path("config/input_schema.json")
+OUTPUT_FILE = Path("docs/user_guide/input_schema.md")
+
+schema = json.loads(SCHEMA_FILE.read_text())
+
+lines = []
+
+lines.append(f"# {schema.get('title', 'Schema')}\n\n")
+
+if schema.get("description"):
+    lines.append(f"{schema['description']}\n\n")
+
+lines.append("## Schema Information\n\n")
+lines.append("| Property | Value |\n")
+lines.append("|----------|-------|\n")
+lines.append(f"| Fields | {len(schema['fields'])} |\n")
+lines.append(f"| Field Matching | {', '.join(schema.get('fieldsMatch', []))} |\n")
+
+missing_values = ", ".join(f"`{v}`" for v in schema.get("missingValues", []))
+lines.append(f"| Missing Values | {missing_values} |\n\n")
+
+lines.append("## Field Summary\n\n")
+lines.append("| Field | Type | Description |\n")
+lines.append("|-------|------|-------------|\n")
+
+for field in schema["fields"]:
+    description = field.get("description", "")
+    lines.append(f"| {field['name']} | {field['type']} | {description} |\n")
+
+OUTPUT_FILE.write_text("".join(lines), encoding="utf-8")
+
+print(f"Generated {OUTPUT_FILE}")

docs/reference/pipeline_steps.md

@@ -57,6 +57,8 @@ Reads the raw Excel input, validates the schema, normalizes all client and vacci
 
 - `output/artifacts/preprocessed_clients_<run_id>.json` — canonical client artifact
 - `output/logs/preprocess_<run_id>.log` — processing log
+- `output/incomplete_addresses.csv` — records dropped due to missing address fields (written when any are found)
+- `output/incomplete_clients.csv` — records with missing required client fields, retained in processing (written when any are found)
 - `phix_exact.csv`, `phix_inexact.csv`, `phix_no_match.csv` — school match audit CSVs (when PHIX validation enabled)
 
 **Processing:**
@@ -67,10 +69,12 @@ Reads the raw Excel input, validates the schema, normalizes all client and vacci
 4. Expands vaccine codes to disease names using `vaccine_reference.json`
 5. Filters diseases against `chart_diseases_header`; collapses unlisted diseases to "Other"
 6. Computes client ages relative to `date_notice_delivery` (determines parent vs. student addressing)
-7. Sorts clients deterministically: school → last name → first name → client ID
-8. Assigns stable sequence numbers (`00001`, `00002`, …)
-9. Synthesizes missing school/board identifiers where needed
-10. Writes the canonical JSON artifact
+7. Checks address completeness: records missing `address`, `city`, `province`, or `postal_code` are logged, written to `output/incomplete_addresses.csv`, and **dropped** by default from further processing
+8. Checks client information completeness: records missing `first_name`, `last_name`, `date_of_birth`, `client_id`, `school_name`, `overdue_disease`, or `imms_given` are logged and written to `output/incomplete_clients.csv`, and **dropped** by default from further processing
+9. Sorts clients deterministically: school → last name → first name → client ID
+10. Assigns stable sequence numbers (`00001`, `00002`, …)
+11. Synthesizes missing school/board identifiers where needed
+12. Writes the canonical JSON artifact
 
 ---
 

docs/user_guide/getting_started.md

@@ -27,6 +27,35 @@ uv run pre-commit install
 
 Input files must be `.xlsx` format with a single worksheet, extracted from [Panorama PEAR](https://accessonehealth.ca/).
 
+The pipeline enforces a strict column schema — column names must match exactly (no fuzzy matching). The following columns are **required**:
+
+| Column name | Notes |
+|---|---|
+| `school_name` | |
+| `client_id` | 10-digit numeric string |
+| `first_name` | |
+| `last_name` | |
+| `date_of_birth` | ISO 8601 date (`YYYY-MM-DD`) |
+| `street_address_line_1` | |
+| `street_address_line_2` | May be blank |
+| `city` | |
+| `province` | |
+| `postal_code` | |
+| `overdue_disease` | May be blank |
+| `overdue_agent` | May be blank |
+| `imms_given` | May be blank |
+
+The following columns are **optional** and will be used when present:
+
+| Column name |
+|---|
+| `board_name` |
+| `board_id` |
+| `school_id` |
+| `version_id` |
+
+The full schema is defined in `config/input_schema.json`. If the file is missing any required column, the pipeline will stop immediately with a clear error message listing the missing columns.
+
 Place input files in the `input/` subdirectory (not tracked by Git):
 
 ```

input/rodent_dataset.csv

@@ -0,0 +1,8 @@
+school_name,client_id,first_name,last_name,date_of_birth,street_address_line_1,street_address_line_2,city,province,postal_code,overdue_disease,overdue_agent,imms_given,Disease(s)/Agent(s),Imms History by Agent
+WHISKER ELEMENTARY-1009876543,1009876543,Squeak,McCheese,2013-06-15,14 Burrow Lane,,Cheddarville,Ontario,M1C3E5,"Varicella, HPV, Hepatitis B","Var, HPV-9, Men-C-ACYW-135,","Aug 20, 2013 - DTaP-IPV-Hib; Aug 20, 2013 - Pneu-C-13; Aug 20, 2013 - rota-unspecified; Nov 18, 2013 - DTaP-IPV-Hib; Nov 18, 2013 - Pneu-C-13; Jan 25, 2014 - DTaP-IPV-Hib; May 12, 2014 - MMR; May 12, 2014 - Men-C-C; Oct 3, 2014 - Var; Apr 14, 2024 - Tdap-IPV;",Varicella (Var),"[2013 AUG 20: DTaP-IPV-Hib, Pneu-C-13, rota-unspecified] [2013 NOV 18: DTaP-IPV-Hib, Pneu-C-13] [2014 JAN 25: DTaP-IPV-Hib] [2014 MAY 12: MMR, Men-C-C] [2014 OCT 03: Var] [2024 APR 14: Tdap-IPV, MMR-Var]"
+CHEESE WHEEL ACADEMY-1009876544,1009876544,Nibble,Sharpcheddar,2014-04-22,22 Gouda St,,Fromage City,Ontario,C3H3Z9,"Measles,","MMR,","Jul 10, 2014 - DTaP-IPV-Hib; Jul 10, 2014 - Pneu-C-13; Sep 15, 2014 - DTaP-IPV-Hib; Nov 20, 2014 - rota-unspecified; Mar 2, 2015 - MMR; Mar 2, 2015 - Men-C-C; Aug 7, 2015 - Var; Oct 1, 2015 - DTaP-IPV-Hib; May 19, 2024 - Tdap-IPV;",Measles (MMR),"[2014 JUL 10: DTaP-IPV-Hib, Pneu-C-13] [2014 SEP 15: DTaP-IPV-Hib] [2014 NOV 20: rota-unspecified] [2015 MAR 02: MMR, Men-C-C] [2015 AUG 07: Var] [2015 OCT 01: DTaP-IPV-Hib] [2024 MAY 19: Tdap-IPV]"
+BURROW PUBLIC SCHOOL-1009876545,1009876545,Scurry,Nutcracker,2012-11-30,7 Tunnel Road,Unit 2,Gnawtown,Ontario,G9N8R2,"Hepatitis B,","HB,","Jan 5, 2013 - DTaP-IPV-Hib; Jan 5, 2013 - rota-unspecified; Mar 7, 2013 - Pneu-C-13; May 9, 2013 - DTaP-IPV-Hib; Jun 11, 2013 - MMR; Oct 23, 2013 - Men-C-C; Feb 2, 2014 - Var; May 6, 2014 - Pneu-C-13; Sep 12, 2014 - DTaP-IPV-Hib; May 1, 2024 - Tdap-IPV;",Hepatitis B (HB),"[2013 JAN 05: DTaP-IPV-Hib, rota-unspecified] [2013 MAR 07: Pneu-C-13] [2013 MAY 09: DTaP-IPV-Hib] [2013 JUN 11: MMR] [2013 OCT 23: Men-C-C] [2014 FEB 02: Var] [2014 MAY 06: Pneu-C-13] [2014 SEP 12: DTaP-IPV-Hib] [2024 MAY 01: Tdap-IPV]"
+TUNNEL ACADEMY-1009876546,1009876546,Whiskers,Greyfur,2013-09-10,88 Haystack Drive,,Burrowville,Ontario,H8Y6T5,"Mumps,","MMR,","Oct 15, 2013 - DTaP-IPV-Hib; Dec 12, 2013 - rota-unspecified; Jan 17, 2014 - Pneu-C-13; Apr 8, 2014 - DTaP-IPV-Hib; Jun 19, 2014 - MMR; Oct 22, 2014 - Men-C-C; Feb 4, 2015 - Var; Sep 9, 2015 - DTaP-IPV-Hib; Apr 10, 2024 - Tdap-IPV;",Mumps (MMR),[2013 OCT 15: DTaP-IPV-Hib] [2013 DEC 12: rota-unspecified] [2014 JAN 17: Pneu-C-13] [2014 APR 08: DTaP-IPV-Hib] [2014 JUN 19: MMR] [2014 OCT 22: Men-C-C] [2015 FEB 04: Var] [2015 SEP 09: DTaP-IPV-Hib] [2024 APR 10: Tdap-IPV]
+NUTCRACKER ACADEMY-1009876547,1009876547,Chisel,Teetherson,2014-02-28,3 Acorn Ave,Suite 1,Hazelton,Ontario,N4U2L1,"HPV,","HPV-9,","Mar 12, 2014 - DTaP-IPV-Hib; Mar 12, 2014 - rota-unspecified; May 14, 2014 - Pneu-C-13; Jul 19, 2014 - DTaP-IPV-Hib; Sep 21, 2014 - MMR; Nov 25, 2014 - Men-C-C; Apr 17, 2015 - Var; Sep 13, 2015 - DTaP-IPV-Hib; May 5, 2024 - Tdap-IPV;",HPV (HPV-9),"[2014 MAR 12: DTaP-IPV-Hib, rota-unspecified] [2014 MAY 14: Pneu-C-13] [2014 JUL 19: DTaP-IPV-Hib] [2014 SEP 21: MMR] [2014 NOV 25: Men-C-C] [2015 APR 17: Var] [2015 SEP 13: DTaP-IPV-Hib] [2024 MAY 05: Tdap-IPV]"
+NUTCRACKER ACADEMY-1009876547,1009876548,Ratty,Teetherson,2009-02-28,,,Hazelton,Ontario,N4U2L1,"HPV,","HPV-9,","Mar 12, 2014 - DTaP-IPV-Hib; Mar 12, 2014 - rota-unspecified; May 14, 2014 - Pneu-C-13; Jul 19, 2014 - DTaP-IPV-Hib; Sep 21, 2014 - MMR; Nov 25, 2014 - Men-C-C; Apr 17, 2015 - Var; Sep 13, 2015 - DTaP-IPV-Hib; May 5, 2024 - Tdap-IPV;",HPV (HPV-9),"[2014 MAR 12: DTaP-IPV-Hib, rota-unspecified] [2014 MAY 14: Pneu-C-13] [2014 JUL 19: DTaP-IPV-Hib] [2014 SEP 21: MMR] [2014 NOV 25: Men-C-C] [2015 APR 17: Var] [2015 SEP 13: DTaP-IPV-Hib] [2024 MAY 05: Tdap-IPV]"
+TUNNEL ACADEMY-1009876550,1009876550,Cheddarina,Swiftpaws,2014-09-14,44 Hayloft Road,,Burrowville,Ontario,H8Y6T6,MMR,MMR,"Jan 10, 2015 - DTaP-IPV-Hib; Jan 29, 2015 - Pneu-C-13; Feb 18, 2015 - rota-unspecified; Mar 07, 2015 - DTaP-IPV-Hib; Mar 28, 2015 - MMR; Apr 15, 2015 - Men-C-C; May 02, 2015 - Var; May 27, 2015 - DTaP-IPV-Hib; Jun 16, 2015 - Pneu-C-13; Jul 09, 2015 - Influenza (IIV4); Aug 01, 2015 - Influenza (IIV4); Aug 29, 2015 - MMR; Sep 22, 2015 - Var; Oct 11, 2015 - DTaP-IPV-Hib; Nov 05, 2015 - Pneu-C-13; Dec 03, 2015 - Men-C-C; Jan 14, 2016 - MMR; Feb 06, 2016 - Influenza (IIV4); Mar 12, 2016 - Hep A; Apr 04, 2016 - Hep A booster; May 18, 2016 - Yellow Fever; Jun 07, 2016 - Rabies (pre-exposure); Jun 30, 2016 - Rabies (pre-exposure) dose 2; Jul 23, 2016 - Rabies (pre-exposure) dose 3; Aug 15, 2016 - Var; Sep 08, 2016 - DTaP-IPV-Hib; Oct 01, 2016 - Pneu-C-13; Oct 27, 2016 - Influenza (IIV4); Nov 19, 2016 - MMR; Dec 14, 2016 - Men-C-C; Jan 09, 2017 - Var; Feb 03, 2017 - DTaP-IPV-Hib; Mar 01, 2017 - Pneu-C-13; Mar 29, 2017 - MMR; Apr 18, 2017 - Influenza (IIV4); May 10, 2017 - COVID-19 (Pfizer Pediatric); Jun 02, 2017 - COVID-19 (Pfizer Pediatric) dose 2; Jun 28, 2017 - COVID-19 Booster; Jul 20, 2017 - Var; Aug 12, 2017 - Men-C-C; Sep 03, 2017 - Influenza (IIV4); Oct 25, 2017 - DTaP-IPV-Hib; Nov 16, 2017 - Pneu-C-13; Dec 08, 2017 - MMR; May 02, 2023 - Tdap; Jan 18, 2024 - Men-C-ACYW-135; May 01, 2024 - Tdap-IPV",Measles (MMR),[2015 JAN 10: DTaP-IPV-Hib] [2015 JAN 29: Pneu-C-13] [2015 FEB 18: rota-unspecified] [2015 MAR 07: DTaP-IPV-Hib] [2015 MAR 28: MMR] [2015 APR 15: Men-C-C] [2015 MAY 02: Var] [2015 MAY 27: DTaP-IPV-Hib] [2015 JUN 16: Pneu-C-13] [2015 JUL 09: Influenza (IIV4)] [2015 AUG 01: Influenza (IIV4)] [2015 AUG 29: MMR] [2015 SEP 22: Var] [2015 OCT 11: DTaP-IPV-Hib] [2015 NOV 05: Pneu-C-13] [2015 DEC 03: Men-C-C] [2016 JAN 14: MMR] [2016 FEB 06: Influenza (IIV4)] [2016 MAR 12: Hep A] [2016 APR 04: Hep A booster] [2016 MAY 18: Yellow Fever] [2016 JUN 07: Rabies (pre-exposure)] [2016 JUN 30: Rabies (pre-exposure) dose 2] [2016 JUL 23: Rabies (pre-exposure) dose 3] [2016 AUG 15: Var] [2016 SEP 08: DTaP-IPV-Hib] [2016 OCT 01: Pneu-C-13] [2016 OCT 27: Influenza (IIV4)] [2016 NOV 19: MMR] [2016 DEC 14: Men-C-C] [2017 JAN 09: Var] [2017 FEB 03: DTaP-IPV-Hib] [2017 MAR 01: Pneu-C-13] [2017 MAR 29: MMR] [2017 APR 18: Influenza (IIV4)] [2017 MAY 10: COVID-19 (Pfizer Pediatric)] [2017 JUN 02: COVID-19 (Pfizer Pediatric) dose 2] [2017 JUN 28: COVID-19 Booster] [2017 JUL 20: Var] [2017 AUG 12: Men-C-C] [2017 SEP 03: Influenza (IIV4)] [2017 OCT 25: DTaP-IPV-Hib] [2017 NOV 16: Pneu-C-13] [2017 DEC 08: MMR] [2023 MAY 02: Tdap] [2024 JAN 18: Men-C-ACYW-135] [2024 MAY 01: Tdap-IPV]

mkdocs.yml

@@ -97,6 +97,7 @@ nav:
     - Getting Started: user_guide/getting_started.md
     - Configuration Reference: user_guide/configuration.md
     - PHU Templates: user_guide/phu_templates.md
+    - Input Schema: user_guide/input_schema.md
     - PDF Validation: user_guide/pdf_validation.md
     - Email Package:
       - Overview: user_guide/email_package/index.md

pipeline/data_models.py

@@ -40,20 +40,19 @@ class ClientRecord:
     person : Dict[str, Any]
 
         - Person details:
-            - full_name: Combined first and last name
-            - first_name: Given name (optional)
-            - last_name: Family name (optional)
+            - first_name: Given name
+            - last_name: Family name
             - date_of_birth: Display format (e.g., "Jan 8, 2025")
             - date_of_birth_iso: ISO format (YYYY-MM-DD)
             - date_of_birth_display: Localized display format
-            - age: Calculated age in years
+            - age: Calculated age in years (as string)
             - over_16: Boolean flag for age >= 16
 
     school : Dict[str, Any]
-        School information: name, id, code, type.
+        School information: name, id.
 
     board : Dict[str, Any]
-        School board information: name, id, code.
+        School board information: name, id.
 
     contact : Dict[str, Any]
         Contact address: street, city, province, postal_code.