Organization Validation for Service Inventory

.github/workflows/pytest.yml

@@ -274,6 +274,8 @@ jobs:
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db upgrade
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db pending-migrations --apply
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db pending-migrations --apply
+          ckan -c /srv/app/src/ckanext-canada/test-core.ini recombinant create-ref-tables
+          ckan -c /srv/app/src/ckanext-canada/test-core.ini pd load-ref-data -v
       - name: Download Test Durations
         if: needs.define-matrix.outputs.restored == 'true'
         uses: actions/download-artifact@v4

.github/workflows/si_release_listener.yml

@@ -0,0 +1,34 @@
+name: Service Inventory Release Listener
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  make_pr:
+    if: startsWith(github.event.issue.title, '[SI-RELEASE]')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Run Ref Data Generation
+        run: |
+          cd bin;
+          python service_generate_reference_data.py;
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: |
+            build(pd): service inventory;
+
+            - Auto build ref data from issue ${{ github.event.issue.number }}.
+          branch: "si-release/${{ github.event.issue.number }}"
+          title: "[SI-RELEASE] Issue #${{ github.event.issue.number }}"
+          body: |
+            Newly generated reference data for Service Inventory.
+
+            This PR was created automatically from issue #${{ github.event.issue.number }}
+
+            cc: @ge-tbs
+          reviewers: JVickery-TBS, RabiaSajjad

bin/filter/filter_service.py

@@ -3,7 +3,6 @@
 import csv
 import os
 import sys
-import yaml
 
 from typing import Dict, Any
 
@@ -37,13 +36,31 @@
 
 BOM = "\N{bom}"
 
-PROGRAM_IDS_FILE = os.path.join(
-                    os.path.split(__file__)[0],
-                    '../../ckanext/canada/tables/choices'
-                    '/service_program_ids.yaml')
+SERVICE_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_service_ids.csv')
+SERVICE_IDS = {}
+with open(SERVICE_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        SERVICE_IDS[row['service_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
+
+PROGRAM_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_program_ids.csv')
 PROGRAM_IDS = {}
-with open(PROGRAM_IDS_FILE, 'r') as f:
-    PROGRAM_IDS = yaml.safe_load(f)
+with open(PROGRAM_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        PROGRAM_IDS[row['program_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
 
 
 def test(record: Dict[str, Any]) -> Dict[str, Any]:
@@ -79,12 +96,24 @@ def process_row(row: Dict[str, Any]) -> Dict[str, Any]:
             count = int(row[field])
         row['num_applications_total'] += count
 
+    # populate service names from ids
+    row['service_name_en'] = SERVICE_IDS[row['service_id']]['en']
+    row['service_name_fr'] = SERVICE_IDS[row['service_id']]['fr']
+
     # populate program names from ids
     row['program_name_en'] = []
     row['program_name_fr'] = []
     program_ids = row['program_id'].split(',')
     for id in program_ids:
+        if id.endswith('-INV'):
+            # NOTE: not adding the suffix label to the published
+            #       program_name_en and program_name_fr fields.
+            id = id.replace('-INV', '')
         if id not in PROGRAM_IDS:
+            row['program_name_en'].append(
+                f'"Unknown Program Name for {id}"')
+            row['program_name_fr'].append(
+                f'"Inconnue désignation du programme pour {id}"')
             continue
         # NOTE: we add double quotes as Program Names can have
         #       single quotes and commas in them

bin/filter/filter_service_std.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import csv
+import os
 import sys
 
 from typing import Dict, Any
@@ -25,6 +26,19 @@
 
 BOM = "\N{bom}"
 
+SERVICE_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_service_ids.csv')
+SERVICE_IDS = {}
+with open(SERVICE_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        SERVICE_IDS[row['service_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
+
 
 def test(record: Dict[str, Any]) -> Dict[str, Any]:
     return process_row(record)
@@ -76,6 +90,10 @@ def process_row(row: Dict[str, Any]) -> Dict[str, Any]:
     else:
         row['target_met'] = 'N'
 
+    # populate service names from ids
+    row['service_name_en'] = SERVICE_IDS[row['service_id']]['en']
+    row['service_name_fr'] = SERVICE_IDS[row['service_id']]['fr']
+
     return row
 
 

bin/service_generate_reference_data.py

@@ -0,0 +1,233 @@
+#!/usr/bin/env python3
+"""
+Compiles all the reference data for Service Inventory.
+
+All the required files are in the release of
+https://github.com/gcperformance/service-data
+
+https://github.com/gcperformance/service-data/blob/master/src/utils.py
+    NOTE: program_list compiles fiscal years, orgs, program_ids, and their labels
+    NOTE: sid_list compiles fiscal years, orgs, service_ids, and their labels
+https://github.com/gcperformance/service-data/blob/master/src/export.py
+    NOTE: CSV files in the release use semicolon(;) as the delimiter
+"""
+
+import os
+import re
+import csv
+import json
+import requests
+
+
+ORG_LIST_URI = 'https://open.canada.ca/data/api/action/organization_list'
+RELEASE_URI = 'https://api.github.com/repos/gcperformance/service-data/releases/latest'
+
+SERVICE_ID_OUTPUT_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../ckanext/canada/tables/references/data/ref_service_service_ids.csv')
+SERVICE_ID_HEADERS = ['service_id', 'label_en', 'label_fr', 'org_years']
+
+PROGRAM_ID_OUTPUT_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../ckanext/canada/tables/references/data/ref_service_program_ids.csv')
+PROGRAM_ID_HEADERS = ['program_id', 'label_en', 'label_fr', 'org_years']
+
+REQUEST_HEADERS = {'User-Agent': 'CKAN/open-gov/service/gen_ref_data'}
+
+WHITE_SPACE_SUBS = re.compile(r'[\xa0\s]+')
+TIMESTAMP_MATCH = re.compile(r'^Timestamp:')
+
+ORG_VARIANTS_FILENAME = 'org_var.csv'
+PROGRAM_IDS_FILENAME = 'program_list.csv'
+SERVICE_IDS_FILENAME = 'sid_list.csv'
+
+
+def _clean_intake_text(text: str) -> str:
+    """
+    Normalize special characters and extra spaces
+    """
+    text = text.replace('\u200b', '')  # zero width char
+    text = re.sub(WHITE_SPACE_SUBS, ' ', text)
+    return text.strip()
+
+
+def _generate_data():
+    """
+    Generate the reference data for Service Inventory
+
+    Gather Program IDs and their English and French names from the directory
+    https://api.github.com/repos/gcperformance/service-data/releases/latest/program_list.csv
+
+    Gather Service IDs and their English and French names from
+    https://api.github.com/repos/gcperformance/service-data/releases/latest/sid_list.csv
+
+    NOTE: we only start in the 2018-2019 fiscal year as per Policy.
+
+    NOTE: we take the latest Program Names as the ones to use.
+    """
+    # get available org abbreviations on open gov
+    open_orgs = []
+    skipped_orgs = set()
+    with requests.get(ORG_LIST_URI, stream=True, headers=REQUEST_HEADERS) as response:
+        open_orgs = response.json()['result']
+    assert open_orgs
+
+    # gather the csv download URIs from the latest release
+    org_variants_uri = None
+    program_ids_uri = None
+    service_ids_uri = None
+    with requests.get(RELEASE_URI, stream=False) as response:
+        release_assets = response.json()['assets']
+        for f in release_assets:
+            if f['name'] == ORG_VARIANTS_FILENAME:
+                org_variants_uri = f['browser_download_url']
+                continue
+            if f['name'] == PROGRAM_IDS_FILENAME:
+                program_ids_uri = f['browser_download_url']
+                continue
+            if f['name'] == SERVICE_IDS_FILENAME:
+                service_ids_uri = f['browser_download_url']
+                continue
+    assert org_variants_uri
+    assert program_ids_uri
+    assert service_ids_uri
+
+    # compile list of Open Canada org abbreviations and Service Inventory org_ids
+    org_id_abbr_map = {}
+    with requests.get(org_variants_uri, stream=True) as response:
+        response.encoding = 'utf-8-sig'
+        c = csv.DictReader(response.iter_lines(decode_unicode=True))
+
+        assert 'org_name_variant' in c.fieldnames
+        assert 'org_id' in c.fieldnames
+
+        for row in c:
+            oname = _clean_intake_text(row['org_name_variant'])
+            oid = _clean_intake_text(row['org_id'])
+            if oname in open_orgs:
+                # NOTE: org names can share an ID for historical purposes.
+                # e.g. nsira-ossnr,302 is also sirc-csars,302 (abbr,umd)
+                # and the Registry may have both abbreviations.
+                if oid not in org_id_abbr_map:
+                    org_id_abbr_map[oid] = []
+                org_id_abbr_map[_clean_intake_text(row['org_id'])].append(oname)
+    assert org_id_abbr_map
+
+    # compile map of program_ids
+    program_id_map = {}
+    with requests.get(program_ids_uri, stream=True) as response:
+        response.encoding = 'utf-8-sig'
+        c = csv.DictReader(response.iter_lines(decode_unicode=True),
+                           delimiter=';')
+
+        assert 'org_id' in c.fieldnames
+        assert 'program_id' in c.fieldnames
+        assert 'latest_valid_fy' in c.fieldnames
+        assert 'program_en' in c.fieldnames
+        assert 'program_fr' in c.fieldnames
+
+        for row in c:
+            if not row['program_id']:
+                continue
+            program_id = _clean_intake_text(row['program_id'])
+            if not program_id:
+                continue
+            if program_id not in program_id_map:
+                program_id_map[program_id] = {}
+
+            label_en = _clean_intake_text(row['program_en'])
+            label_fr = _clean_intake_text(row['program_fr'])
+            if 'label_en' not in program_id_map[program_id]:
+                # take first occuring label
+                program_id_map[program_id]['label_en'] = label_en
+            if 'label_fr' not in program_id_map[program_id]:
+                # take first occuring label
+                program_id_map[program_id]['label_fr'] = label_fr
+
+            org = _clean_intake_text(row['org_id'])
+            if org not in org_id_abbr_map:
+                # org not in open gov, skip
+                if org not in skipped_orgs:
+                    print('Organization %s not available '
+                          'in Open Gov Registry. Skipping...' % org)
+                    skipped_orgs.add(org)
+                continue
+            mapped_orgs = org_id_abbr_map[org]
+
+            for _org in mapped_orgs:
+                if 'org_years' not in program_id_map[program_id]:
+                    program_id_map[program_id]['org_years'] = {}
+                if _org not in program_id_map[program_id]['org_years']:
+                    program_id_map[program_id]['org_years'][_org] = []
+                year = _clean_intake_text(row['latest_valid_fy'])
+                if year in program_id_map[program_id]['org_years'][_org]:
+                    continue
+                program_id_map[program_id]['org_years'][_org].append(year)
+    assert program_id_map
+
+    # write program_id ref data
+    with open(PROGRAM_ID_OUTPUT_FILE, 'w') as f:
+        writer = csv.DictWriter(f, PROGRAM_ID_HEADERS)
+        writer.writeheader()
+        for program_id, program_data in program_id_map.items():
+            writer.writerow({
+                'program_id': program_id,
+                'label_en': program_data['label_en'],
+                'label_fr': program_data['label_fr'],
+                'org_years': json.dumps(program_data['org_years'])
+                if 'org_years' in program_data else None})
+
+    # write service_id ref data
+    inserted_service_ids = set()
+    with requests.get(service_ids_uri, stream=True) as response:
+        response.encoding = 'utf-8-sig'
+        c = csv.DictReader(response.iter_lines(decode_unicode=True),
+                           delimiter=';')
+
+        assert 'service_id' in c.fieldnames
+        assert 'service_name_en' in c.fieldnames
+        assert 'service_name_fr' in c.fieldnames
+        assert 'org_id' in c.fieldnames
+        assert 'fiscal_yr_first' in c.fieldnames
+        assert 'fiscal_yr_latest' in c.fieldnames
+
+        with open(SERVICE_ID_OUTPUT_FILE, 'w') as f:
+            writer = csv.DictWriter(f, SERVICE_ID_HEADERS)
+            writer.writeheader()
+            for row in c:
+                if not row['service_id']:
+                    continue
+                service_id = _clean_intake_text(row['service_id'])
+                if not service_id or re.search(TIMESTAMP_MATCH, service_id):
+                    continue
+                if service_id in inserted_service_ids:
+                    raise Exception('%s already added...' % service_id)
+
+                org = _clean_intake_text(row['org_id'])
+                if org not in org_id_abbr_map:
+                    # org not in open gov, skip
+                    if org not in skipped_orgs:
+                        print('Organization %s not available '
+                              'in Open Gov Registry. Skipping...' % org)
+                        skipped_orgs.add(org)
+                    continue
+                mapped_orgs = org_id_abbr_map[org]
+
+                inserted_service_ids.add(service_id)
+
+                # just make same format as program_id
+                # org_years to make queries the same
+                org_years = {}
+                for _org in mapped_orgs:
+                    org_years[_org] = [_clean_intake_text(row['fiscal_yr_latest'])]
+
+                writer.writerow({
+                    'service_id': service_id,
+                    'label_en': _clean_intake_text(row['service_name_en']),
+                    'label_fr': _clean_intake_text(row['service_name_fr']),
+                    'org_years': json.dumps(org_years) if mapped_orgs else None})
+    assert inserted_service_ids
+
+
+if __name__ == '__main__':
+    _generate_data()

changes/1668.a.changes

@@ -0,0 +1 @@
+Updated the Service Inventory fields to use the new Recombinant features `choices_reference_table` and `choices_fiscal_year`

changes/1668.a.feature

@@ -0,0 +1 @@
+Added a bin script to generate Service Inventory Service IDs, Service Names, Program IDs, and Program Names (along with valid Organizations) for reference data from `https://github.com/gcperformance/service-data`. The data is tracked in CSV files in `ckanext.canada.tables.references.data`

changes/1668.b.changes

@@ -0,0 +1 @@
+Updated the Service Inventory triggers to query the new reference tables `ref_service_program_ids` and `ref_service_service_ids`

changes/1668.b.feature

@@ -0,0 +1 @@
+Added a `service.sql` pSQL script for `recombinant.reference_definitions`. This creates the tables `ref_service_program_ids` and `ref_service_service_ids` along with some indices for them.

changes/1668.c.changes

@@ -0,0 +1 @@
+The temporary session pSQL table `datastore_user` (now `datastore_app_context`) now has an `org_name` column which is the Organization name/abbreviation that the Resource belongs to.

changes/1668.c.feature

@@ -0,0 +1 @@
+Added a new subcommand `load-ref-data` which loads any generated reference data into their custom DataStore tables. This is done via a pSQL copy expert, copying the CSVs directly into the tables.