Organization Validation for Service Inventory

.github/test-dependencies.json

@@ -10,6 +10,16 @@
         "*": "canada-staging"
       }
     },
+    "ckanext-recombinant": {
+      "branches": {
+        "master": "master",
+        "dev": "dev",
+        "staging": "dev",
+        "feature/*": "dev",
+        "fix/*": "dev",
+        "*": "dev"
+      }
+    },
     "ckanext-xloader": {
       "branches": {
         "master": "canada-v2.10",

.github/workflows/pytest.yml

@@ -273,6 +273,8 @@ jobs:
           ckan -c /srv/app/src/ckanext-canada/test-core.ini recombinant create-triggers -a
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db upgrade
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db pending-migrations --apply
+          ckan -c /srv/app/src/ckanext-canada/test-core.ini recombinant create-ref-tables
+          ckan -c /srv/app/src/ckanext-canada/test-core.ini pd load-ref-data -v
       - name: Download Test Durations
         if: needs.define-matrix.outputs.restored == 'true'
         uses: actions/download-artifact@v4

.github/workflows/si_release_listener.yml

@@ -0,0 +1,34 @@
+name: Service Inventory Release Listener
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  make_pr:
+    if: startsWith(github.event.issue.title, '[SI-RELEASE]')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Run Ref Data Generation
+        run: |
+          cd bin;
+          python service_generate_reference_data.py;
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: |
+            build(pd): service inventory;
+
+            - Auto build ref data from issue ${{ github.event.issue.number }}.
+          branch: "si-release/${{ github.event.issue.number }}"
+          title: "[SI-RELEASE] Issue #${{ github.event.issue.number }}"
+          body: |
+            Newly generated reference data for Service Inventory.
+
+            This PR was created automatically from issue #${{ github.event.issue.number }}
+
+            cc: @ge-tbs
+          reviewers: JVickery-TBS, RabiaSajjad

bin/filter/filter_service.py

@@ -3,7 +3,6 @@
 import csv
 import os
 import sys
-import yaml
 
 from typing import Dict, Any
 
@@ -37,13 +36,31 @@
 
 BOM = "\N{bom}"
 
-PROGRAM_IDS_FILE = os.path.join(
-                    os.path.split(__file__)[0],
-                    '../../ckanext/canada/tables/choices'
-                    '/service_program_ids.yaml')
+SERVICE_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_service_ids.csv')
+SERVICE_IDS = {}
+with open(SERVICE_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        SERVICE_IDS[row['service_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
+
+PROGRAM_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_program_ids.csv')
 PROGRAM_IDS = {}
-with open(PROGRAM_IDS_FILE, 'r') as f:
-    PROGRAM_IDS = yaml.safe_load(f)
+with open(PROGRAM_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        PROGRAM_IDS[row['program_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
 
 
 def test(record: Dict[str, Any]) -> Dict[str, Any]:
@@ -79,12 +96,24 @@ def process_row(row: Dict[str, Any]) -> Dict[str, Any]:
             count = int(row[field])
         row['num_applications_total'] += count
 
+    # populate service names from ids
+    row['service_name_en'] = SERVICE_IDS[row['service_id']]['en']
+    row['service_name_fr'] = SERVICE_IDS[row['service_id']]['fr']
+
     # populate program names from ids
     row['program_name_en'] = []
     row['program_name_fr'] = []
     program_ids = row['program_id'].split(',')
     for id in program_ids:
+        if id.endswith('-INV'):
+            # NOTE: not adding the suffix label to the published
+            #       program_name_en and program_name_fr fields.
+            id = id.replace('-INV', '')
         if id not in PROGRAM_IDS:
+            row['program_name_en'].append(
+                f'"Unknown Program Name for {id}"')
+            row['program_name_fr'].append(
+                f'"Inconnue désignation du programme pour {id}"')
             continue
         # NOTE: we add double quotes as Program Names can have
         #       single quotes and commas in them

bin/filter/filter_service_std.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import csv
+import os
 import sys
 
 from typing import Dict, Any
@@ -25,6 +26,19 @@
 
 BOM = "\N{bom}"
 
+SERVICE_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_service_ids.csv')
+SERVICE_IDS = {}
+with open(SERVICE_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        SERVICE_IDS[row['service_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
+
 
 def test(record: Dict[str, Any]) -> Dict[str, Any]:
     return process_row(record)
@@ -76,6 +90,10 @@ def process_row(row: Dict[str, Any]) -> Dict[str, Any]:
     else:
         row['target_met'] = 'N'
 
+    # populate service names from ids
+    row['service_name_en'] = SERVICE_IDS[row['service_id']]['en']
+    row['service_name_fr'] = SERVICE_IDS[row['service_id']]['fr']
+
     return row