Rework Code Base for Single Instance

.github/workflows/pytest.yml

@@ -213,7 +213,7 @@ jobs:
               printf "\n    ${srcName}: pip info\n";
               pip show ${srcName};
               if [[ "${reslog}" == "" && "${DO_PIP_REINSTALL}" == "false" ]]; then
-                  printf "${srcName}: Up to date. Skipping..."
+                printf "${srcName}: Up to date. Skipping..."
               else
                 git pull --ff-only || true;
                 printf "\n${srcName}: Pulled updated code. Re-initializing in python environment..."
@@ -231,6 +231,15 @@ jobs:
                   python $dir/setup.py develop || true;
                 fi;
               fi;
+              if [[ -f "$dir/dev-requirements.txt" ]]; then
+                pip install -r $dir/dev-requirements.txt || true;
+              fi;
+              if [[ -f "$dir/test-requirements.txt" ]]; then
+                pip install -r $dir/test-requirements.txt || true;
+              fi;
+              if [[ -f "$dir/setup.py" ]]; then
+                python $dir/setup.py develop || true;
+              fi;
             else
               printf "\n${srcName}: Not a source repository. Skipping..."
             fi;
@@ -255,6 +264,8 @@ jobs:
           pip install -r test-requirements.txt
           pip install setuptools==70.0.0
           python3 setup.py develop
+          npm install -D
+          npm run build
           cd /srv/app/src/ckan
           pip install -e .
           python3 setup.py develop
@@ -267,13 +278,16 @@ jobs:
           . /srv/app/src/ckan/bin/postgres_init/2_create_ckan_datastore_db.sh
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db init
           ckan -c /srv/app/src/ckanext-canada/test-core.ini datastore set-permissions | psql -U postgres --set ON_ERROR_STOP=1
+          ckan -c /srv/app/src/ckanext-canada/test-core.ini xloader db-init
           ckan -c /srv/app/src/ckanext-canada/test-core.ini validation init-db
           ckan -c /srv/app/src/ckanext-canada/test-core.ini security migrate
           ckan -c /srv/app/src/ckanext-canada/test-core.ini canada update-triggers
           ckan -c /srv/app/src/ckanext-canada/test-core.ini recombinant create-triggers -a
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db upgrade
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db pending-migrations --apply
           ckan -c /srv/app/src/ckanext-canada/test-core.ini db pending-migrations --apply
+          ckan -c /srv/app/src/ckanext-canada/test-core.ini recombinant create-ref-tables
+          ckan -c /srv/app/src/ckanext-canada/test-core.ini pd load-ref-data -v
       - name: Download Test Durations
         if: needs.define-matrix.outputs.restored == 'true'
         uses: actions/download-artifact@v4
@@ -384,7 +398,7 @@ jobs:
           for dir in /srv/app/src/*; do
             srcName=$(echo "$dir" | awk -F'/' '{print $NF}');
             if [[ -d "$dir/.git"  ]]; then
-                printf "\n${srcName}: Checking for updates..."
+                printf "\n\n${srcName}: Checking for updates..."
                 cd $dir;
                 current_branch=$(git rev-parse --abbrev-ref HEAD);
                 if [[ "$srcName" == "ckan" ]]; then
@@ -395,10 +409,10 @@ jobs:
                     reslog=$(git log HEAD..origin/$current_branch --oneline);
                 fi;
                 if [[ "${reslog}" == "" ]]; then
-                    printf "\n${srcName}: Up to date. Skipping..."
+                    printf "    ${srcName}: Up to date. Skipping..."
                 else
                     git pull --ff-only || true;
-                    printf "\n${srcName}: Pulled updated code. Re-initializing in python environment..."
+                    printf "\n    ${srcName}: Pulled updated code. Re-initializing in python environment..."
                     cd /srv/app;
                     if [[ -f "$dir/requirements.txt" ]]; then
                         pip install -r $dir/requirements.txt || true;

.github/workflows/si_release_listener.yml

@@ -0,0 +1,34 @@
+name: Service Inventory Release Listener
+on:
+  issues:
+    types: [opened]
+
+jobs:
+  make_pr:
+    if: startsWith(github.event.issue.title, '[SI-RELEASE]')
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v4
+
+      - name: Run Ref Data Generation
+        run: |
+          cd bin;
+          python service_generate_reference_data.py;
+
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v6
+        with:
+          commit-message: |
+            build(pd): service inventory;
+
+            - Auto build ref data from issue ${{ github.event.issue.number }}.
+          branch: "si-release/${{ github.event.issue.number }}"
+          title: "[SI-RELEASE] Issue #${{ github.event.issue.number }}"
+          body: |
+            Newly generated reference data for Service Inventory.
+
+            This PR was created automatically from issue #${{ github.event.issue.number }}
+
+            cc: @ge-tbs
+          reviewers: JVickery-TBS, RabiaSajjad

.gitignore

@@ -2,7 +2,6 @@ ckanext_canada.egg-info
 *.pyc
 .project
 .pydevproject
-build
 dist
 *.swp
 development.ini

README.md

@@ -15,6 +15,7 @@
 | [Configurations](#configurations)    |
 | [SOLR](#solr)    |
 | [Localization](#localization)    |
+| [WebAssets](#webassets)    |
 | [Migrations](#migrations)    |
 | [Data Flows](#data-flows)    |
 
@@ -259,6 +260,17 @@ updated translations by running:
 python setup.py compile_catalog
 ```
 
+## WebAssets
+
+CKAN uses python webassets to compile and serve front-end assets like CSS stylesheets and JS scripts. Though webassets offers filters for Sass and JS Minify, it is better practice to not have Node build utilities on production servers. Thus we compile the assets locally via Node and commit to the source GitHub repository (though this is also not best practice, it is required due to deployment constraints).
+
+If you change webassets, you will need to build them:
+
+```
+npm install -D
+npm run build
+```
+
 ## Migrations
 
 ### Creating a plugin migration

bin/filter/filter_service.py

@@ -3,7 +3,6 @@
 import csv
 import os
 import sys
-import yaml
 
 from typing import Dict, Any
 
@@ -37,13 +36,31 @@
 
 BOM = "\N{bom}"
 
-PROGRAM_IDS_FILE = os.path.join(
-                    os.path.split(__file__)[0],
-                    '../../ckanext/canada/tables/choices'
-                    '/service_program_ids.yaml')
+SERVICE_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_service_ids.csv')
+SERVICE_IDS = {}
+with open(SERVICE_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        SERVICE_IDS[row['service_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
+
+PROGRAM_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_program_ids.csv')
 PROGRAM_IDS = {}
-with open(PROGRAM_IDS_FILE, 'r') as f:
-    PROGRAM_IDS = yaml.safe_load(f)
+with open(PROGRAM_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        PROGRAM_IDS[row['program_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
 
 
 def test(record: Dict[str, Any]) -> Dict[str, Any]:
@@ -79,12 +96,26 @@ def process_row(row: Dict[str, Any]) -> Dict[str, Any]:
             count = int(row[field])
         row['num_applications_total'] += count
 
+    # populate service names from ids
+    # though part of service_id controlled list, still require
+    # users to input the Service Name. (2026-09-01;Sept.)
+    # row['service_name_en'] = SERVICE_IDS[row['service_id']]['en']
+    # row['service_name_fr'] = SERVICE_IDS[row['service_id']]['fr']
+
     # populate program names from ids
     row['program_name_en'] = []
     row['program_name_fr'] = []
     program_ids = row['program_id'].split(',')
     for id in program_ids:
+        if id.endswith('-INV'):
+            # NOTE: not adding the suffix label to the published
+            #       program_name_en and program_name_fr fields.
+            id = id.replace('-INV', '')
         if id not in PROGRAM_IDS:
+            row['program_name_en'].append(
+                f'"Unknown Program Name for {id}"')
+            row['program_name_fr'].append(
+                f'"Inconnue désignation du programme pour {id}"')
             continue
         # NOTE: we add double quotes as Program Names can have
         #       single quotes and commas in them

bin/filter/filter_service_std.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 
 import csv
+import os
 import sys
 
 from typing import Dict, Any
@@ -25,6 +26,19 @@
 
 BOM = "\N{bom}"
 
+SERVICE_ID_REF_DATA_FILE = os.path.join(
+    os.path.split(__file__)[0],
+    '../../ckanext/canada/tables/references'
+    '/data/ref_service_service_ids.csv')
+SERVICE_IDS = {}
+with open(SERVICE_ID_REF_DATA_FILE, 'r') as f:
+    c = csv.DictReader(f)
+    for row in c:
+        SERVICE_IDS[row['service_id']] = {
+            'en': row['label_en'],
+            'fr': row['label_fr']
+        }
+
 
 def test(record: Dict[str, Any]) -> Dict[str, Any]:
     return process_row(record)
@@ -76,6 +90,12 @@ def process_row(row: Dict[str, Any]) -> Dict[str, Any]:
     else:
         row['target_met'] = 'N'
 
+    # populate service names from ids
+    # though part of service_id controlled list, still require
+    # users to input the Service Name. (2026-09-01;Sept.)
+    # row['service_name_en'] = SERVICE_IDS[row['service_id']]['en']
+    # row['service_name_fr'] = SERVICE_IDS[row['service_id']]['fr']
+
     return row