From 5a5967cb5ca17fe1a21fc7551dfb0c272c4dd725 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Fri, 20 May 2022 13:49:11 -0400 Subject: [PATCH 1/4] update dac member_name to controlled list --- bin/migrate/migrate_dac_2022_05.py | 120 +++++++++++++++++++++++++++++ ckanext/canada/tables/dac.yaml | 13 ++-- 2 files changed, 128 insertions(+), 5 deletions(-) create mode 100644 bin/migrate/migrate_dac_2022_05.py diff --git a/bin/migrate/migrate_dac_2022_05.py b/bin/migrate/migrate_dac_2022_05.py new file mode 100644 index 000000000..98a4c4daa --- /dev/null +++ b/bin/migrate/migrate_dac_2022_05.py @@ -0,0 +1,120 @@ +#!/usr/bin/env python +# coding=utf-8 + +import codecs +from decimal import Decimal +from re import sub +import sys +import unicodecsv +from unicodedata import normalize +import yaml + + +FIELDNAMES = [ + u'reporting_period', + u'line_number', + u'member_name', + u'province', + u'role', + u'meeting_hours', + u'other_hours', + u'remuneration', + u'travel_expenses', + u'notes_en', + u'notes_fr', + u'record_created', + u'record_modified', + u'user_modified', + u'owner_org', + u'owner_org_title', +] + +assert sys.stdin.read(3) == codecs.BOM_UTF8 + +in_csv = unicodecsv.DictReader(sys.stdin, encoding='utf-8') +out_csv = unicodecsv.DictWriter(sys.stdout, fieldnames=FIELDNAMES, encoding='utf-8') +out_csv.writeheader() + +error_csv = None +if sys.argv[1:]: + error_csv = unicodecsv.DictWriter(open(sys.argv[1], 'wb'), + fieldnames=FIELDNAMES, + encoding='utf-8') + error_csv.writeheader() + +# get DAC member names +dac_member_yaml = open("../../ckanext/canada/tables/choices/dac_members.yaml", "r") +members = yaml.safe_load(dac_member_yaml) + +try: + for line in in_csv: + # change money values such as $7,300.99 to 7300.99 + line['remuneration'] = Decimal(sub(r'[^\d.]', '', line['remuneration'])) + line['travel_expenses'] = Decimal(sub(r'[^\d.]', '', line['travel_expenses'])) + + # change free text member names to a controlled list + line[u'member_name'] = line[u'member_name'].strip() + + # map member names with typos where obvious + if line[u'member_name'] == u'Adrian deSaldanha': + best_match = u'de Saldanha, Adrian' + elif line[u'member_name'] == u'Angie Gillis': + best_match = u'Gillis, Angeline' + elif line[u'member_name'] == u'Carola Swan': + best_match = u'Swan, Carole' + elif line[u'member_name'] == u'Cassandra': + best_match = u'Dorrington, Cassandra' + elif line[u'member_name'] == u'Fredrick Gorbet': + best_match = u'Gorbet, Frederick' + elif line[u'member_name'] == u'Inga Shaene': + best_match = u'Sheane, Inga' + elif line[u'member_name'] == u'Almeida Iris-Côté': + best_match = u'Almeida-Côté, Iris' + elif line[u'member_name'] in [u'M. Sheikh', u'M. Shiekh', u'Munir Shiekh']: + best_match = u'Sheikh, Munir' + elif line[u'member_name'] == u'M. Roberts': + best_match = u'Roberts, Meena' + elif line[u'member_name'] == u'N. Whipp': + best_match = u'Whipp, Nancy' + elif line[u'member_name'] == u'Norman Turbull': + best_match = u'Turnbull, Norman' + elif line[u'member_name'] == u'Raymond Crabbe': + best_match = u'Crabbe, Ray' + elif line[u'member_name'] == u'Ruby Philip-Kaytal': + best_match = u'Philip-Katyal, Ruby' + elif line[u'member_name'] == u'Wendy Key': + best_match = u'Kei, Wendy' + + else: + name_parts = line[u'member_name'].split(' ') + matches = [] + for part in name_parts: + part = part.strip() + part = part.strip(',') + part = part.lower() + part = unicode(normalize('NFKD', part) + .encode('ASCII', 'ignore'), + 'utf-8') + + for member in map(unicode, members.keys()): + if part in unicode( + normalize('NFKD', member.lower()).encode('ASCII', 'ignore'), + 'utf-8'): + matches.append(member) + if len(matches): + best_match = max(set(matches), key=matches.count) + else: + best_match = 'NIL' + if matches.count(best_match) > 1: + line[u'member_name'] = best_match + else: + if error_csv: + error_csv.writerow(line) + + out_csv.writerow(line) + +except KeyError: + if 'warehouse' in sys.argv: + sys.exit(85) + else: + raise diff --git a/ckanext/canada/tables/dac.yaml b/ckanext/canada/tables/dac.yaml index ac0a06e34..ca0b23457 100644 --- a/ckanext/canada/tables/dac.yaml +++ b/ckanext/canada/tables/dac.yaml @@ -68,11 +68,13 @@ resources: obligation: Mandatory excel_required: true form_required: true - format_type: Free text + format_type: Controlled List validation: This field must not be empty + occurrence: Single datastore_type: text - form_attrs: - size: 60 + excel_full_text_choices: True + choices_file: choices/dac_members.yaml + excel_error_formula: 'FALSE' - datastore_id: province label: @@ -211,7 +213,7 @@ resources: obligation: Mandatory excel_required: true form_required: true - datastore_type: money + datastore_type: numeric - datastore_id: travel_expenses label: @@ -223,7 +225,7 @@ resources: obligation: Mandatory excel_required: true form_required: true - datastore_type: money + datastore_type: numeric - datastore_id: notes_en label: @@ -288,6 +290,7 @@ resources: errors := errors || choice_error(NEW.reporting_period, {reporting_period}, 'reporting_period'); errors := errors || required_error(NEW.line_number, 'line_number'); errors := errors || required_error(NEW.member_name, 'member_name'); + errors := errors || choice_error(NEW.member_name, {member_name}, 'member_name'); errors := errors || required_error(NEW.province, 'province'); errors := errors || choice_error(NEW.province, {province}, 'province'); errors := errors || required_error(NEW.meeting_hours, 'meeting_hours'); From e730f5198b219be6a70e9e091926c1bdb1f74954 Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Wed, 25 May 2022 13:48:26 -0400 Subject: [PATCH 2/4] include a copy of dac members list for dw migrations --- bin/migrate/choices/dac_members_2022_05.yaml | 133 +++++++++++++++++++ bin/migrate/migrate_dac_2022_05.py | 2 +- 2 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 bin/migrate/choices/dac_members_2022_05.yaml diff --git a/bin/migrate/choices/dac_members_2022_05.yaml b/bin/migrate/choices/dac_members_2022_05.yaml new file mode 100644 index 000000000..a759a344d --- /dev/null +++ b/bin/migrate/choices/dac_members_2022_05.yaml @@ -0,0 +1,133 @@ +# Departmental Audit Committee member names to be used in dropdown on recombinant templates + +Alexander, Jim: Alexander, Jim +Almeida-Côté, Iris: Almeida-Côté, Iris +Anderson, Roxanne: Anderson, Roxanne +Bains, Manjit: Bains, Manjit +Baker, William: Baker, William +Bates, Paul: Bates, Paul +Bradbury, Victoria: Bradbury, Victoria +Brant, Daniel: Brant, Daniel +Cochrane, Ken: Cochrane, Ken +Cook-Bennett, Gail: Cook-Bennett, Gail +Corriveau, Pierre: Corriveau, Pierre +Costante, Kevin: Costante, Kevin +Cotton, Katherine: Cotton, Katherine +Crabbe, Ray: Crabbe, Ray +Cramer, Joy: Cramer, Joy +Dancey, Kevin: Dancey, Kevin +De La Durantaye, Josée: De La Durantaye, Josée +de Saldanha, Adrian: de Saldanha, Adrian +del Val, Helen: del Val, Helen +Denhamer, Janet: Denhamer, Janet +Dhaliwal, Maninder: Dhaliwal, Maninder +Dias D'Souza, Jeanette: Dias D'Souza, Jeanette +Dicerni, Richard: Dicerni, Richard +Domm, John: Domm, John +Dorrington, Cassandra: Dorrington, Cassandra +Drouin, Nathalie: Drouin, Nathalie +Duranceau, France-Élaine: Duranceau, France-Élaine +Edwards, Laurie: Edwards, Laurie +Ell, Lori: Ell, Lori +Ferguson, Don: Ferguson, Don +Finn, Patrick: Finn, Patrick +Forster, John: Forster, John +Fortin, Hélène: Fortin, Hélène +Fregin, Clifford: Fregin, Clifford +Gaudet, Serge: Gaudet, Serge +Gillis, Angeline: Gillis, Angeline +Girard, Aline: Girard, Aline +Godbout, Claude: Godbout, Claude +Gorbet, Frederick: Gorbet, Frederick +Gouin, Suzanne: Gouin, Suzanne +Hanlon, Robert: Hanlon, Robert +Hanna, John: Hanna, John +Harnish, Victoria: Harnish, Victoria +Harvey, Manon: Harvey, Manon +Hollins, Leah: Hollins, Leah +Jackson, Karen: Jackson, Karen +Jacobsen, Patricia: Jacobsen, Patricia +Jamal, Karim: Jamal, Karim +Janes, Glenn: Janes, Glenn +Jeraj, Shenaz: Jeraj, Shenaz +Jung, Dr. Hans: Jung, Dr. Hans +Katiya, Alan: Katiya, Alan +Kei, Wendy: Kei, Wendy +Kiamanesh, Mitra: Kiamanesh, Mitra +Kieley, Barbara: Kieley, Barbara +Kirkpatrick, Kent: Kirkpatrick, Kent +Kirvan, Myles: Kirvan, Myles +La Rose, Jean: La Rose, Jean +Lafortune, Andrée: Lafortune, Andrée +Lahey, James: Lahey, James +Lalli, Sandip: Lalli, Sandip +Lamoureux, Kenneth K.: Lamoureux, Kenneth K. +Leduc, Raymond: Leduc, Raymond +Liston-Heyes, Catherine: Liston-Heyes, Catherine +Lizotte-MacPherson, Linda: Lizotte-MacPherson, Linda +Lussier, Gaétan: Lussier, Gaétan +Maheu, Lorraine: Maheu, Lorraine +Malik, Tariq: Malik, Tariq +Maracle, Thomas: Maracle, Thomas +Marcoux, Rennie: Marcoux, Rennie +Maxwell, Dr. Linda: Maxwell, Dr. Linda +Maxwell, Neil: Maxwell, Neil +McKenzie, Michele: McKenzie, Michele +McLaughlin, Michael: McLaughlin, Michael +McLellan, George: McLellan, George +McWhinnie, John: McWhinnie, John +Meredith, Daphne: Meredith, Daphne +Miller, David: Miller, David +Minto, Shahid: Minto, Shahid +Mitchell, James (Jim): Mitchell, James (Jim) +Morris, Suzanne: Morris, Suzanne +Nadeau, Elisabeth: Nadeau, Elisabeth +Negris, Cybele: Negris, Cybele +Nelson, Michael: Nelson, Michael +Newman, Deborah: Newman, Deborah +Norminton, Monica: Norminton, Monica +Ostapchuk, Peter: Ostapchuk, Peter +Oxley, Loraine: Oxley, Loraine +Paquette, Jacques: Paquette, Jacques +Patry, Gilles: Patry, Gilles +Pelman, Dr. Alan: Pelman, Dr. Alan +Philip-Katyal, Ruby: Philip-Katyal, Ruby +Poitras, Anne-Marie: Poitras, Anne-Marie +Pollard, Carol: Pollard, Carol +Pratt, Betty-Anne: Pratt, Betty-Anne +Proulx, Robert: Proulx, Robert +Péan, Chantal: Péan, Chantal +Rahnema, Ali: Rahnema, Ali +Renaud, Anne-Marie: Renaud, Anne-Marie +Roberts, Meena: Roberts, Meena +Rossetti, John: Rossetti, John +Ruta, Basia: Ruta, Basia +Scott, Kimberly: Scott, Kimberly +Sheane, Inga: Sheane, Inga +Sheikh, Munir: Sheikh, Munir +Sinclair, Helen: Sinclair, Helen +Singleton, Jon: Singleton, Jon +Smith, David: Smith, David +Smith, Mindy: Smith, Mindy +Somani, Moyez: Somani, Moyez +Soria, Elizabeth (Liz): Soria, Elizabeth (Liz) +Stark, Dr. Deborah: Stark, Dr. Deborah +Sumara, Joyce: Sumara, Joyce +Swan, Carole: Swan, Carole +Sweetnam, Albert: Sweetnam, Albert +Talbot-Allan, Laura: Talbot-Allan, Laura +Tessier, Sylvie: Tessier, Sylvie +Thompson, John: Thompson, John +Thompson, Stanley: Thompson, Stanley +Tremblay, Pierre: Tremblay, Pierre +Turnbull, Norman: Turnbull, Norman +Van-Erum, Micheline: Van-Erum, Micheline +Voghel, Sylvie: Voghel, Sylvie +Wallace, Stephen: Wallace, Stephen +Weeks, Joanne: Weeks, Joanne +Whipp, Nancy: Whipp, Nancy +Wong-Alafriz, Kay: Wong-Alafriz, Kay +Yeates, Glenda: Yeates, Glenda +Yeates, Neil: Yeates, Neil +Zaarour, Roula: Zaarour, Roula +Zussman, Dr. David: Zussman, Dr. David diff --git a/bin/migrate/migrate_dac_2022_05.py b/bin/migrate/migrate_dac_2022_05.py index 98a4c4daa..3fc6924f0 100644 --- a/bin/migrate/migrate_dac_2022_05.py +++ b/bin/migrate/migrate_dac_2022_05.py @@ -43,7 +43,7 @@ error_csv.writeheader() # get DAC member names -dac_member_yaml = open("../../ckanext/canada/tables/choices/dac_members.yaml", "r") +dac_member_yaml = open("choices/dac_members_2022_05.yaml", "r") members = yaml.safe_load(dac_member_yaml) try: From 5285318add14395a7144c577a6703e6cc34f377d Mon Sep 17 00:00:00 2001 From: Rabia Sajjad Date: Wed, 25 May 2022 15:05:15 -0400 Subject: [PATCH 3/4] use os.path for dac members list --- bin/migrate/migrate_dac_2022_05.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/bin/migrate/migrate_dac_2022_05.py b/bin/migrate/migrate_dac_2022_05.py index 3fc6924f0..dc9a3ff97 100644 --- a/bin/migrate/migrate_dac_2022_05.py +++ b/bin/migrate/migrate_dac_2022_05.py @@ -3,6 +3,7 @@ import codecs from decimal import Decimal +import os from re import sub import sys import unicodecsv @@ -43,7 +44,9 @@ error_csv.writeheader() # get DAC member names -dac_member_yaml = open("choices/dac_members_2022_05.yaml", "r") +dac_member_yaml = open( + os.path.join(os.path.dirname(__file__), + "choices/dac_members_2022_05.yaml"), "r") members = yaml.safe_load(dac_member_yaml) try: From 0f456cbf90a52470a71c3a58c96146a0a2ba1cc5 Mon Sep 17 00:00:00 2001 From: JVickery-TBS Date: Mon, 8 May 2023 18:35:22 +0000 Subject: [PATCH 4/4] feat(tests): added dac member name tests; - Updated test to handle dac member choices. --- ckanext/canada/tests/test_dac.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ckanext/canada/tests/test_dac.py b/ckanext/canada/tests/test_dac.py index c87386b0f..f2816b4a3 100644 --- a/ckanext/canada/tests/test_dac.py +++ b/ckanext/canada/tests/test_dac.py @@ -6,6 +6,7 @@ from ckan.tests.factories import Organization from ckanext.recombinant.tables import get_chromo +from ckanext.recombinant.helpers import recombinant_choice_fields class TestDAC(FunctionalTestBase): def setup(self): @@ -19,6 +20,12 @@ def setup(self): def test_example(self): lc = LocalCKAN() record = get_chromo('dac')['examples']['record'] + choices_fields = recombinant_choice_fields('dac') + for f in choices_fields: + if f['datastore_id'] != 'member_name': + continue + record['member_name'] = f['choices'][0][0] + break lc.action.datastore_upsert( resource_id=self.resource_id, records=[record])