Sample with unique PK generation

changes/155.changes

@@ -0,0 +1 @@
+Create test sample recombinant definition with example of custom primary key generation

ckanext/recombinant/cli.py

@@ -536,7 +536,11 @@ def _create_triggers(dataset_types: Optional[List[str]],
     """
     lc = LocalCKAN()
     for dtype in _expand_dataset_types(dataset_types, all_types):
-        for chromo in get_geno(dtype)['resources']:
+        geno = get_geno(dtype)
+        if seq := geno.get('datastore_create_sequence'):
+            lc = LocalCKAN()
+            lc.action.datastore_sequence_create(name=seq, if_not_exists=True)
+        for chromo in geno['resources']:
             _update_triggers(lc, chromo)
 
 
@@ -618,10 +622,9 @@ def _load_one_csv_file(name: str) -> int:
 
     dataset_type = chromo['dataset_type']
     method = 'upsert' if chromo.get('datastore_primary_key') else 'insert'
-    lc = LocalCKAN()
+    lc = LocalCKAN(context={'datastore_import': True})  # required for importing _id
     errors = 0
 
-    # dynamic fields
     dynamic_fields = [
         'owner_org',
         'owner_org_title',
@@ -770,9 +773,10 @@ def _write_one_csv(lc: LocalCKAN,
                    chromo: Dict[str, Any],
                    outfile: TextIO):
     out = csv.writer(outfile)
-    column_ids = [
-        f['datastore_id'] for f in chromo['fields'] if
-        not f.get('published_resource_computed_field')] + \
+    column_ids = \
+        (['_id'] if chromo.get('edit_using__id') else []) + \
+        [f['datastore_id'] for f in chromo['fields'] if
+            not f.get('published_resource_computed_field')] + \
         chromo.get('csv_org_extras', []) + \
         ['owner_org', 'owner_org_title']
     out.writerow(column_ids)

ckanext/recombinant/read_csv.py

@@ -45,11 +45,12 @@ def csv_data_batch(csv_path: str,
                     if f not in ignore_fields]
 
         if strict:
-            expected = [f['datastore_id'] for f in chromo['fields'] if not f.get(
-                'published_resource_computed_field')]
+            expected = \
+                (['_id'] if chromo.get('edit_using__id') else []) + \
+                [f['datastore_id'] for f in chromo['fields'] if not f.get(
+                    'published_resource_computed_field')]
             if ignore_fields:
-                expected = [f['datastore_id'] for f in chromo['fields'] if
-                            f['datastore_id'] not in ignore_fields]
+                expected = [f for f in expected if f not in ignore_fields]
             assert cols == expected, 'column mismatch:\n{0}\n{1}'.format(
                 cols, expected)
 

ckanext/recombinant/read_excel.py

@@ -66,27 +66,51 @@ def read_excel(f: Union[str, FlaskFileStorage, FieldStorage],
         names_row = next(rowiter)
 
         org_name = organization_row[0].value
-        if org_name and names_row[0].value != 'v3':
+        sig = names_row[0].value
+        if org_name and not sig.startswith('v3'):
             # v2 template
             yield (
                 _sheetname,
                 org_name,
                 [c.value for c in names_row],
                 # type_ignore_reason: incomplete typing
-                _filter_bumf(rowiter, HEADER_ROWS_V2))
+                _filter_bumf(rowiter, HEADER_ROWS_V2),
+                'upsert')
             continue
 
         # type_ignore_reason: incomplete typing
         next(rowiter)
         example_row = next(rowiter)
-        if example_row[0].value != 'e.g.' and example_row[0].value != 'ex.':
+        if (
+            sig != 'v3-update'  # we remove the example from update-only template
+            and example_row[0].value not in ('e.g.', 'ex.')
+        ):
             raise BadExcelData('Example record on row 5 is missing')
 
-        yield (
-            _sheetname,
-            names_row[1].value,
-            [c.value for c in names_row[2:]],
-            _filter_bumf((row[2:] for row in rowiter), HEADER_ROWS_V3))
+        if sig == 'v3-update':
+            yield (
+                _sheetname,
+                names_row[1].value,
+                ['_id'] + [c.value for c in names_row[2:]],
+                _filter_bumf((row[1:] for row in rowiter), HEADER_ROWS_V3),
+                'update')
+        elif sig == 'v3-insert':
+            yield (
+                _sheetname,
+                names_row[1].value,
+                [c.value for c in names_row[2:]],
+                _filter_bumf((row[2:] for row in rowiter), HEADER_ROWS_V3),
+                'insert')
+        elif sig == 'v3':
+            yield (
+                _sheetname,
+                names_row[1].value,
+                [c.value for c in names_row[2:]],
+                _filter_bumf((row[2:] for row in rowiter), HEADER_ROWS_V3),
+                'upsert')
+        else:
+            raise BadExcelData(
+                f'Unknown signature {sig!r}, must be one: v3, v3-insert, v3-update')
 
 
 def _filter_bumf(rowiter: Iterator[Any],

ckanext/recombinant/tests/samples/sample_pk_deluxe.yaml

@@ -0,0 +1,117 @@
+dataset_type: sampledeluxe
+target_dataset: sampledeluxe
+
+
+title: Sample PK-Deluxe Dataset Type
+shortname: Sample PK-Deluxe Dataset Type
+notes: Sample PK-Deluxe Dataset Type
+
+datastore_create_sequence: sample_pk_deluxe_seq
+
+template_version: 3
+
+portal_type: dataset
+collection: sample
+
+resources:
+- title: Sample Resource
+  resource_name: sampledeluxe
+
+  published_resource_id: 1495491e-338c-43ec-9995-ecb48c67beef
+  edit_using__id: true
+
+  create_form: true
+  edit_form: true
+  fields:
+  - datastore_id: year
+    datastore_type: year
+    label: Year
+    description:
+      en: Sample Year Field
+      fr: SampleYear Field FR
+    obligation: Mandatory
+    excel_required: true
+    form_required: true
+    validation: This field must not be empty
+    excel_column_width: 10
+    extract_date_year: true
+    form_attrs:
+      size: 20
+
+  - datastore_id: value
+    datastore_type: int
+    label: Value
+    obligation: Mandatory
+    excel_required: true
+    form_required: true
+    validation: This field must not be empty
+    excel_column_width: 10
+
+  datastore_primary_key: year
+  datastore_indexes: ""
+
+  default_preview_sort: year
+
+  excel_example_height: 100
+  excel_data_num_rows: 500
+
+  triggers:
+  - test_trigger_1: |
+      DECLARE
+        errors text[][] := '{{}}';
+        crval RECORD;
+        importing boolean NOT NULL := (SELECT importing
+            FROM datastore_user LIMIT 1);
+      BEGIN
+        errors := errors || required_error(NEW.year, 'year');
+        errors := errors || required_error(NEW.value, 'value');
+
+        IF TG_OP = 'UPDATE' THEN
+          errors := errors || required_error(NEW._id, '_id');
+        END IF;
+
+        IF errors = '{{}}' THEN
+          IF NOT importing AND TG_OP = 'INSERT' THEN
+            -- open_canada_id generation rule --
+
+            -- Assign unique _id across *all* records and so that (_id % 97) == 5
+            -- These _id values are exported as open_canada_id in the combined csv dataset
+
+            -- The last two _id digits are check digits. Mod 97 prevents almost all numeric
+            -- typos (Mod 97 is also used for international bank account check digits)
+
+            -- 5 chosen only because this is a Sample and 5 looks like an S.
+            -- open_canada_id values for other record types use values
+            -- different than 5 so that open_canada_ids are unique across all record types
+
+            -- Use check digits from 03-99 because numbers ending in 00 look approximate,
+            -- i.e. for all open_canada_id values (_id % 100) > 2
+
+            NEW._id := nextval('sample_pk_deluxe_seq') * 100 + 99;
+            NEW._id := NEW._id - (NEW._id - 5) % 97;
+          END IF;
+          RETURN NEW;
+        END IF;
+        RAISE EXCEPTION E'TAB-DELIMITED\t%', array_to_string(errors, E'\t');
+      END;
+
+  examples:
+    record:
+      year: 2026
+      value: 42
+    filter_one:
+      year: 2026
+    sort: year desc
+
+excel_edge_style:
+  PatternFill:
+    fgColor: FF336B87
+    patternType: solid
+excel_header_style:
+  PatternFill:
+    patternType: solid
+    fgColor: FF6832e3
+excel_column_heading_style:
+  PatternFill:
+    patternType: solid
+    fgColor: FFEFEFEF

ckanext/recombinant/views.py

@@ -300,12 +300,13 @@ def template(dataset_type: str, lang: str, owner_org: str) -> Union[Response, st
     except NotFound:
         return abort(404, _('Not found'))
 
-    try:
-        book = excel_template(dataset_type, org)
-    except RecombinantException as e:
-        return abort(400, _('Unable to download template.\n%s') % e)
+    if request.method == 'GET':
+        try:
+            book = excel_template(dataset_type, org)
+        except RecombinantException as e:
+            return abort(400, _('Unable to download template.\n%s') % e)
 
-    if request.method == 'POST':
+    elif request.method == 'POST':
         filters = {}
         resource_name = request.form.get('resource_name', '')
         for r in dataset['resources']:
@@ -320,6 +321,10 @@ def template(dataset_type: str, lang: str, owner_org: str) -> Union[Response, st
         chromo = get_chromo(resource['name'])
         record_data = []
 
+        edit_using__id = bool(chromo.get('edit_using__id'))
+        if edit_using__id:
+            pk_fields = [{'datastore_id': '_id'}]
+
         for keys in primary_keys:
             temp = keys.split(",")
             for f, pkf in zip(temp, pk_fields):
@@ -333,7 +338,13 @@ def template(dataset_type: str, lang: str, owner_org: str) -> Union[Response, st
             record_data += result['records']
 
         try:
-            append_data(book, record_data, chromo)
+            book = excel_template(
+                dataset_type, org, edit_using__id, len(record_data))
+        except RecombinantException as e:
+            return abort(400, _('Unable to download template.\n%s') % e)
+
+        try:
+            append_data(book, record_data, chromo, edit_using__id)
         except RecombinantFieldError as e:
             h.flash_error(render('recombinant/snippets/outdated_error.html',
                                  extra_vars={'key_errors': str(e).replace("'", ''),
@@ -792,7 +803,7 @@ def _process_upload_file(lc: LocalCKAN,
     try:
         while True:
             try:
-                sheet_name, org_name, column_names, rows = next(upload_data)
+                sheet_name, org_name, column_names, rows, method = next(upload_data)
             except StopIteration:
                 break
             except BadExcelData as e:
@@ -841,6 +852,12 @@ def _process_upload_file(lc: LocalCKAN,
             expected_columns = [f['datastore_id'] for f in chromo['fields']
                                 if f.get('import_template_include', True) and
                                 not f.get('published_resource_computed_field')]
+            if method == 'update':
+                expected_columns = ['_id'] + expected_columns
+                pk = ['_id']
+            else:
+                pk = chromo.get('datastore_primary_key', [])
+
             if column_names != expected_columns:
                 raise BadExcelData(
                     _("This template is out of date. "
@@ -850,7 +867,6 @@ def _process_upload_file(lc: LocalCKAN,
                       "{support} so we may investigate.").format(
                           support=h.support_email_address()))
 
-            pk = chromo.get('datastore_primary_key', [])
             choice_fields = {
                 f['datastore_id']:
                     'full' if f.get('excel_full_text_choices') else True
@@ -859,12 +875,13 @@ def _process_upload_file(lc: LocalCKAN,
 
             records = get_records(
                 rows,
-                [f for f in chromo['fields'] if f.get(
+                ([{'datastore_id': '_id', 'datastore_type': 'int'}]
+                    if method == 'update' else [])
+                + [f for f in chromo['fields'] if f.get(
                     'import_template_include', True) and not f.get(
                         'published_resource_computed_field')],
                 pk,
                 choice_fields)
-            method = 'upsert' if pk else 'insert'
             total_records += len(records)
             if not records:
                 continue
@@ -896,9 +913,12 @@ def _process_upload_file(lc: LocalCKAN,
                     else:
                         key = _('unknown')
                     raise RecombinantFieldError(key)
-                else:
+                elif 'records' in e.error_dict:
                     # type_ignore_reason: incomplete typing
                     pgerror = e.error_dict['records'][0]  # type: ignore
+                elif 'key' in e.error_dict:
+                    # type_ignore_reason: incomplete typing
+                    pgerror = e.error_dict['key'][0]  # type: ignore
                 if isinstance(pgerror, dict):
                     pgerror = '; '.join(
                         (h.recombinant_language_text(