DataStore PG Dump

changes/191.canada.feature

@@ -0,0 +1 @@
+Adds `sql` format to DataStore dump, which is a custom `pg_dump` export of the DataStore table.

ckanext/datastore/blueprint.py

@@ -28,6 +28,9 @@
     json_writer,
     xml_writer,
 )
+# (canada fork only): psql dump format
+import subprocess
+from ckanext.datastore.backend import DatastoreBackend
 # (canada fork only): filename to save stream to
 import re
 
@@ -42,7 +45,8 @@
 default = cast(ValidatorFactory, get_validator(u'default'))
 unicode_only = get_validator(u'unicode_only')
 
-DUMP_FORMATS = u'csv', u'tsv', u'json', u'xml'
+# (canada fork only): psql dump format
+DUMP_FORMATS = u'csv', u'tsv', u'json', u'xml', 'sql'
 PAGINATE_BY = 32000
 
 datastore = Blueprint(u'datastore', __name__)
@@ -116,6 +120,10 @@ def dump(resource_id: str):
                                             'limit': 0})
     except ObjectNotFound:
         abort(404, _('DataStore resource not found'))
+    # (canada fork only): handle 403
+    # TODO: upstream contrib!!
+    except NotAuthorized:
+        return abort(403)
 
     data, errors = dict_fns.validate(request.args.to_dict(), dump_schema())
     if errors:
@@ -162,6 +170,11 @@ def dump(resource_id: str):
         content_disposition = 'attachment; filename="{name}.xml"'.format(
                                     name=filename)  # (canada fork only): filename to save stream to
         content_type = b'text/xml; charset=utf-8'
+    # (canada fork only): psql dump format
+    elif fmt == 'sql':
+        content_disposition = 'attachment; filename="{name}.sql"'.format(
+                                    name=resource_id)
+        content_type = b'application/sql; charset=utf-8'
     else:
         abort(404, _('Unsupported format'))
 
@@ -292,6 +305,19 @@ def dump_to(
     elif fmt == 'xml':
         writer_factory = xml_writer
         records_format = 'objects'
+    # (canada fork only): psql dump format
+    elif fmt == 'sql':
+        datastore_uri = str(DatastoreBackend.get_active_backend()._get_write_engine().url)
+        # dump clean table, schema, and data.
+        # quote all for blank values.
+        cmd = subprocess.Popen(
+            ['pg_dump', datastore_uri, '--clean', '--if-exists', '--disable-triggers',
+             '--no-owner', '--quote-all-identifiers', '-t', resource_id],
+            stdout=subprocess.PIPE)
+        def stream_sql(process):
+            for c in iter(lambda: process.stdout.read(-1), b""):
+                yield c
+        return stream_sql(cmd)
     else:
         assert False, 'Unsupported format'
 

ckanext/datastore/templates/ajax_snippets/api_info.html

@@ -128,6 +128,22 @@ <h2 class="accordion-heading">
                 </div>
               </div>
             </div>
+            {# (canada fork only): psql dump format #}
+            <div class="accordion-item">
+              <div class="accordion-heading">
+                <button class="accordion-button" type="button" data-bs-toggle="collapse" data-bs-target="#collapse-pg-dump" aria-expanded="false" aria-controls="collapse-pg-dump"> {{ _('Database Table Export') }} &raquo;</button>
+              </div>
+              <div id="collapse-pg-dump" class="accordion-collapse collapse" aria-labelledby="collapse-pg-dump" data-bs-parent="#accordion2">
+                <div class="accordion-body">
+                  <p>{{ _('You can download the entire DataStore table in a pg_dump format to load into your own Postgres database for more advanced querying. This requires you to have write access to a running Postgres server and have the psql command line utility installed.') }}</p>
+                  <p><a target="_blank" href="{{ h.url_for('datastore.dump', resource_id=resource_id, format='sql') }}" class="btn btn-primary">{{ _('Download SQL File') }}</a></p>
+                  <p>{{ _('In your Command Prompt, PowerShell, or Terminal enter the command:') }}</p>
+                  <pre>
+psql -h <em>[{{ _('your database address') }}]</em> -p <em>[{{ _('your database port') }}]</em> -U <em>[{{ _('your database username') }}]</em> -d <em>[{{ _('your database name') }}]</em> -v ON_ERROR_STOP=0 -f {{ resource_id }}.sql
+                  </pre>
+                </div>
+              </div>
+            </div>
         </div>
 
       </div>