Skip to content

Commit 2f8a413

Browse files
ksfikasclaude
andcommitted
CKAN load, part 2: the Load Box connects a pipeline to a CKAN dataset (search, link or upload, push now, change, disconnect), acting as the signed-in person
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 443d9c8 commit 2f8a413

6 files changed

Lines changed: 546 additions & 5 deletions

File tree

‎providerharvest_service/app.py‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
from providerharvest_service.ftp_relay_ingest import router as ftp_relay_ingest_router
3636
from providerharvest_service.load_api import router as load_api_router
3737
from providerharvest_service.ui_load import router as load_ui_router
38+
from providerharvest_service.ui_ckan import router as ckan_ui_router
3839
from providerharvest_service.ui_source_tables import router as source_tables_ui_router
3940
from providerharvest_service.push_ingest import router as push_ingest_router
4041
from providerharvest_service.ui_admin import router as admin_sources_ui_router
@@ -88,6 +89,7 @@ async def lifespan(app: FastAPI):
8889
app.include_router(oidc_login_router)
8990
app.include_router(load_api_router)
9091
app.include_router(load_ui_router)
92+
app.include_router(ckan_ui_router)
9193
app.include_router(push_ingest_router)
9294
app.include_router(ftp_relay_ingest_router)
9395

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
{% extends "base.html" %}
2+
{% import "_macros.html" as macros %}
3+
{% block title %}CKAN -- {{ source.title }} -- loader.data.gov.gr{% endblock %}
4+
{% block content %}
5+
<style>
6+
.form-head { display: flex; align-items: center; gap: 0.8rem; margin-bottom: 0.2rem; }
7+
.form-head h1 { margin: 0; }
8+
.form-head + .subtitle { margin: 0.5rem 0 1.1rem; }
9+
a.back-arrow { padding: 0.45rem 0.6rem; }
10+
.dataset-hit { display: block; width: 100%; text-align: left; background: var(--color-surface); color: var(--color-text); border: 1px solid var(--color-border); border-radius: 6px; padding: 0.5rem 0.8rem; margin: 0.25rem 0; cursor: pointer; font-weight: 400; }
11+
.dataset-hit:hover, .dataset-hit.chosen { border-color: var(--color-primary-mid); background: #f2f6fc; }
12+
.dataset-hit small { color: var(--color-gray-700); display: block; }
13+
.ckan-status { border: 1px solid var(--color-border); border-radius: var(--radius); padding: 0.8rem 1rem; margin: 0 0 1.2rem; max-width: 46rem; }
14+
</style>
15+
<div class="panel">
16+
<div class="form-head">
17+
<a class="btn btn-secondary btn-sm back-arrow" href="/load" aria-label="Back" title="Back">{{ macros.icon("arrow-left", 16) }}</a>
18+
<h1>CKAN</h1>
19+
</div>
20+
<p class="subtitle">{{ source.title }} <span class="name-slug">({{ source.name }})</span> / pipeline <strong>{{ pipeline.name }}</strong> -- add it to a dataset in data.gov.gr as a resource.</p>
21+
{% if flash %}<div class="flash flash-{{ flash_kind }}">{{ flash }}</div>{% endif %}
22+
{% if problem %}<div class="flash flash-error">{{ problem }}</div>{% endif %}
23+
24+
{% if target %}
25+
<div class="ckan-status">
26+
<div style="font-weight:600;">Connected to
27+
{% if dataset_url %}<a href="{{ dataset_url }}" target="_blank" rel="noopener">{{ target.dataset_title or target.dataset_name }}</a>{% else %}{{ target.dataset_title or target.dataset_name }}{% endif %}
28+
<span class="muted">-- {{ "a link to the published address" if target.mode == "link" else "an uploaded copy of the newest output" }}</span></div>
29+
<div style="margin-top:0.3rem;">
30+
{% if target.last_status == "ok" and target.resource_id %}<span class="badge badge-ok">{{ macros.icon("active", 13) }}<span>in CKAN</span></span> <span class="muted">pushed {{ target.last_pushed_at|short_utc }} UTC</span>
31+
{% elif target.last_status == "error" %}<span class="badge badge-error">{{ macros.icon("rejected", 13) }}<span>failed</span></span> <span class="muted">{{ target.last_error }}</span>
32+
{% else %}<span class="badge badge-pending"><span>not in CKAN</span></span> <span class="muted">{% if not published and target.mode == "link" %}it is unpublished, so its resource was taken out of CKAN; publish it again and push{% else %}not pushed yet{% endif %}</span>{% endif %}
33+
</div>
34+
<div style="display:flex; gap:0.5rem; margin-top:0.7rem;">
35+
<form method="post" action="{{ base }}/ckan/push" style="margin:0;"><button type="submit" class="btn-sm"{% if problem %} disabled{% endif %}>{{ macros.icon("push", 14) }} Push now</button></form>
36+
<form method="post" action="{{ base }}/ckan/disconnect" style="margin:0;" onsubmit="return confirm('Remove this resource from CKAN and disconnect the pipeline?');"><button type="submit" class="btn-secondary btn-sm"{% if problem %} disabled{% endif %}>{{ macros.icon("trash", 14) }} Disconnect</button></form>
37+
</div>
38+
</div>
39+
{% endif %}
40+
41+
<form method="post" action="{{ base }}/ckan" id="ckan-form" style="max-width:46rem;">
42+
<h2 style="font-size:1.05rem; margin:0 0 0.4rem;">{{ "Change" if target else "Connect" }}</h2>
43+
44+
<label>What goes into CKAN</label>
45+
<div class="segmented">
46+
<span><input type="radio" id="mode-link" name="mode" value="link"{% if not target or target.mode == "link" %} checked{% endif %}><label for="mode-link">A link to the published address</label></span>
47+
<span><input type="radio" id="mode-upload" name="mode" value="upload"{% if target and target.mode == "upload" %} checked{% endif %}{% if is_http %} disabled{% endif %}><label for="mode-upload">An uploaded copy of the newest output</label></span>
48+
</div>
49+
<p class="muted" id="mode-note" style="margin:0.4rem 0 0.8rem;">
50+
<span id="note-link"><strong>Link</strong>: the resource points to <code>{{ link }}</code>, which always answers with the current data. Nothing is copied; the pipeline must be published{% if not published %} (it is not yet){% endif %}.
51+
{% if not has_public_base %}<br><strong>LOADER_PUBLIC_BASE_URL is not set</strong>, so the address cannot be built.{% endif %}</span>
52+
<span id="note-upload" hidden><strong>Upload</strong>: the newest output is uploaded into CKAN's own storage, and again each time a new one is made.
53+
{% if is_http %}Not available for an API source: it stores nothing.{% elif newest %}Newest now: {{ newest.filename }}.{% else %}There is no output yet.{% endif %}</span>
54+
</p>
55+
56+
<label for="ds-q">Dataset <span class="req" style="color:var(--status-rejected);">*</span></label>
57+
<input type="hidden" id="dataset_id" name="dataset_id" value="{{ target.dataset_id if target else '' }}">
58+
<div id="ds-chosen" class="muted" style="margin:0.2rem 0;">{% if target %}Now: <strong>{{ target.dataset_title or target.dataset_name }}</strong>. Search to choose another.{% else %}Nothing chosen yet.{% endif %}</div>
59+
<input type="text" id="ds-q" placeholder="Search the datasets you can add to (title or words)" autocomplete="off"{% if problem %} disabled{% endif %}>
60+
<div id="ds-results" aria-live="polite"></div>
61+
62+
<label for="resource_name">Resource name <span class="req" style="color:var(--status-rejected);">*</span></label>
63+
<input type="text" id="resource_name" name="resource_name" maxlength="200" value="{{ target.resource_name if target else default_name }}">
64+
<label for="description">Description (optional)</label>
65+
<input type="text" id="description" name="description" maxlength="1000" value="{{ target.description if target else '' }}">
66+
67+
<div style="margin-top:1.2rem;"><button type="submit"{% if problem %} disabled{% endif %}>{{ macros.icon("push", 16) }} {{ "Save and push" if target else "Connect and push" }}</button></div>
68+
</form>
69+
</div>
70+
71+
<script>
72+
(function () {
73+
var results = document.getElementById('ds-results'), q = document.getElementById('ds-q'), chosen = document.getElementById('ds-chosen'), idField = document.getElementById('dataset_id');
74+
var timer = null;
75+
function render(list) {
76+
results.textContent = '';
77+
if (!list.length) { results.innerHTML = '<p class="muted">No dataset found that you may add to.</p>'; return; }
78+
list.forEach(function (d) {
79+
var b = document.createElement('button'); b.type = 'button'; b.className = 'dataset-hit';
80+
b.textContent = d.title; var s = document.createElement('small'); s.textContent = (d.organization ? d.organization + ' -- ' : '') + d.name; b.appendChild(s);
81+
b.addEventListener('click', function () {
82+
idField.value = d.id; chosen.innerHTML = ''; chosen.append('Chosen: '); var st = document.createElement('strong'); st.textContent = d.title; chosen.appendChild(st);
83+
results.textContent = '';
84+
});
85+
results.appendChild(b);
86+
});
87+
}
88+
function search() {
89+
fetch('/load/ckan/datasets?q=' + encodeURIComponent(q.value), {credentials: 'same-origin'}).then(function (r) { return r.json(); }).then(function (data) {
90+
if (data.error) { results.innerHTML = ''; var p = document.createElement('p'); p.className = 'muted'; p.textContent = data.error; results.appendChild(p); return; }
91+
render(data.datasets);
92+
}).catch(function () { results.textContent = 'The search failed.'; });
93+
}
94+
q.addEventListener('input', function () { clearTimeout(timer); timer = setTimeout(search, 300); });
95+
q.addEventListener('focus', function () { if (!results.children.length) search(); });
96+
function modeNote() {
97+
var upload = document.getElementById('mode-upload').checked;
98+
document.getElementById('note-link').hidden = upload; document.getElementById('note-upload').hidden = !upload;
99+
}
100+
document.querySelectorAll('input[name="mode"]').forEach(function (r) { r.addEventListener('change', modeNote); });
101+
modeNote();
102+
})();
103+
</script>
104+
{% endblock %}

‎providerharvest_service/templates/load/overview.html‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ <h1>Load Box</h1>
1717
<div class="empty-state">{{ macros.icon("inbox", 40) }}<p>No sources yet. Register one in the Extract Box first.</p></div>
1818
{% else %}
1919
<table>
20-
<thead><tr><th>Pipeline</th><th>What it answers with</th><th>Published address</th><th></th></tr></thead>
20+
<thead><tr><th>Pipeline</th><th>What it answers with</th><th>Published address</th><th>CKAN</th><th></th></tr></thead>
2121
<tbody>
2222
{% for r in rows %}
2323
<tr>
@@ -49,6 +49,17 @@ <h1>Load Box</h1>
4949
<div class="muted">would answer at <code>{{ r.path }}</code></div>
5050
{% else %}<span class="muted">-</span>{% endif %}
5151
</td>
52+
<td>
53+
{% if r.ckan %}
54+
{% if r.ckan.last_status == "error" %}<span class="badge badge-error">{{ macros.icon("rejected", 13) }}<span>failed</span></span>
55+
{% elif r.ckan.resource_id %}<span class="badge badge-ok">{{ macros.icon("active", 13) }}<span>in CKAN</span></span>
56+
{% else %}<span class="badge badge-pending"><span>not in CKAN</span></span>{% endif %}
57+
<div class="muted" style="font-size:0.85rem;">{{ r.ckan.dataset_title or r.ckan.dataset_name }} &middot; {{ r.ckan.mode }}</div>
58+
<a href="{{ r.base }}/ckan">Manage</a>
59+
{% elif r.pipeline and ckan_ready %}<a href="{{ r.base }}/ckan">Push to CKAN</a>
60+
{% elif r.pipeline %}<span class="muted" title="CKAN is not set up on this deployment">-</span>
61+
{% else %}<span class="muted">-</span>{% endif %}
62+
</td>
5263
<td>
5364
{% if r.pipeline or r.api %}
5465
<form method="post" action="{{ r.base }}/api" style="margin:0;"{% if r.published %} onsubmit="return confirm('Stop publishing {{ r.path }}? Anyone using its address will get a 404.');"{% endif %}>

0 commit comments

Comments
 (0)