Skip to content

Commit bbd41df

Browse files
ksfikasclaude
andcommitted
Transform Box: Greek letters in field names, and Latin names suggested for table columns
An index with Greek or symbol-laden headers (for example KAT/SI, X(WGS84,ZONE 34)) now gets a usable form: names may use Greek letters, and each column's "call it" box is pre-filled with a transliterated, safe Latin name (kat_si, ch_WGS84_ZONE_34) that can be changed. Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
1 parent 3c5866c commit bbd41df

6 files changed

Lines changed: 81 additions & 39 deletions

File tree

‎.github/workflows/index-look.yml‎

Lines changed: 0 additions & 30 deletions
This file was deleted.

‎providerharvest_service/templates/transform/step.html‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ <h1>{{ "Edit" if number else "Add" }}: {{ title }}</h1>
8989
<tr>
9090
<td><input type="checkbox" name="path" value="{{ f.path }}" aria-label="Keep {{ f.path }}"{% if f.path in r.selected or (not r.selected and not number) %} checked{% endif %}></td>
9191
<td><code>{{ f.path }}</code>{% if f.repeats %} <span class="muted" title="Occurs more than once in a record">&#8635;</span>{% endif %}</td>
92-
<td><input type="text" name="name::{{ f.path }}" value="{{ r.selected.get(f.path, '') }}" placeholder="(automatic)" maxlength="60"></td>
92+
<td><input type="text" name="name::{{ f.path }}" value="{{ r.selected.get(f.path, '') }}" placeholder="{{ suggest_name(f.path, None, r.kind in ('csv_records', 'xlsx_records')) }}" maxlength="60"></td>
9393
<td class="muted">{{ f.example or "" }}</td>
9494
</tr>
9595
{% endfor %}
@@ -128,7 +128,7 @@ <h1>{{ "Edit" if number else "Add" }}: {{ title }}</h1>
128128
{% for c in columns %}
129129
{% set added = (e.get('add') or []) | selectattr('column', 'equalto', c) | list %}
130130
<tr><td><input type="checkbox" name="add" value="{{ c }}"{% if added %} checked{% endif %}></td><td><code>{{ c }}</code></td>
131-
<td><input type="text" name="as::{{ c }}" value="{{ added[0]['as'] if added else '' }}" placeholder="call it {{ c }}" maxlength="60"></td></tr>
131+
<td><input type="text" name="as::{{ c }}" value="{{ added[0]['as'] if added else suggest_name(c, None, true) }}" maxlength="60" aria-label="Name for {{ c }}"></td></tr>
132132
{% endfor %}
133133
</tbody></table>
134134
</div>

‎providerharvest_service/transform_engine.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@
4141
MAX_FIELDS = 200
4242
MAX_LOOKUP_KEYS_SHOWN = 20
4343
PREVIEW_RECORDS = 50
44-
_NAME = re.compile(r"^[A-Za-z_][A-Za-z0-9_]{0,63}$")
44+
_NAME = re.compile(r"^[^\W\d]\w{0,63}$") # letters (any script, Greek included), digits and underscores; not starting with a digit
4545
_PATH = re.compile(r"^(?:[^/@\s]+(?:/[^/@\s]+)*)?(?:@[A-Za-z_][\w.-]*)?$")
4646

4747
Record = dict[str, Any]
@@ -66,7 +66,7 @@ def _text(value: object) -> str | None:
6666

6767
def _check_name(name: object, what: str) -> str:
6868
if not isinstance(name, str) or not _NAME.match(name):
69-
raise TransformError("%s must be letters, digits and underscores, starting with a letter (got %r)." % (what, name))
69+
raise TransformError("%s must be letters, digits and underscores, starting with a letter -- no spaces or symbols (got %r)." % (what, name))
7070
return name
7171

7272

‎providerharvest_service/transform_forms.py‎

Lines changed: 23 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
from __future__ import annotations
1010

11+
import re
1112
from typing import Any, Mapping
1213

1314
from providerharvest_service import transform_engine as te
@@ -84,10 +85,27 @@ def summarize(step: dict) -> str:
8485

8586
# -- building steps from forms ---------------------------------------------------------------------------------------------
8687

87-
def suggest_name(path: str, taken: set[str]) -> str:
88-
"""A field name for a path: its last part, made a legal name, made unique."""
89-
base = path.rsplit("@", 1)[-1] if "@" in path else path.rsplit("/", 1)[-1]
90-
base = "".join(ch if (ch.isalnum() or ch == "_") else "_" for ch in base).strip("_") or "field"
88+
_GREEK = {
89+
"α": "a", "β": "v", "γ": "g", "δ": "d", "ε": "e", "ζ": "z", "η": "i", "θ": "th", "ι": "i", "κ": "k", "λ": "l", "μ": "m",
90+
"ν": "n", "ξ": "x", "ο": "o", "π": "p", "ρ": "r", "σ": "s", "ς": "s", "τ": "t", "υ": "y", "φ": "f", "χ": "ch", "ψ": "ps", "ω": "o",
91+
}
92+
93+
94+
def latinize(text: str) -> str:
95+
"""Greek letters as Latin ones (accents dropped, lower case), anything else as it was -- for suggesting field names."""
96+
import unicodedata
97+
98+
plain = "".join(ch for ch in unicodedata.normalize("NFD", text) if not unicodedata.combining(ch))
99+
return "".join(_GREEK.get(ch.lower(), ch) if ch.lower() in _GREEK else ch for ch in plain)
100+
101+
102+
def suggest_name(path: str, taken: set[str] | None = None, plain: bool = False) -> str:
103+
"""A legal field name for a path or column: its last part, Greek turned into Latin letters, symbols into
104+
underscores, made unique among ``taken``."""
105+
taken = taken or set()
106+
# ``plain``: a table column, whose whole text is the name (a "/" in it is just a character, not a path)
107+
base = path if plain else (path.rsplit("@", 1)[-1] if "@" in path else path.rsplit("/", 1)[-1])
108+
base = re.sub(r"\W+", "_", latinize(base)).strip("_") or "field"
91109
if not (base[0].isalpha() or base[0] == "_"):
92110
base = "f_" + base
93111
base, name, n = base[:60], base[:60], 2
@@ -105,7 +123,7 @@ def build_reader(kind: str, form: Mapping) -> dict:
105123
explode_path = _one(form, "explode")
106124
taken: set[str] = set()
107125
for path in chosen:
108-
name = _one(form, "name::" + path) or suggest_name(path, taken)
126+
name = _one(form, "name::" + path) or suggest_name(path, taken, plain=kind in TABULAR)
109127
taken.add(name)
110128
if kind in TABULAR:
111129
fields.append({"name": name, "path": path})

‎providerharvest_service/ui_transform.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@
4040
templates.env.filters["short_utc"] = lambda value: value.strftime("%m-%d %H:%M") if value else ""
4141
templates.env.filters["step_label"] = tf.label_of
4242
templates.env.filters["step_summary"] = tf.summarize
43+
templates.env.globals["suggest_name"] = tf.suggest_name
4344

4445
_TABLE_NAME = re.compile(r"^[a-z][a-z0-9_]{1,40}$")
4546
_SECTIONS = {"read": "Read the file", "lookup": "Add columns from an index table", "filter": "Keep only matching records",

‎tests/test_transform_ui.py‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -414,3 +414,56 @@ def test_a_csv_source_is_built_with_the_same_pages(client, source, _isolated_db)
414414
assert _steps(_isolated_db, source["id"])[0]["fields"][0] == {"name": "station", "path": "Station ID"}
415415
page = client.post("/transform/sources/%s/preview" % source["id"]).text
416416
assert ">A<" in page and ">7<" in page
417+
418+
419+
# -- names: Greek letters are fine, and Latin names are suggested --------------------------------------------------------------
420+
421+
@pytest.mark.parametrize("column, expected", [
422+
("MS", "MS"), ("ΟΝΟΜΑ", "onoma"), ("ΔΗΜΟΣ", "dimos"), ("ΚΑΤ/ΣΗ", "kat_si"), ("ΠΕΡΙΓΡΑΦΗ ΘΕΣΗΣ", "perigrafi_thesis"),
423+
("Χ(WGS84,ZONE 34)", "ch_WGS84_ZONE_34"), ("Station ID", "Station_ID"), ("1st", "f_1st"), ("", "field"),
424+
])
425+
def test_a_legal_name_is_suggested_for_any_column_header(column, expected):
426+
from providerharvest_service import transform_forms as tf
427+
assert tf.suggest_name(column, None, True) == expected
428+
429+
430+
def test_a_path_is_named_by_its_last_part_and_duplicates_get_a_number():
431+
from providerharvest_service import transform_forms as tf
432+
assert tf.suggest_name("data/value@count") == "count" and tf.suggest_name("identifier/ident") == "ident"
433+
assert tf.suggest_name("a/id", {"id"}) == "id_2"
434+
435+
436+
def test_field_names_may_use_greek_letters_but_not_spaces_or_symbols():
437+
from providerharvest_service import transform_engine as te
438+
for ok in ("όνομα", "δήμος_2", "name", "_x", "ΟΔΟΣ"):
439+
assert te._check_name(ok, "n") == ok
440+
for bad in ("ΚΑΤ/ΣΗ", "two words", "1abc", "a-b", "x(y)", "", "a" * 70):
441+
with pytest.raises(te.TransformError):
442+
te._check_name(bad, "n")
443+
444+
445+
GREEK_INDEX = "MS,ΟΝΟΜΑ,ΔΗΜΟΣ,ΚΑΤ/ΣΗ,Χ(WGS84,ZONE 34)\nMS812,Κηφισίας,ΜΑΡΟΥΣΙ,ΝΔ,\"1,234\"\nMS810,Μεσογείων,ΠΑΠΑΓΟΥ,ΒΑ,\"5,678\"\n".encode("utf-8")
446+
447+
448+
def test_an_index_with_greek_and_symbol_headers_gets_a_usable_form_with_suggested_names_and_works(client, source, _isolated_db):
449+
client.post("/transform/tables", data={"name": "stations"}, files={"file": ("s.csv", GREEK_INDEX, "text/csv")})
450+
_upload_data(client, source)
451+
_build(client, source, READER_FORM)
452+
page = client.get("/transform/sources/%s/steps/new?type=lookup" % source["id"]).text
453+
for suggested in ('value="onoma"', 'value="dimos"', 'value="kat_si"', 'value="ch_WGS84_ZONE_34"'):
454+
assert suggested in page, suggested
455+
form = {"kind": "lookup", "table": "stations", "key": "id", "table_key": "MS", "add": ["ΟΝΟΜΑ", "ΔΗΜΟΣ", "ΚΑΤ/ΣΗ"],
456+
"as::ΟΝΟΜΑ": "onoma", "as::ΔΗΜΟΣ": "dimos", "as::ΚΑΤ/ΣΗ": "kat_si", "on_missing": "keep"}
457+
assert "added" in _flash(_go(client, source, "steps/new", data=form))
458+
preview = client.post("/transform/sources/%s/preview" % source["id"]).text
459+
assert "Κηφισίας" in preview and "ΜΑΡΟΥΣΙ" in preview and "kat_si" in preview.lower()
460+
461+
462+
def test_leaving_a_name_as_the_suggestion_or_using_greek_ones_both_work(client, source, _isolated_db):
463+
client.post("/transform/tables", data={"name": "stations"}, files={"file": ("s.csv", GREEK_INDEX, "text/csv")})
464+
_upload_data(client, source)
465+
_build(client, source, READER_FORM)
466+
form = {"kind": "lookup", "table": "stations", "key": "id", "table_key": "MS", "add": ["ΟΝΟΜΑ"], "as::ΟΝΟΜΑ": "όνομα"}
467+
assert "added" in _flash(_go(client, source, "steps/new", data=form))
468+
bad = {"kind": "lookup", "table": "stations", "key": "id", "table_key": "MS", "add": ["ΚΑΤ/ΣΗ"], "as::ΚΑΤ/ΣΗ": "ΚΑΤ/ΣΗ"}
469+
assert "no spaces or symbols" in _go(client, source, "steps/new", data=bad).text

0 commit comments

Comments
 (0)