From 22e4a1c6c192bf14c00f232116696b577f0febd8 Mon Sep 17 00:00:00 2001 From: Drew Leonard Date: Wed, 19 Nov 2025 15:00:08 +0000 Subject: [PATCH 1/4] Change processing of Fido results to create a new column for r_0 --- dkist/net/client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/dkist/net/client.py b/dkist/net/client.py index e22d3df30..b16397038 100644 --- a/dkist/net/client.py +++ b/dkist/net/client.py @@ -99,9 +99,13 @@ def _process_table(results: "DKISTQueryResponseTable") -> "DKISTQueryResponseTab results[colname] = u.Quantity(results[colname], unit=unit) if "Average Fried Parameter" in results.colnames: - r_none_values = np.array(results["Average Fried Parameter"] == None) - if r_none_values.any(): - results["Average Fried Parameter"][r_none_values] = np.nan + # We need to replace Nones with nans here for sorting purposes + # We also need to recreate the whole row so that it can have a numerical dtype + # Without this is doesn't sort properly and any nans up in strange places + old_r = results["Average Fried Parameter"] + results["Average Fried Parameter"] = [np.nan] * len(results) + notnone = results["Average Fried Parameter"] != None + results["Average Fried Parameter"][notnone] = old_r[notnone] if results and "Wavelength" not in results.colnames: results["Wavelength"] = u.Quantity([results["Wavelength Min"], results["Wavelength Max"]]).T From d2d42d5cf05c3e84b4d711966d9d7be0662a5d4e Mon Sep 17 00:00:00 2001 From: Drew Leonard Date: Wed, 19 Nov 2025 15:56:13 +0000 Subject: [PATCH 2/4] Changelog --- changelog/651.bugfix.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 changelog/651.bugfix.rst diff --git a/changelog/651.bugfix.rst b/changelog/651.bugfix.rst new file mode 100644 index 000000000..a74129c6a --- /dev/null +++ b/changelog/651.bugfix.rst @@ -0,0 +1 @@ +Fix sorting of UnifiedResponse table From 2b18305f684e047b63f3f5f83825d01119a4a203 Mon Sep 17 00:00:00 2001 From: Drew Leonard Date: Thu, 20 Nov 2025 08:58:40 +0000 Subject: [PATCH 3/4] Factor it out --- dkist/net/client.py | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/dkist/net/client.py b/dkist/net/client.py index b16397038..7db1616c8 100644 --- a/dkist/net/client.py +++ b/dkist/net/client.py @@ -32,6 +32,17 @@ __all__ = ["DKISTClient", "DKISTQueryResponseTable"] +def process_nones(results, key, replacement=np.nan): + # We need to replace Nones with nans here for sorting purposes + # We also need to recreate the whole row so that it can have a numerical dtype + # Without this is doesn't sort properly and any nans up in strange places + if key in results.colnames: + old_r = results[key] + results[key] = [replacement] * len(results) + notnone = results[key] != None + results[key][notnone] = old_r[notnone] + + class DKISTQueryResponseTable(QueryResponseTable): """ Results of a DKIST Dataset search. @@ -98,14 +109,7 @@ def _process_table(results: "DKISTQueryResponseTable") -> "DKISTQueryResponseTab results[colname][none_values] = np.nan results[colname] = u.Quantity(results[colname], unit=unit) - if "Average Fried Parameter" in results.colnames: - # We need to replace Nones with nans here for sorting purposes - # We also need to recreate the whole row so that it can have a numerical dtype - # Without this is doesn't sort properly and any nans up in strange places - old_r = results["Average Fried Parameter"] - results["Average Fried Parameter"] = [np.nan] * len(results) - notnone = results["Average Fried Parameter"] != None - results["Average Fried Parameter"][notnone] = old_r[notnone] + process_nones(results, "Average Fried Parameter") if results and "Wavelength" not in results.colnames: results["Wavelength"] = u.Quantity([results["Wavelength Min"], results["Wavelength Max"]]).T From 91621315652f71758a25a0040c5dd81a5802caae Mon Sep 17 00:00:00 2001 From: Drew Leonard Date: Thu, 20 Nov 2025 12:26:30 +0000 Subject: [PATCH 4/4] Test sorting r0 --- dkist/net/tests/test_client.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/dkist/net/tests/test_client.py b/dkist/net/tests/test_client.py index 55266767f..879766c0d 100644 --- a/dkist/net/tests/test_client.py +++ b/dkist/net/tests/test_client.py @@ -1,3 +1,4 @@ +import copy import json import hypothesis.strategies as st @@ -104,6 +105,20 @@ def example_api_response(): } +@pytest.fixture +def example_api_response_multiple_r0(example_api_response): + """ + A larger dummy API response with varied Fried parameter values to test sorting + """ + for _ in range(4): + example_api_response["searchResults"].append(copy.copy(example_api_response["searchResults"][0])) + example_api_response["searchResults"][1]["qualityAverageFriedParameter"] = 5 + example_api_response["searchResults"][3]["qualityAverageFriedParameter"] = 1 + example_api_response["searchResults"][4]["qualityAverageFriedParameter"] = 3 + + return example_api_response + + @pytest.fixture def expected_table_keys(): translated_keys = set(INVENTORY_KEY_MAP.values()) @@ -151,6 +166,13 @@ def test_query_response_from_results(empty_query_response, example_api_response, assert np.isnan(qr["Average Fried Parameter"][0]) +def test_sort_fried_parameter(example_api_response_multiple_r0): + qr = DKISTQueryResponseTable.from_results([example_api_response_multiple_r0], client=DKISTClient()) + qr.sort("Average Fried Parameter") + assert all(qr["Average Fried Parameter"][:3] == [1.0, 3.0, 5.0]) + assert all(np.isnan(qr["Average Fried Parameter"][3:])) + + def test_query_response_from_results_unknown_field(empty_query_response, example_api_response, expected_table_keys): """ This test asserts that if the API starts returning new fields we don't error, they get passed though verbatim.