Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions dgf/src/transform/BUILD
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@ py_library(
srcs = ["table_2_graph.py"],
type_checking = True,
deps = [
":conform",
"//dgf/src/analyse:text_detection",
"//dgf/src/data:in_memory_graph",
"//dgf/src/data:schema",
Expand Down
53 changes: 48 additions & 5 deletions dgf/src/transform/table_2_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,21 +14,43 @@

"""Utility to convert tables to In-Memory Graphs and their schemas."""

from typing import Any

from dgf.src.analyse import text_detection
from dgf.src.data import in_memory_graph as in_memory_graph_lib
from dgf.src.data import schema as schema_lib
from dgf.src.io import feature_format
from dgf.src.transform import conform as conform_lib
import numpy as np
import pandas as pd


# `pd.api.types.infer_dtype` results (missing values skipped) of object
# columns that hold only numbers and missing values.
_NULLABLE_NUMERIC_INFERRED_TYPES = (
"integer",
"floating",
"mixed-integer-float",
)


def _infer_feature_schema(
name: str, arr: np.ndarray, detect_semantic: bool = True
) -> schema_lib.FeatureSchema:
"""Infers the FeatureSchema from a numpy array."""
fmt = feature_format.NP_DTYPE_TO_FEATURE_FORMAT.get(
arr.dtype.type, schema_lib.FeatureFormat.BYTES
)
if (
arr.dtype == np.object_
and pd.api.types.infer_dtype(arr, skipna=True)
in _NULLABLE_NUMERIC_INFERRED_TYPES
):
# Numbers with missing values: integers cannot hold NaN, so use a float
# format. FLOAT_64 keeps integers exact up to 2**53 (FLOAT_32 only up to
# 2**24, which would corrupt e.g. IDs, counts or timestamps).
fmt = schema_lib.FeatureFormat.FLOAT_64
else:
fmt = feature_format.NP_DTYPE_TO_FEATURE_FORMAT.get(
arr.dtype.type, schema_lib.FeatureFormat.BYTES
)
shape = arr.shape[1:] if arr.ndim > 1 else ()

semantic = schema_lib.FeatureSemantic.UNKNOWN
Expand Down Expand Up @@ -57,6 +79,19 @@ def _infer_feature_schema(
)


def _column_to_numpy(values: Any) -> np.ndarray:
"""Converts a table column to a numpy array, without choosing a format."""
if isinstance(values, pd.Series):
# Nullable pandas columns (e.g. Int64, Float64, string) hold pd.NA, which
# numpy cannot store; they become object arrays with None.
if values.hasnans and isinstance(
values.dtype, pd.api.extensions.ExtensionDtype
):
return values.to_numpy(dtype=object, na_value=None)
return values.to_numpy()
return np.asarray(values)


def table2graph(
table: dict[str, np.ndarray] | pd.DataFrame,
nodeset_name: str = "nodes",
Expand All @@ -83,6 +118,10 @@ def table2graph(
detect_semantic: Whether to automatically infer the semantic of the
features.

Missing values: numeric columns with missing values (None, pd.NA, NaN) get
the FLOAT_64 format and NaN for missing. Missing values in string/bytes
columns are replaced with `b""`.

Returns:
A tuple of (InMemoryGraph, GraphSchema).

Expand All @@ -97,9 +136,9 @@ def table2graph(
)

if isinstance(table, pd.DataFrame):
features = {col: table[col].to_numpy() for col in table.columns}
features = {col: _column_to_numpy(table[col]) for col in table.columns}
else:
features = {k: np.asarray(v) for k, v in table.items()}
features = {k: _column_to_numpy(v) for k, v in table.items()}

if not features:
raise ValueError("Input table is empty")
Expand Down Expand Up @@ -135,4 +174,8 @@ def table2graph(
edge_sets={},
)

# Convert the columns to the inferred formats (e.g. missing -> NaN).
graph = conform_lib.conform_graph(
graph, schema, replace_none_bytes=b"", verbose=False
)
return graph, schema
64 changes: 64 additions & 0 deletions dgf/src/transform/table_2_graph_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,58 @@ def test_dataframe_input(self):
schema_lib.FeatureSemantic.NUMERICAL,
)

def test_dataframe_nullable_numeric_columns(self):
df = pd.DataFrame({
"nullable_float": pd.array([1.5, None, 3.0], dtype="Float64"),
"nullable_int_missing": pd.array([1, None, 3], dtype="Int64"),
"nullable_int_full": pd.array([1, 2, 3], dtype="Int64"),
"object_float": np.array([1.0, None, 2.0], dtype=np.object_),
"timestamp": np.array(
[1_760_000_060, None, 1_760_000_001], dtype=np.object_
),
})
graph, schema = table_2_graph_lib.table2graph(df)
features = graph.node_sets["nodes"].features
nodeset_schema = schema.node_sets["nodes"]

for name in (
"nullable_float",
"nullable_int_missing",
"object_float",
"timestamp",
):
self.assertEqual(features[name].dtype, np.float64, name)
self.assertTrue(np.isnan(features[name][1]), name)
self.assertEqual(
nodeset_schema.features[name].format,
schema_lib.FeatureFormat.FLOAT_64,
name,
)
self.assertEqual(
nodeset_schema.features[name].semantic,
schema_lib.FeatureSemantic.NUMERICAL,
name,
)
# Timestamps must stay exact (float32 would quantize them to 128 s).
self.assertEqual(features["timestamp"][0], 1_760_000_060.0)

self.assertEqual(features["nullable_int_full"].dtype, np.int64)
self.assertEqual(
nodeset_schema.features["nullable_int_full"].format,
schema_lib.FeatureFormat.INTEGER_64,
)

def test_dict_object_numeric_column_with_none(self):
table = {"a": np.array([1.0, None, 2.0], dtype=np.object_)}
graph, schema = table_2_graph_lib.table2graph(table)
values = graph.node_sets["nodes"].features["a"]
self.assertEqual(values.dtype, np.float64)
np.testing.assert_array_equal(values, [1.0, np.nan, 2.0])
self.assertEqual(
schema.node_sets["nodes"].features["a"].format,
schema_lib.FeatureFormat.FLOAT_64,
)

def test_detect_semantic_false(self):
table = {
"feature_a": np.array([1, 2, 3]),
Expand Down Expand Up @@ -218,6 +270,18 @@ def test_mismatched_lengths(self):
):
table_2_graph_lib.table2graph(table)

def test_missing_strings_become_empty_bytes(self):
table = pd.DataFrame({"s": pd.Series(["a", None, "c"], dtype="string")})
graph, schema = table_2_graph_lib.table2graph(table)
self.assertEqual(
schema.node_sets["nodes"].features["s"].format,
schema_lib.FeatureFormat.BYTES,
)
np.testing.assert_array_equal(
graph.node_sets["nodes"].features["s"],
np.array([b"a", b"", b"c"], dtype=np.bytes_),
)


if __name__ == "__main__":
absltest.main()
Loading