Skip to content

Commit a3e0fdb

Browse files
graph-learning-teamcopybara-github
authored andcommitted
Explicit missing-value policy with a clear error on NaN/±inf
PiperOrigin-RevId: 1013779953
1 parent 6f4e048 commit a3e0fdb

2 files changed

Lines changed: 143 additions & 0 deletions

File tree

‎dgf/src/transform/normalize.py‎

Lines changed: 61 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,17 @@
4242
TextEmbeddingNormalizer = normalize_text.TextEmbeddingNormalizer
4343

4444

45+
class NumericalMissingValuePolicy(enum.StrEnum):
46+
"""What `auto_normalize` does when float features have NaN/±inf values.
47+
48+
Checked once on the feature statistics, when the normalizer is created.
49+
"""
50+
51+
# Missing values are an error (no fallback).
52+
ERROR = "error"
53+
# TODO(dtl): Add other policies, like FILL, INDICATOR, and FILL_LEARNED.
54+
55+
4556
@normalizer_registry.register
4657
@dataclasses_json.dataclass_json
4758
@dataclasses.dataclass(kw_only=True)
@@ -1155,6 +1166,9 @@ class AutoNormalizeConfig:
11551166
be normalized into dense embeddings using `TextEmbeddingNormalizer`.
11561167
text_embedder: Name of a built-in text embedder, a model handle/path, or a
11571168
`TextEmbedderConfig` used when `text_normalize` is True.
1169+
numerical_missing_value_policy: What to do when the statistics of a float
1170+
feature report NaN or ±inf values. Checked once, when the normalizer is
1171+
created; values at normalization time are not checked.
11581172
"""
11591173

11601174
categorical_bytes_to_index: bool = True
@@ -1180,6 +1194,39 @@ class AutoNormalizeConfig:
11801194
calendar_features: tuple[CalendarFeature, ...] = DEFAULT_CALENDAR_FEATURES
11811195
text_normalize: bool = True
11821196
text_embedder: str | normalize_text.TextEmbedderConfig = "nnlm-en-dim128"
1197+
numerical_missing_value_policy: NumericalMissingValuePolicy = (
1198+
dataclasses.field(
1199+
default=NumericalMissingValuePolicy.ERROR,
1200+
metadata=dataclasses_json.config(
1201+
decoder=NumericalMissingValuePolicy
1202+
),
1203+
)
1204+
)
1205+
1206+
1207+
def _check_numerical_missing_values(
1208+
nodeset_name: str,
1209+
feature_name: str,
1210+
feature_stats: statistics_lib.FeatureStatistics,
1211+
policy: NumericalMissingValuePolicy,
1212+
) -> None:
1213+
"""Applies `policy` to the NaN/±inf counts of a numerical float feature."""
1214+
if policy == NumericalMissingValuePolicy.ERROR:
1215+
num_missing = (
1216+
feature_stats.num_nan
1217+
+ feature_stats.num_posinf
1218+
+ feature_stats.num_neginf
1219+
)
1220+
if num_missing:
1221+
raise ValueError(
1222+
f"Feature '{feature_name}' in node set '{nodeset_name}' has"
1223+
f" {feature_stats.num_nan} NaN, {feature_stats.num_posinf} +inf and"
1224+
f" {feature_stats.num_neginf} -inf values in its statistics, which"
1225+
" conflicts with numerical_missing_value_policy='error'. Please"
1226+
" clean or impute these values."
1227+
)
1228+
else:
1229+
raise ValueError(f"Unsupported numerical missing value policy: {policy!r}")
11831230

11841231

11851232
def auto_normalize(
@@ -1271,6 +1318,20 @@ def auto_normalize(
12711318
nodeset_normalizers = []
12721319
nodeset_stats = stats.node_sets[nodeset_name]
12731320
for feature_name, feature_schema in nodeset_schema.features.items():
1321+
# Only checks the statistics for NaN/±inf here; the normalizers are
1322+
# created below. This runs first so that raw (kept) features, e.g. a
1323+
# regression label, are checked too.
1324+
if (
1325+
feature_schema.semantic == schema_lib.FeatureSemantic.NUMERICAL
1326+
and feature_schema.format.is_float()
1327+
and feature_name in nodeset_stats.features
1328+
):
1329+
_check_numerical_missing_values(
1330+
nodeset_name,
1331+
feature_name,
1332+
nodeset_stats.features[feature_name],
1333+
config.numerical_missing_value_policy,
1334+
)
12741335
if (
12751336
(nodeset_name, feature_name) in config.keep_raw_features
12761337
# Mask features should be preserved.

‎dgf/src/transform/normalize_test.py‎

Lines changed: 82 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1920,6 +1920,88 @@ def test_auto_normalize_mask(self):
19201920
)
19211921

19221922

1923+
def _missing_value_schema_and_stats(
1924+
**feature_stats_kwargs,
1925+
) -> tuple[schema_lib.GraphSchema, statistics_lib.GraphFeatureStatistics]:
1926+
schema = schema_lib.GraphSchema(
1927+
node_sets={
1928+
"nodes": schema_lib.NodeSchema(
1929+
features={
1930+
"f": schema_lib.FeatureSchema(
1931+
format=schema_lib.FeatureFormat.FLOAT_32,
1932+
semantic=schema_lib.FeatureSemantic.NUMERICAL,
1933+
shape=(),
1934+
)
1935+
}
1936+
)
1937+
},
1938+
edge_sets={},
1939+
)
1940+
stats = statistics_lib.GraphFeatureStatistics(
1941+
node_sets={
1942+
"nodes": statistics_lib.FeatureSetStatistics(
1943+
features={
1944+
"f": statistics_lib.FeatureStatistics(
1945+
count=10,
1946+
minimum=0.0,
1947+
maximum=1.0,
1948+
mean=0.5,
1949+
quantiles=[0.0, 0.25, 0.5, 0.75, 1.0],
1950+
**feature_stats_kwargs,
1951+
)
1952+
}
1953+
)
1954+
}
1955+
)
1956+
return schema, stats
1957+
1958+
1959+
class NumericalMissingValuePolicyTest(parameterized.TestCase):
1960+
1961+
def test_default_is_error(self):
1962+
self.assertEqual(
1963+
normalize_lib.AutoNormalizeConfig().numerical_missing_value_policy,
1964+
normalize_lib.NumericalMissingValuePolicy.ERROR,
1965+
)
1966+
1967+
def test_clean_stats_build(self):
1968+
schema, stats = _missing_value_schema_and_stats()
1969+
normalizer = normalize_lib.auto_normalize(schema, stats)
1970+
self.assertIn(
1971+
"f_SOFT_QUANTILE",
1972+
normalizer.output_schema().node_sets["nodes"].features,
1973+
)
1974+
1975+
@parameterized.parameters(
1976+
dict(num_nan=1), dict(num_posinf=2), dict(num_neginf=3)
1977+
)
1978+
def test_non_finite_stats_raise(self, **kwargs):
1979+
schema, stats = _missing_value_schema_and_stats(**kwargs)
1980+
with self.assertRaisesRegex(
1981+
ValueError,
1982+
r"Feature 'f' in node set 'nodes' has .*"
1983+
r"numerical_missing_value_policy='error'",
1984+
):
1985+
normalize_lib.auto_normalize(schema, stats)
1986+
1987+
def test_kept_raw_feature_is_checked(self):
1988+
schema, stats = _missing_value_schema_and_stats(num_nan=1)
1989+
config = normalize_lib.AutoNormalizeConfig(
1990+
keep_raw_features={("nodes", "f")}
1991+
)
1992+
with self.assertRaisesRegex(ValueError, "numerical_missing_value_policy"):
1993+
normalize_lib.auto_normalize(schema, stats, config)
1994+
1995+
def test_config_json_round_trip(self):
1996+
config = normalize_lib.AutoNormalizeConfig.from_json( # pyrefly: ignore[missing-attribute]
1997+
normalize_lib.AutoNormalizeConfig().to_json() # pyrefly: ignore[missing-attribute]
1998+
)
1999+
self.assertIs(
2000+
config.numerical_missing_value_policy,
2001+
normalize_lib.NumericalMissingValuePolicy.ERROR,
2002+
)
2003+
2004+
19232005
class TimedeltaNormalizerTest(parameterized.TestCase):
19242006

19252007
def test_output_schema(self):

0 commit comments

Comments
 (0)