4242TextEmbeddingNormalizer = normalize_text .TextEmbeddingNormalizer
4343
4444
45+ class NumericalMissingValuePolicy (enum .StrEnum ):
46+ """What `auto_normalize` does when float features have NaN/±inf values.
47+
48+ Checked once on the feature statistics, when the normalizer is created.
49+ """
50+
51+ # Missing values are an error (no fallback).
52+ ERROR = "error"
53+ # TODO(dtl): Add other policies, like FILL, INDICATOR, and FILL_LEARNED.
54+
55+
4556@normalizer_registry .register
4657@dataclasses_json .dataclass_json
4758@dataclasses .dataclass (kw_only = True )
@@ -1155,6 +1166,9 @@ class AutoNormalizeConfig:
11551166 be normalized into dense embeddings using `TextEmbeddingNormalizer`.
11561167 text_embedder: Name of a built-in text embedder, a model handle/path, or a
11571168 `TextEmbedderConfig` used when `text_normalize` is True.
1169+ numerical_missing_value_policy: What to do when the statistics of a float
1170+ feature report NaN or ±inf values. Checked once, when the normalizer is
1171+ created; values at normalization time are not checked.
11581172 """
11591173
11601174 categorical_bytes_to_index : bool = True
@@ -1180,6 +1194,39 @@ class AutoNormalizeConfig:
11801194 calendar_features : tuple [CalendarFeature , ...] = DEFAULT_CALENDAR_FEATURES
11811195 text_normalize : bool = True
11821196 text_embedder : str | normalize_text .TextEmbedderConfig = "nnlm-en-dim128"
1197+ numerical_missing_value_policy : NumericalMissingValuePolicy = (
1198+ dataclasses .field (
1199+ default = NumericalMissingValuePolicy .ERROR ,
1200+ metadata = dataclasses_json .config (
1201+ decoder = NumericalMissingValuePolicy
1202+ ),
1203+ )
1204+ )
1205+
1206+
1207+ def _check_numerical_missing_values (
1208+ nodeset_name : str ,
1209+ feature_name : str ,
1210+ feature_stats : statistics_lib .FeatureStatistics ,
1211+ policy : NumericalMissingValuePolicy ,
1212+ ) -> None :
1213+ """Applies `policy` to the NaN/±inf counts of a numerical float feature."""
1214+ if policy == NumericalMissingValuePolicy .ERROR :
1215+ num_missing = (
1216+ feature_stats .num_nan
1217+ + feature_stats .num_posinf
1218+ + feature_stats .num_neginf
1219+ )
1220+ if num_missing :
1221+ raise ValueError (
1222+ f"Feature '{ feature_name } ' in node set '{ nodeset_name } ' has"
1223+ f" { feature_stats .num_nan } NaN, { feature_stats .num_posinf } +inf and"
1224+ f" { feature_stats .num_neginf } -inf values in its statistics, which"
1225+ " conflicts with numerical_missing_value_policy='error'. Please"
1226+ " clean or impute these values."
1227+ )
1228+ else :
1229+ raise ValueError (f"Unsupported numerical missing value policy: { policy !r} " )
11831230
11841231
11851232def auto_normalize (
@@ -1271,6 +1318,20 @@ def auto_normalize(
12711318 nodeset_normalizers = []
12721319 nodeset_stats = stats .node_sets [nodeset_name ]
12731320 for feature_name , feature_schema in nodeset_schema .features .items ():
1321+ # Only checks the statistics for NaN/±inf here; the normalizers are
1322+ # created below. This runs first so that raw (kept) features, e.g. a
1323+ # regression label, are checked too.
1324+ if (
1325+ feature_schema .semantic == schema_lib .FeatureSemantic .NUMERICAL
1326+ and feature_schema .format .is_float ()
1327+ and feature_name in nodeset_stats .features
1328+ ):
1329+ _check_numerical_missing_values (
1330+ nodeset_name ,
1331+ feature_name ,
1332+ nodeset_stats .features [feature_name ],
1333+ config .numerical_missing_value_policy ,
1334+ )
12741335 if (
12751336 (nodeset_name , feature_name ) in config .keep_raw_features
12761337 # Mask features should be preserved.
0 commit comments