diff --git a/kdp/features.py b/kdp/features.py
index a2237bd..e41e3db 100644
--- a/kdp/features.py
+++ b/kdp/features.py
@@ -387,6 +387,64 @@ def __init__(
self.kwargs = kwargs
+# Where each calendar component appears in a format string, in both the
+# strftime spelling and the plain one.
+_YEAR_TOKENS = ("%Y", "%y", "YYYY", "YY")
+_MONTH_TOKENS = ("%m", "%b", "%B", "MM")
+_DAY_TOKENS = ("%d", "DD")
+
+# Options a `DateFeature` is given that nothing reads. The date encoding is
+# fixed -- year, month and day, each as a sine and cosine pair -- so there is no
+# component to select.
+_UNREAD_DATE_OPTIONS = ("output_format", "extract")
+
+
+def _first_index(text: str, tokens: tuple[str, ...]) -> int | None:
+ """Where the earliest of `tokens` appears in `text`, if any.
+
+ Args:
+ text: The format string to search.
+ tokens: The spellings of one calendar component.
+
+ Returns:
+ The index of the earliest match, or None when none appears.
+ """
+ found = [text.index(token) for token in tokens if token in text]
+ return min(found) if found else None
+
+
+def _check_date_format(name: str, date_format: str) -> None:
+ """Refuse a format the parser cannot read, while the caller can still fix it.
+
+ The parser reads year, then month, then day, with `-` or `/` between them
+ and an optional time after. It does not read the format string, so a
+ day-first format was accepted here, ignored, and then met at the first batch
+ as an assertion failure inside a graph error -- long after the line that
+ caused it.
+
+ Args:
+ name: The feature's name, for the message.
+ date_format: The format the caller gave.
+
+ Raises:
+ ValueError: If the format does not put year, then month, then day.
+ """
+ year = _first_index(date_format, _YEAR_TOKENS)
+ month = _first_index(date_format, _MONTH_TOKENS)
+ day = _first_index(date_format, _DAY_TOKENS)
+
+ if None not in (year, month, day) and year < month < day:
+ return
+
+ raise ValueError(
+ f"{name}: format={date_format!r} is not a format this library reads. "
+ f"Dates are parsed as year, then month, then day, separated by '-' or "
+ f"'/', optionally followed by a time -- '%Y-%m-%d', '%Y/%m/%d' and "
+ f"'%Y-%m-%d %H:%M:%S' among them. Convert the column to one of those "
+ f"before passing it in.",
+ )
+
+
class DateFeature(Feature):
"""TextFeature with dynamic kwargs passing."""
@@ -405,6 +463,25 @@ def __init__(
"""
super().__init__(name, feature_type, **kwargs)
self.dtype = tf.string
+
+ # `date_format` is the spelling most callers reach for, and the one the
+ # layer's own argument is called; the pipeline reads `format`. It was
+ # accepted under either name and honoured under neither.
+ if "date_format" in kwargs and "format" not in kwargs:
+ kwargs["format"] = kwargs["date_format"]
+
+ if kwargs.get("format"):
+ _check_date_format(name, kwargs["format"])
+
+ for option in _UNREAD_DATE_OPTIONS:
+ if option in kwargs:
+ logger.warning(
+ f"{name}: {option}={kwargs[option]!r} is accepted and not "
+ f"used. A date feature always produces the same eight "
+ f"columns -- year, month and day, each as a sine and a "
+ f"cosine -- plus four more when add_season=True.",
+ )
+
self.kwargs = kwargs
diff --git a/kdp/layers/date_parsing_layer.py b/kdp/layers/date_parsing_layer.py
index 1a57896..c80baf8 100644
--- a/kdp/layers/date_parsing_layer.py
+++ b/kdp/layers/date_parsing_layer.py
@@ -8,9 +8,10 @@ def __init__(self, date_format: str = "YYYY-MM-DD", **kwargs) -> None:
"""Initializing DateParsingLayer.
Args:
- date_format (str): Accepted and not used. Both supported
- spellings, YYYY-MM-DD and YYYY/MM/DD, are parsed whatever this
- says: the separator is normalised before the date is read.
+ date_format (str): Accepted and not used. Every supported
+ spelling -- YYYY-MM-DD and YYYY/MM/DD, each optionally followed
+ by a time -- is parsed whatever this says: the separator is
+ normalised and any time dropped before the date is read.
kwargs (dict): other params to pass to the class.
"""
super().__init__(**kwargs)
@@ -29,17 +30,38 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor:
"""
def parse_date(date_str: str) -> tf.Tensor:
- # Handle missing/invalid dates
+ # Handle missing/invalid dates. A time part is allowed after the
+ # date -- "2021-06-15 13:45:00" and its ISO "T" spelling are what a
+ # timestamp column holds, and rejecting them left a very ordinary
+ # kind of column with no way through this layer at all: the model
+ # built, then died on the first batch inside a graph error deep
+ # enough to bury the reason.
is_valid = tf.strings.regex_full_match(
date_str,
- r"^\d{1,4}[-/]\d{1,2}[-/]\d{1,2}$",
+ r"^\d{1,4}[-/]\d{1,2}[-/]\d{1,2}([ T].*)?$",
)
- tf.debugging.assert_equal(
+ # `tf.Assert` rather than `assert_equal`, so the value that failed
+ # can be printed alongside the reason. The old assertion surfaced
+ # as a graph error naming an internal node and nothing else, which
+ # is all a caller saw when one row held an empty string or a null.
+ tf.Assert(
is_valid,
- True,
- message="Invalid date format. Expected YYYY-MM-DD or YYYY/MM/DD",
+ [
+ tf.constant(
+ "Invalid date value. Expected YYYY-MM-DD or "
+ "YYYY/MM/DD, optionally followed by a time. An empty "
+ "value counts: a date column cannot hold nulls, so "
+ "fill or drop those rows before preprocessing. The "
+ "value that failed was:",
+ ),
+ date_str,
+ ],
)
+ # Everything below is built from the calendar date, so drop any
+ # time that came with it.
+ date_str = tf.strings.regex_replace(date_str, r"[ T].*$", "")
+
# First, standardize the separator to '-' in case of YYYY/MM/DD format
date_str = tf.strings.regex_replace(date_str, "/", "-")
diff --git a/kdp/layers/distribution_transform_layer.py b/kdp/layers/distribution_transform_layer.py
index 04408d4..cb088aa 100644
--- a/kdp/layers/distribution_transform_layer.py
+++ b/kdp/layers/distribution_transform_layer.py
@@ -7,8 +7,13 @@
from typing import Any
from loguru import logger
from keras import ops, KerasTensor
+import numpy as np
import tensorflow as tf
+# The gap between 1.0 and the next float32 below it. A clipping margin narrower
+# than this rounds away, leaving the bound it was meant to hold open.
+_FLOAT32_STEP = float(np.finfo(np.float32).eps)
+
@keras.saving.register_keras_serializable(package="kdp.layers")
class DistributionTransformLayer(keras.layers.Layer):
@@ -29,9 +34,10 @@ class DistributionTransformLayer(keras.layers.Layer):
lambda_param: Parameter for parameterized transformations like Box-Cox and Yeo-Johnson.
Default is 0.0.
epsilon: Small value added to prevent numerical issues like log(0). Default is 1e-10.
- min_value: Minimum value for min-max scaling. Default is 0.0.
- max_value: Maximum value for min-max scaling. Default is 1.0.
- clip_values: Whether to clip values to the specified range in min-max scaling. Default is True.
+ min_value: Low end of the range 'min-max' scales onto. Default is 0.0.
+ max_value: High end of the range 'min-max' scales onto. Default is 1.0.
+ clip_values: Whether 'min-max' clips its result into
+ [min_value, max_value]. Default is True.
auto_candidates: list of transformation types to consider when transform_type is 'auto'.
If None, all available transformations will be considered. Default is None.
name: Optional name for the layer.
@@ -458,8 +464,14 @@ def apply_none() -> tf.Tensor:
return x
def apply_log() -> tf.Tensor:
- # Add epsilon to avoid log(0)
- return tf.math.log(x + self.epsilon)
+ # `log` is defined for positive values only. Every neighbouring
+ # transform handles its own domain -- `sqrt` clamps at zero,
+ # `box-cox` at epsilon -- but this one handed negatives straight to
+ # `log` and returned NaN, which then spreads through the rest of the
+ # model and every gradient taken from it, silently. Clamp the same
+ # way `box-cox` does, so a negative value lands on the floor of the
+ # transform rather than outside it.
+ return tf.math.log(tf.maximum(x, 0.0) + self.epsilon)
def apply_sqrt() -> tf.Tensor:
# Ensure non-negative values
@@ -510,9 +522,16 @@ def apply_yeo_johnson() -> tf.Tensor:
)
def apply_arcsinh() -> tf.Tensor:
- # Inverse hyperbolic sine transformation
- # Works well for both positive and negative values with heavy tails
- return tf.math.log(x + tf.sqrt(tf.square(x) + 1.0))
+ # Inverse hyperbolic sine transformation.
+ # Works well for both positive and negative values with heavy tails.
+ #
+ # Spelled out as `log(x + sqrt(x**2 + 1))` this loses the whole
+ # negative tail: for x = -1e6 the square is 1e12, where float32
+ # cannot hold the +1, its root is exactly 1e6, the sum is exactly
+ # zero and the logarithm is -inf. `x**2` also overflows to infinity
+ # well before x does. `asinh` computes the same function without
+ # either step.
+ return tf.math.asinh(x)
def apply_cube_root() -> tf.Tensor:
# Cube root transformation
@@ -534,27 +553,53 @@ def apply_cube_root() -> tf.Tensor:
)
def apply_logit() -> tf.Tensor:
- # Logit transformation: log(x / (1 - x))
- # Clip values to the valid range with a small epsilon
- x_clipped = tf.clip_by_value(x, self.epsilon, 1.0 - self.epsilon)
+ # Logit transformation: log(x / (1 - x)), which needs x strictly
+ # inside (0, 1).
+ #
+ # The margin has to survive the arithmetic that follows. The default
+ # epsilon of 1e-10 is far below the float32 gap at 1.0, so
+ # `1 - epsilon` rounded straight back to 1.0: the upper clip did
+ # nothing, `1 - x_clipped` was exactly zero, and every value at or
+ # above 1 came out as +inf. Keep the margin at least one float32
+ # step wide.
+ margin = tf.maximum(
+ tf.cast(self.epsilon, x.dtype),
+ tf.cast(_FLOAT32_STEP, x.dtype),
+ )
+ x_clipped = tf.clip_by_value(x, margin, 1.0 - margin)
return tf.math.log(x_clipped / (1.0 - x_clipped))
def apply_min_max() -> tf.Tensor:
- # Min-Max scaling to [0, 1]
- if self.clip_values:
- # Use predefined min/max values
- min_val = self.min_value
- max_val = self.max_value
- else:
- # Compute min/max from data
- min_val = tf.reduce_min(x)
- max_val = tf.reduce_max(x)
+ # Scale each feature's own range onto [min_value, max_value].
+ #
+ # Those two used to be read as the *source* range instead --
+ # `(x - min_value) / (max_value - min_value)` -- so with their
+ # defaults of 0 and 1 the transform returned its input untouched,
+ # and no setting could scale onto a range other than [0, 1].
+ # `clip_values` chose between that and the data's own range, which
+ # is not what "whether to clip values to the specified range" says
+ # and left the output far outside the range it names.
+ #
+ # Per feature, like `robust-scale` and `quantile` below: a single
+ # min and max across the whole tensor would let one wide column
+ # flatten every other one.
+ data_min = tf.reduce_min(x, axis=0, keepdims=True)
+ data_max = tf.reduce_max(x, axis=0, keepdims=True)
+
+ spread = data_max - data_min
+ spread = tf.where(
+ tf.abs(spread) < self.epsilon,
+ tf.ones_like(spread),
+ spread,
+ )
- # Avoid division by zero
- denom = max_val - min_val
- denom = tf.where(tf.abs(denom) < self.epsilon, tf.ones_like(denom), denom)
+ low = tf.cast(self.min_value, x.dtype)
+ high = tf.cast(self.max_value, x.dtype)
+ scaled = low + (x - data_min) / spread * (high - low)
- return (x - min_val) / denom
+ if self.clip_values:
+ scaled = tf.clip_by_value(scaled, low, high)
+ return scaled
def apply_robust_scale() -> tf.Tensor:
# Robust scaling: centre on the per-feature median and divide by the
diff --git a/kdp/layers/time_series/tsfresh_feature_layer.py b/kdp/layers/time_series/tsfresh_feature_layer.py
index 9f02ea3..8ae9625 100644
--- a/kdp/layers/time_series/tsfresh_feature_layer.py
+++ b/kdp/layers/time_series/tsfresh_feature_layer.py
@@ -134,7 +134,28 @@ def build(self, input_shape) -> None:
Args:
input_shape: Shape of the input tensor.
+
+ Raises:
+ ValueError: If there is only one step to summarize. Every statistic
+ this layer computes is degenerate over a window of one -- the
+ mean, the minimum and the maximum are all the value itself and
+ the standard deviation is zero -- and `window_size` is clamped
+ to the number of steps available, so a `TimeSeriesFeature` that
+ hands over one column per row got its own column back once per
+ requested statistic, plus a column of zeros. It looked like
+ four engineered features and carried nothing.
"""
+ time_steps = tuple(input_shape)[1] if len(input_shape) > 1 else None
+ if time_steps is not None and time_steps < 2:
+ raise ValueError(
+ "TSFreshFeatureLayer needs at least two time steps on axis 1 "
+ f"and received {time_steps}. In a `TimeSeriesFeature` this means "
+ "combining `tsfresh_feature_config` with a config that widens "
+ "the feature first -- `lag_config`, `rolling_stats_config` or "
+ "`moving_average_config` -- so there is a window to summarize. "
+ "Used directly, pass `(batch, time_steps)` or "
+ "`(batch, time_steps, features)`.",
+ )
super().build(input_shape)
def call(self, inputs, training=None) -> tf.Tensor:
diff --git a/kdp/model_advisor.py b/kdp/model_advisor.py
index f787070..4bbab37 100644
--- a/kdp/model_advisor.py
+++ b/kdp/model_advisor.py
@@ -810,6 +810,11 @@ def generate_code_snippet(self) -> str:
code.append("")
code.append("# Create preprocessing model with recommended configuration")
code.append("model = PreprocessingModel(")
+ # Without this the snippet cannot be run: every configuration the
+ # advisor recommends has numeric, text or vocabulary-based features in
+ # it, and building one of those needs the data. Pasting the snippet
+ # raised "neither `features_stats` nor `path_data` was provided".
+ code.append(' path_data="your_data.csv", # the data these came from')
code.append(" features_specs=features,")
# Add global configuration
diff --git a/kdp/processor.py b/kdp/processor.py
index e89deb1..94763cc 100644
--- a/kdp/processor.py
+++ b/kdp/processor.py
@@ -17,10 +17,12 @@
from pathlib import Path
import json
import numpy as np
+import pandas as pd
from loguru import logger
from kdp.layers.global_numerical_embedding_layer import GlobalNumericalEmbedding
+from kdp.inference.base import InferenceFormatter
from kdp.features import (
CategoricalFeature,
FeatureSpaceConverter,
@@ -63,15 +65,35 @@ def _to_tensor_mapping(data: dict) -> dict:
converted = {}
for key, value in data.items():
if isinstance(value, tf.Tensor) or tf.is_tensor(value):
- converted[key] = value
+ converted[key] = _as_column(value)
continue
try:
- converted[key] = tf.convert_to_tensor(value)
+ converted[key] = _as_column(tf.convert_to_tensor(value))
except (ValueError, TypeError, tf.errors.OpError):
converted[key] = value
return converted
+def _as_column(tensor: tf.Tensor) -> tf.Tensor:
+ """Give a per-row value list the (rows, 1) shape every input is declared with.
+
+ Every feature is a `keras.Input(shape=(1,))`, so a column of N values has to
+ arrive as (N, 1). A flat list -- `{"age": [35.0, 41.0]}`, and everything
+ `InferenceFormatter` produces -- converts to shape (N,), which `predict()`
+ rejected with "as_list() is not defined on an unknown TensorShape". The
+ values were never wrong, only the axis they were missing.
+
+ Args:
+ tensor: A converted feature column.
+
+ Returns:
+ The same values with a trailing axis when they arrived without one.
+ """
+ if len(tensor.shape) == 1:
+ return tf.expand_dims(tensor, axis=-1)
+ return tensor
+
+
class CallableDict(dict):
"""A dictionary that can be called like a function.
@@ -313,6 +335,9 @@ def __init__(
self.features_specs = features_specs or {}
self.features_stats_path = features_stats_path or "features_stats.json"
self.feature_crosses = feature_crosses or []
+ # Filled in by `_add_pipeline_cross`: the names the crossed tensors
+ # are stored under in `processed_features`.
+ self._cross_feature_names: list[str] = []
self.output_mode = output_mode
self.overwrite_stats = overwrite_stats
self.use_caching = use_caching
@@ -1300,8 +1325,14 @@ def _adapted_text_vectorizer(
"""
if not self.path_data:
raise ValueError(
- f"{feature_name}: output_mode='tf_idf' needs the data to compute "
- "IDF weights from, so `path_data` is required.",
+ f"{feature_name}: this configuration "
+ f"(output_mode={kwargs.get('output_mode')!r}, "
+ f"max_tokens={kwargs.get('max_tokens')!r}, "
+ f"ngrams={kwargs.get('ngrams')!r}, "
+ f"standardize={kwargs.get('standardize')!r}, "
+ f"split={kwargs.get('split')!r}) needs its vocabulary built from "
+ f"the column itself -- the collected statistics cannot describe "
+ f"it -- so `path_data` is required.",
)
vectorizer = keras.layers.TextVectorization(**kwargs)
@@ -1319,8 +1350,53 @@ def _adapted_text_vectorizer(
f"Adapted the text vectorizer for {feature_name} on the data; "
f"vocabulary size {len(vectorizer.get_vocabulary())}",
)
+
+ if kwargs.get("output_mode") == TextVectorizerOutputOptions.TF_IDF:
+ vectorizer = self._restatable_tf_idf_vectorizer(vectorizer, **kwargs)
+
return vectorizer
+ @staticmethod
+ def _restatable_tf_idf_vectorizer(
+ adapted: keras.layers.TextVectorization,
+ **kwargs: Any,
+ ) -> keras.layers.TextVectorization:
+ """Rebuild an adapted `tf_idf` vectorizer so that it can be reloaded.
+
+ A `TextVectorization` in `tf_idf` mode that learned its state by
+ adapting cannot be read back from a saved model: Keras writes the IDF
+ weights as layer variables, and on load assigns them to a layer that has
+ no vocabulary yet and therefore no such variable, which surfaces as
+ "object of type 'bool' has no len()". The model saves without
+ complaint, so the loss shows up only when the model is next needed. This
+ happens with `TextVectorization` alone, with no KDP in the picture.
+
+ Passing the vocabulary and the weights to the constructor puts both in
+ the layer's config, where loading reads them. The layer computes the
+ same numbers either way.
+
+ Args:
+ adapted: The vectorizer that has already read the column.
+ **kwargs: The arguments it was constructed with.
+
+ Returns:
+ An equivalent vectorizer holding its state in its config.
+ """
+ vocabulary = [str(word) for word in adapted.get_vocabulary()]
+ specials = len(vocabulary) - len(
+ adapted.get_vocabulary(include_special_tokens=False),
+ )
+ weights = np.asarray(adapted._lookup_layer.idf_weights_const.numpy())
+
+ arguments = dict(kwargs)
+ arguments.pop("vocabulary", None)
+ arguments.pop("idf_weights", None)
+ return keras.layers.TextVectorization(
+ vocabulary=vocabulary[specials:],
+ idf_weights=weights[specials:],
+ **arguments,
+ )
+
@_monitor_performance
def _add_pipeline_text(self, feature_name: str, input_layer, stats: dict) -> None:
"""Add a text preprocessing step to the pipeline.
@@ -1368,13 +1444,62 @@ def _add_pipeline_text(self, feature_name: str, input_layer, stats: dict) -> Non
):
_feature.kwargs["output_sequence_length"] = 35
+ # `max_tokens` says how large a vocabulary to keep, and the
+ # statistics pass collects every distinct word it sees. Handing a
+ # larger vocabulary to a capped layer is refused by Keras --
+ # "Attempted to set a vocabulary larger than the maximum vocab
+ # size" -- so asking for a smaller vocabulary than the data holds,
+ # which is the only reason to set the option, could not build.
+ # `int` reserves two slots (padding and out-of-vocabulary), the
+ # other modes one. Adapting on the column keeps the most frequent
+ # `max_tokens` words, which is what the cap means; the statistics
+ # carry no counts to pick them with.
+ _max_tokens = _feature.kwargs.get("max_tokens")
+ _reserved = (
+ 2
+ if _feature.kwargs.get("output_mode", TextVectorizerOutputOptions.INT)
+ == TextVectorizerOutputOptions.INT
+ else 1
+ )
+ _vocab_too_large = (
+ _max_tokens is not None and len(_vocab) + _reserved > _max_tokens
+ )
+
+ # The statistics collect single words, standardized the way
+ # `TextVectorization` standardizes by default. Any option that
+ # changes what a token is puts the two out of step: `ngrams` asks
+ # for word pairs the statistics never recorded, and a custom
+ # `standardize` or `split` spells them differently. The vocabulary
+ # was handed over regardless, so every n-gram and every differently
+ # spelled token landed in the one out-of-vocabulary slot -- the
+ # widths and the counts all looked right. Read the column instead
+ # when that happens, and let the layer build the vocabulary it will
+ # actually use.
+ _tokenizes_differently = any(
+ _feature.kwargs.get(option) is not None
+ for option in ("ngrams", "standardize", "split")
+ )
+ _needs_its_own_vocabulary = _vocab_too_large or _tokenizes_differently
+ if _needs_its_own_vocabulary:
+ logger.info(
+ f"{feature_name}: the statistics vocabulary cannot describe "
+ f"this vectorizer (max_tokens={_max_tokens}, "
+ f"ngrams={_feature.kwargs.get('ngrams')}, "
+ f"standardize={_feature.kwargs.get('standardize')}, "
+ f"split={_feature.kwargs.get('split')}); adapting it on the "
+ f"column instead.",
+ )
+
# `tf_idf` weights every token by how rare it is across documents,
# and Keras refuses a vocabulary in that mode without the matching
# `idf_weights`, which KDP's statistics do not collect -- so the
# documented, exported `TF_IDF` output mode could not build at all.
# Letting the layer adapt on the column computes both the vocabulary
# and the weights, from the same data the statistics came from.
- if _feature.kwargs.get("output_mode") == TextVectorizerOutputOptions.TF_IDF:
+ if (
+ _feature.kwargs.get("output_mode") == TextVectorizerOutputOptions.TF_IDF
+ or _needs_its_own_vocabulary
+ ):
vectorizer = self._adapted_text_vectorizer(
feature_name=feature_name,
name=f"text_vactorizer_{feature_name}",
@@ -1696,11 +1821,23 @@ def _add_pipeline_cross(self) -> None:
3. Converting output to float32 for compatibility
4. Adding the result to appropriate output collection based on output mode
"""
+ self._cross_feature_names = []
for feature_a, feature_b, nr_bins in self.feature_crosses:
preprocessor = FeaturePreprocessor(name=f"{feature_a}_x_{feature_b}")
# checking inputs existance for feature A
for _feature_name in [feature_a, feature_b]:
+ # A name that is not a declared feature used to surface as a
+ # bare `KeyError: 'name'` from the line below, with nothing to
+ # say where it came from.
+ if _feature_name not in self.features_specs:
+ raise ValueError(
+ f"feature_crosses asks to cross '{feature_a}' with "
+ f"'{feature_b}', but '{_feature_name}' is not in "
+ f"features_specs. A cross combines two declared "
+ f"features: add it to features_specs, or drop the "
+ f"cross.",
+ )
# getting feature object
_feature = self.features_specs[_feature_name]
_input = self.inputs.get(_feature_name)
@@ -1709,6 +1846,23 @@ def _add_pipeline_cross(self) -> None:
_col_dtype = _feature.dtype
self._add_input_column(feature_name=_feature, dtype=_col_dtype)
+ # `HashedCrossing` hashes the raw values, so it only accepts
+ # integer or string columns. Crossing a float feature used to build
+ # a model that looked fine -- summary, output shape, everything --
+ # and then raise on the first batch it was given. Refuse it here,
+ # where the configuration that caused it is still in view.
+ for _feature_name in (feature_a, feature_b):
+ _dtype = str(self.inputs[_feature_name].dtype)
+ if _dtype != "string" and not _dtype.startswith(("int", "uint")):
+ raise ValueError(
+ f"feature_crosses asks to cross '{feature_a}' with "
+ f"'{feature_b}', but '{_feature_name}' is a "
+ f"{_dtype} column and a hashed cross can only combine "
+ f"integer or string columns. Cross categorical "
+ f"features, or bucket the numeric one into a "
+ f"categorical feature first.",
+ )
+
feature_name = f"{feature_a}_x_{feature_b}"
preprocessor.add_processing_step(
layer_class="HashedCrossing",
@@ -1726,6 +1880,7 @@ def _add_pipeline_cross(self) -> None:
# Process the feature
self.processed_features[feature_name] = _output_pipeline
+ self._cross_feature_names.append(feature_name)
@_monitor_performance
def _score_features_against_each_other(self) -> None:
@@ -1888,6 +2043,32 @@ def _combine_all_features(
else:
raise ValueError("No features available for concatenation")
+ def _concatenation_order(self) -> list[str]:
+ """The processed features to concatenate, in a stable order.
+
+ Declared features come first, in the order `features_specs` lists them,
+ followed by the crosses in the order `feature_crosses` gives them.
+ Anything else is sorted by name so that it cannot move either.
+
+ Returns:
+ The feature names to concatenate, without the selection weights.
+ """
+ ordered = [
+ name for name in self.features_specs if name in self.processed_features
+ ]
+ ordered += [
+ name
+ for name in self._cross_feature_names
+ if name in self.processed_features
+ ]
+ placed = set(ordered)
+ ordered += sorted(
+ name
+ for name in self.processed_features
+ if name not in placed and not name.endswith("_weights")
+ )
+ return ordered
+
def _group_features_by_type(self) -> tuple[list, list]:
"""Group processed features by type for concatenation.
@@ -1909,10 +2090,30 @@ def _group_features_by_type(self) -> tuple[list, list]:
passthrough_names_numeric = []
passthrough_names_string = []
- # Group processed features by type
- for feature_name, feature in self.processed_features.items():
- # Skip feature weights
- if feature_name.endswith("_weights"):
+ # Group processed features by type.
+ #
+ # In the order the features were declared, not the order they finished
+ # being built in. Features are processed in parallel batches, so
+ # `processed_features` is keyed by whichever worker returned first:
+ # building the same configuration twice put the columns of `concat_all`
+ # in a different order, five different layouts in six builds. A model
+ # trained on one build and served by another read every feature as
+ # another one, and nothing said so. The saved `.keras` file was never
+ # affected -- the order is baked into its graph -- so this only bit
+ # callers who rebuilt the preprocessor instead of reloading it.
+ for feature_name in self._concatenation_order():
+ feature = self.processed_features[feature_name]
+
+ # A cross is configured with `feature_crosses`, not by declaring a
+ # column, so it has no entry in `features_specs`. The lookup below
+ # therefore found nothing and skipped it: `_add_pipeline_cross`
+ # built every cross and concatenation then threw them all away, so
+ # in the default output mode the option did nothing at all. A
+ # hashed cross is a categorical index, so it joins that block.
+ if feature_name in self._cross_feature_names:
+ logger.debug(f"Adding cross {feature_name} to categorical features")
+ categorical_features.append(feature)
+ categorical_names.append(feature_name)
continue
# Add to appropriate list based on feature type
@@ -2310,6 +2511,16 @@ def _apply_all_features_transformer(self) -> None:
name=f"transformer_block_{block_idx}_{self.transfo_nr_heads}heads",
)(self.concat_all)
+ # A transformer block adds a sequence axis to a 2-D input and hands it
+ # back, so this placement was the one configuration whose model came out
+ # as (rows, 1, width) instead of (rows, width) -- enough to break a Dense
+ # head bolted on after it. The categorical placement already flattened
+ # the same way, one method above.
+ self.concat_all = keras.layers.Reshape(
+ target_shape=(-1,),
+ name="reshape_transformer_output",
+ )(self.concat_all)
+
def _prepare_dict_mode_outputs(self) -> None:
"""Prepare outputs for dictionary mode."""
# Dictionary mode
@@ -3256,6 +3467,19 @@ def predict(self, data, **kwargs) -> Any:
# Validate time series inference data
self._validate_time_series_inference_data(data)
+ # A DataFrame is the first thing the docstring offers, and it went
+ # straight to Keras, which tried to read every column as a float and
+ # raised "could not convert string to float" on the first categorical
+ # one. `InferenceFormatter` already takes a frame apart column by
+ # column, and it also decides each column's type from the values that
+ # are really there -- so a row with a gap in it survives instead of
+ # raising "Invalid dtype: object" on the mixed column it produces.
+ if isinstance(data, pd.DataFrame):
+ data = InferenceFormatter(self).prepare_inference_data(
+ data,
+ to_tensors=True,
+ )
+
# Keras cannot consume NumPy string arrays, which every categorical,
# text or date feature produces when the caller passes a plain dict.
if isinstance(data, dict):
diff --git a/kdp/stats.py b/kdp/stats.py
index f7cd763..183827e 100644
--- a/kdp/stats.py
+++ b/kdp/stats.py
@@ -22,52 +22,233 @@
class WelfordAccumulator:
- """Accumulator for computing the mean and variance of a sequence of numbers
- using the Welford algorithm (streaming data).
+ """Streaming mean and variance for a sequence of numbers.
+
+ Accumulates in float64 and pools each batch with the Chan-Golub-LeVeque
+ update. Both matter, and float32 was doing real damage: the deviations
+ `x - mean` were computed at the magnitude of the values themselves, so a
+ column of large numbers lost every significant digit to cancellation. A
+ column around 1e8 with a spread of 1e3 came out with a variance of
+ *minus* 1.6e6 against a true 1.0e6, and one with a spread of 1e-4 was
+ wrong by four orders of magnitude.
+
+ Nothing raised. `Normalization` divides by the square root of that
+ variance, so the column arrived at the model as a constant -- every ID,
+ epoch timestamp, amount in cents or sensor reading with a large offset
+ silently flattened, its signal gone.
+
+ Pooling each batch from its own mean keeps the subtraction at the scale of
+ the spread rather than the values, and float64 leaves ten orders of
+ magnitude of headroom on top. Together they hold the relative error near
+ 1e-13 on the cases that used to break.
"""
def __init__(self):
"""Initializes the accumulators for the Welford algorithm."""
self.n = tf.Variable(
0.0,
- dtype=tf.float32,
+ dtype=tf.float64,
trainable=False,
)
self.mean = tf.Variable(
0.0,
- dtype=tf.float32,
+ dtype=tf.float64,
trainable=False,
)
self.M2 = tf.Variable(
0.0,
- dtype=tf.float32,
+ dtype=tf.float64,
+ trainable=False,
+ )
+ # Third and fourth central moments, pooled the same way as M2, and the
+ # running extremes. The advisor reads skewness, kurtosis, min and max;
+ # none of them were ever collected, so every column arrived at it with
+ # the neutral defaults -- skew 0, kurtosis 3 -- and was reported as
+ # "Normal distribution detected" whatever shape it had, while the
+ # rescaling factor derived from min and max always came out as 1.
+ self.M3 = tf.Variable(
+ 0.0,
+ dtype=tf.float64,
+ trainable=False,
+ )
+ self.M4 = tf.Variable(
+ 0.0,
+ dtype=tf.float64,
+ trainable=False,
+ )
+ self.minimum = tf.Variable(
+ float("inf"),
+ dtype=tf.float64,
+ trainable=False,
+ )
+ self.maximum = tf.Variable(
+ float("-inf"),
+ dtype=tf.float64,
trainable=False,
)
self.var = tf.Variable(
0.0,
- dtype=tf.float32,
+ dtype=tf.float64,
trainable=False,
)
@tf.function
def update(self, values: tf.Tensor) -> None:
- """Updates the accumulators with new values using the Welford algorithm.
+ """Pool a batch of values into the running mean and variance.
Args:
values: The new values to add to the accumulators.
"""
- values = tf.cast(values, tf.float32)
- n = self.n + tf.cast(tf.size(values), tf.float32)
- delta = values - self.mean
- self.mean.assign(self.mean + tf.reduce_sum(delta / n))
- self.M2.assign(self.M2 + tf.reduce_sum(delta * (values - self.mean)))
- self.n.assign(n)
+ values = tf.cast(tf.reshape(values, [-1]), tf.float64)
+ batch_count = tf.cast(tf.size(values), tf.float64)
+
+ # An empty batch has to leave everything untouched, and both divisors
+ # below can be zero when it does. Flooring them at one keeps the
+ # arithmetic finite; every numerator carries `batch_count`, so an empty
+ # batch contributes nothing either way.
+ safe_batch_count = tf.maximum(batch_count, 1.0)
+ batch_mean = tf.reduce_sum(values) / safe_batch_count
+ centered = values - batch_mean
+ batch_m2 = tf.reduce_sum(tf.square(centered))
+ batch_m3 = tf.reduce_sum(centered * tf.square(centered))
+ batch_m4 = tf.reduce_sum(tf.square(tf.square(centered)))
+
+ # The identity element rather than a branch: an empty batch reduces to
+ # the infinity that leaves the running extreme alone.
+ infinity = tf.constant([float("inf")], dtype=tf.float64)
+ batch_min = tf.reduce_min(tf.concat([values, infinity], axis=0))
+ batch_max = tf.reduce_max(tf.concat([values, -infinity], axis=0))
+
+ self._pool(
+ count=batch_count,
+ mean=batch_mean,
+ m2=batch_m2,
+ m3=batch_m3,
+ m4=batch_m4,
+ minimum=batch_min,
+ maximum=batch_max,
+ )
+
+ def _pool(
+ self,
+ count: tf.Tensor,
+ mean: tf.Tensor,
+ m2: tf.Tensor,
+ m3: tf.Tensor,
+ m4: tf.Tensor,
+ minimum: tf.Tensor,
+ maximum: tf.Tensor,
+ ) -> None:
+ """Fold a summarized group of values into the running accumulators.
+
+ The pooled forms of the Chan-Golub-LeVeque update, carried up to the
+ fourth moment. Both `update` and `merge` reduce to the same arithmetic.
+
+ Args:
+ count: How many values the group holds.
+ mean: Their mean.
+ m2: The sum of their squared deviations from that mean.
+ m3: The sum of their cubed deviations.
+ m4: The sum of their fourth-power deviations.
+ minimum: The smallest of them, or +inf for an empty group.
+ maximum: The largest of them, or -inf for an empty group.
+ """
+ total = self.n + count
+ safe_total = tf.maximum(total, 1.0)
+ delta = mean - self.mean
+ held = self.n
+
+ self.M4.assign(
+ self.M4
+ + m4
+ + tf.square(tf.square(delta))
+ * held
+ * count
+ * (held * held - held * count + count * count)
+ / (safe_total * tf.square(safe_total))
+ + 6.0
+ * delta
+ * delta
+ * (held * held * m2 + count * count * self.M2)
+ / tf.square(safe_total)
+ + 4.0 * delta * (held * m3 - count * self.M3) / safe_total,
+ )
+ self.M3.assign(
+ self.M3
+ + m3
+ + delta
+ * tf.square(delta)
+ * held
+ * count
+ * (held - count)
+ / tf.square(safe_total)
+ + 3.0 * delta * (held * m2 - count * self.M2) / safe_total,
+ )
+ self.M2.assign(self.M2 + m2 + delta * delta * held * count / safe_total)
+ self.mean.assign(self.mean + delta * count / safe_total)
+ self.minimum.assign(tf.minimum(self.minimum, minimum))
+ self.maximum.assign(tf.maximum(self.maximum, maximum))
+ self.n.assign(total)
+
+ def merge(self, other: "WelfordAccumulator") -> None:
+ """Pool another accumulator's values into this one.
+
+ Args:
+ other: The accumulator whose values are being folded in.
+ """
+ self._pool(
+ count=tf.cast(other.n, tf.float64),
+ mean=tf.cast(other.mean, tf.float64),
+ m2=tf.cast(other.M2, tf.float64),
+ m3=tf.cast(other.M3, tf.float64),
+ m4=tf.cast(other.M4, tf.float64),
+ minimum=tf.cast(other.minimum, tf.float64),
+ maximum=tf.cast(other.maximum, tf.float64),
+ )
@property
def variance(self) -> float:
"""Returns the variance of the accumulated values."""
return self.M2 / (self.n - 1) if self.n > 1 else self.var
+ @property
+ def skewness(self) -> float:
+ """How lopsided the values are: 0 when symmetric, positive to the right.
+
+ Returns:
+ The population (Fisher-Pearson) skewness, or 0.0 when there are too
+ few values, or no spread, to have one.
+ """
+ if self.n < 3 or self.M2 <= 0:
+ return tf.constant(0.0, dtype=tf.float64)
+ return tf.sqrt(self.n) * self.M3 / tf.pow(self.M2, 1.5)
+
+ @property
+ def kurtosis(self) -> float:
+ """How heavy the tails are, on the scale where a normal curve is 3.
+
+ Returns:
+ The population kurtosis, or 3.0 when there are too few values, or
+ no spread, to have one.
+ """
+ if self.n < 4 or self.M2 <= 0:
+ return tf.constant(3.0, dtype=tf.float64)
+ return self.n * self.M4 / tf.square(self.M2)
+
+ @property
+ def smallest(self) -> float:
+ """The smallest value seen, or 0.0 before any has been."""
+ if self.n < 1:
+ return tf.constant(0.0, dtype=tf.float64)
+ return self.minimum.value()
+
+ @property
+ def largest(self) -> float:
+ """The largest value seen, or 0.0 before any has been."""
+ if self.n < 1:
+ return tf.constant(0.0, dtype=tf.float64)
+ return self.maximum.value()
+
@property
def count(self) -> int:
"""Returns the number of accumulated values."""
@@ -120,6 +301,12 @@ def get_unique_values(self) -> list:
return tf.unique(all_values)[0].numpy().tolist()
+# The punctuation `TextVectorization` removes under its default
+# `standardize="lower_and_strip_punctuation"`, copied from that layer so the
+# vocabulary collected here is spelled the way the layer will look it up.
+_KERAS_PUNCTUATION = r'[!"#$%&()\*\+,-\./:;<=>?@\[\\\]^_`{|}~\']'
+
+
class TextAccumulator:
def __init__(self) -> None:
"""Initializes the accumulator for text values, where each entry is a list of words separated by spaces.
@@ -151,10 +338,18 @@ def update(self, new_texts: tf.Tensor) -> None:
f"Unsupported data type for text features: {new_texts.dtype}",
)
- # Split each string into words and flatten the list
- new_texts = tf.strings.regex_replace(new_texts, r"\s+", " ")
+ # Tokenize exactly the way `TextVectorization` will, because this
+ # vocabulary is handed to that layer verbatim. Its default
+ # standardization lowercases and then strips punctuation before
+ # splitting on whitespace; this accumulator lowercased but kept the
+ # punctuation, so a column of ordinary prose produced a vocabulary of
+ # "product," and "it!" while the layer looked up "product" and "it".
+ # The width was right and the counts were right; the words simply were
+ # not there, and a large share of every sentence fell into the
+ # out-of-vocabulary slot.
+ new_texts = tf.strings.lower(new_texts)
+ new_texts = tf.strings.regex_replace(new_texts, _KERAS_PUNCTUATION, "")
split_words = tf.strings.split(new_texts).flat_values
- split_words = tf.strings.lower(split_words)
# Concatenate new words with existing words and update unique words
updated_words = tf.unique(tf.concat([self.words, split_words], axis=0))[0]
@@ -248,6 +443,53 @@ def variance(self) -> dict:
}
+def _warn_if_float32_flattens_the_column(
+ feature: str,
+ mean: float,
+ variance: float,
+) -> None:
+ """Say so when a column's spread is too fine for float32 to hold.
+
+ Everything downstream -- the CSV reader, the model's inputs, the
+ normalization layer -- is float32, which carries about seven significant
+ digits. A column whose values sit far from zero but vary only slightly
+ loses that variation on the way in, before any statistic is computed and
+ before any layer sees it. Unix timestamps in seconds are the usual case:
+ values near 1.6e9 are 128 apart in float32, so a column spread over a
+ minute arrives as two or three distinct numbers. Normalization then works
+ perfectly on what is left, and the feature reaching the model is a
+ constant.
+
+ Nothing raises, because the numbers are the ones the file actually
+ contains. This only makes the loss visible.
+
+ Args:
+ feature: The column's name, for the message.
+ mean: The column's mean.
+ variance: The column's variance.
+ """
+ mean = float(mean)
+ variance = float(variance)
+ if not math.isfinite(mean) or not math.isfinite(variance) or variance <= 0.0:
+ return
+
+ # The gap between neighbouring float32 values at this magnitude.
+ resolution = float(np.spacing(np.float32(abs(mean))))
+ spread = math.sqrt(variance)
+ if resolution <= 0.0 or spread >= 16.0 * resolution:
+ return
+
+ levels = max(int(6.0 * spread / resolution), 1)
+ logger.warning(
+ f"Feature '{feature}' has values around {mean:.6g} varying by only "
+ f"{spread:.6g}, which float32 cannot hold: neighbouring values at that "
+ f"magnitude are {resolution:.6g} apart, so the column arrives as about "
+ f"{levels} distinct value(s) and carries almost no information. "
+ f"Subtract a reference point before training (for a Unix timestamp, "
+ f"the start of your data), or declare the column as a DATE feature.",
+ )
+
+
class DatasetStatistics:
def __init__(
self,
@@ -349,15 +591,32 @@ def _get_csv_file_pattern(path) -> str:
)
file_path = Path(path)
- # Check if the path is a directory
+ # A path that names a file means that file. It used to be replaced with
+ # `
/*.csv`, so `path_data="data/train.csv"` computed statistics
+ # over every CSV sitting beside it -- `test.csv` included. That is the
+ # usage every example in the documentation shows, and the leak was
+ # silent: the numbers came back, just drawn from the wrong rows.
+ if file_path.is_file():
+ return str(file_path)
+ if file_path.is_dir():
+ return str(file_path / "*.csv")
+
+ # Neither, so it is either a glob the caller wrote themselves
+ # ("data/*.csv", "shard-*.csv") or a path that does not exist. A glob is
+ # passed through; anything else is reported here rather than inside
+ # `tf.data`, which answers a missing file with an empty dataset and
+ # statistics quietly computed from nothing.
+ if any(character in str(file_path) for character in "*?["):
+ return str(file_path)
if file_path.suffix:
- # Get the parent directory if the path is a file
- base_path = file_path.parent
- csv_pattern = base_path / "*.csv"
- else:
- csv_pattern = file_path / "*.csv"
-
- return str(csv_pattern)
+ raise FileNotFoundError(
+ f"`path_data` points at {str(file_path)!r}, which does not "
+ "exist. Pass a CSV file, a directory of CSV files, or a glob.",
+ )
+ raise FileNotFoundError(
+ f"`path_data` points at {str(file_path)!r}, which is not a file or "
+ "a directory. Pass a CSV file, a directory of CSV files, or a glob.",
+ )
def _read_data_into_dataset(self) -> tf.data.Dataset:
"""Reading CSV files from the provided path into a tf.data.Dataset."""
@@ -459,6 +718,20 @@ def _process_time_series_data(self) -> dict:
)
continue
+ # A calendar feature's column holds dates, not numbers: everything
+ # it produces is derived from the string by `CalendarFeatureLayer`,
+ # and there is no mean or variance to take. Feeding it to the
+ # accumulator anyway killed the statistics pass with "Cast string to
+ # double is not supported" from inside a `tf.function`, so the
+ # documented way of declaring calendar features could not be built
+ # at all.
+ if feature.dtype == tf.string:
+ logger.debug(
+ f"'{feature_name}' is a calendar feature; its column holds "
+ f"dates, so there are no numeric statistics to collect.",
+ )
+ continue
+
# Prepare for grouped processing if grouping is specified
if feature.group_by and feature.group_by in list(
dataset.element_spec.keys(),
@@ -517,28 +790,25 @@ def _process_time_series_data(self) -> dict:
accumulator.update(sorted_tensor)
group_accumulators[group_key] = accumulator
- # Combine statistics across groups
+ # Combine statistics across groups. Each group is pooled in
+ # whole. Replacing a group with `count` copies of its own mean
+ # -- what this did before -- throws away everything the group
+ # varied by, leaving only the variance *between* the group
+ # means: two groups of spread 25 whose means happened to be
+ # close reported a variance of 0.73 against a true 643.
if group_accumulators:
- # Create overall accumulator to combine statistics
combined_accumulator = WelfordAccumulator()
-
- # Combine all group means weighted by count
- all_values = []
- for _, acc in group_accumulators.items():
- mean_tensor = (
- tf.ones(shape=(int(acc.count.numpy()),), dtype=tf.float32)
- * acc.mean.numpy()
- )
- all_values.append(mean_tensor)
-
- if all_values:
- combined_tensor = tf.concat(all_values, axis=0)
- combined_accumulator.update(combined_tensor)
+ for accumulator in group_accumulators.values():
+ combined_accumulator.merge(accumulator)
# Calculate and store overall statistics
stats = {
"mean": float(combined_accumulator.mean.numpy()),
"var": float(combined_accumulator.variance.numpy()),
+ "min": float(combined_accumulator.smallest.numpy()),
+ "max": float(combined_accumulator.largest.numpy()),
+ "skewness": float(combined_accumulator.skewness.numpy()),
+ "kurtosis": float(combined_accumulator.kurtosis.numpy()),
"count": int(
sum(
acc.count.numpy() for acc in group_accumulators.values()
@@ -613,6 +883,10 @@ def _process_time_series_data(self) -> dict:
stats = {
"mean": float(accumulator.mean.numpy()),
"var": float(accumulator.variance.numpy()),
+ "min": float(accumulator.smallest.numpy()),
+ "max": float(accumulator.largest.numpy()),
+ "skewness": float(accumulator.skewness.numpy()),
+ "kurtosis": float(accumulator.kurtosis.numpy()),
"count": int(accumulator.count.numpy()),
"dtype": feature.dtype.name
if hasattr(feature.dtype, "name")
@@ -705,10 +979,22 @@ def compute_feature_stats(feature: str) -> tuple[str, dict]:
- date: mean and variance for each date component
"""
if feature_type == "numeric":
+ accumulator = self.numeric_stats[feature]
+ mean = accumulator.mean.numpy()
+ variance = accumulator.variance.numpy()
+ _warn_if_float32_flattens_the_column(feature, mean, variance)
return feature, {
- "mean": self.numeric_stats[feature].mean.numpy(),
- "count": self.numeric_stats[feature].count.numpy(),
- "var": self.numeric_stats[feature].variance.numpy(),
+ "mean": mean,
+ "count": accumulator.count.numpy(),
+ "var": variance,
+ # The advisor reads these four to tell one distribution
+ # from another and to size a rescaling factor. None of them
+ # were collected, so it read the neutral defaults and
+ # called every column normal.
+ "min": float(accumulator.smallest.numpy()),
+ "max": float(accumulator.largest.numpy()),
+ "skewness": float(accumulator.skewness.numpy()),
+ "kurtosis": float(accumulator.kurtosis.numpy()),
"dtype": self.features_specs[feature].dtype,
}
elif feature_type == "categorical":
diff --git a/test/layers/test_distribution_transform_math.py b/test/layers/test_distribution_transform_math.py
index f901429..0644985 100644
--- a/test/layers/test_distribution_transform_math.py
+++ b/test/layers/test_distribution_transform_math.py
@@ -121,10 +121,27 @@ def test_output_is_bounded(self):
class TestMinMax(unittest.TestCase):
- """min-max maps data into [0, 1]."""
+ """min-max maps each feature's own range onto [min_value, max_value].
+
+ `min_value` and `max_value` were read as the range the data was assumed to
+ be *in*, not the range to scale it *onto*: the transform computed
+ `(x - min_value) / (max_value - min_value)`, which with their defaults of 0
+ and 1 returns the input untouched. Min-max scaling did nothing at all
+ unless `clip_values` was turned off, and no setting could target a range
+ other than [0, 1]. The only test here covered that one working branch.
+ """
+
+ def test_the_default_settings_scale_into_the_unit_interval(self):
+ """The documented behaviour, and the case that used to be a no-op."""
+ data = _multi_scale_data(seed=5)
+ actual = np.asarray(
+ DistributionTransformLayer(transform_type="min-max")(tf.constant(data)),
+ )
+ self.assertFalse(np.allclose(actual, data))
+ self.assertAlmostEqual(float(actual.min()), 0.0, places=5)
+ self.assertAlmostEqual(float(actual.max()), 1.0, places=5)
- def test_scales_into_unit_interval(self):
- """With clipping disabled, the observed range maps onto [0, 1]."""
+ def test_with_clipping_disabled_too(self):
data = _multi_scale_data(seed=5)
actual = np.asarray(
DistributionTransformLayer(transform_type="min-max", clip_values=False)(
@@ -134,6 +151,53 @@ def test_scales_into_unit_interval(self):
self.assertAlmostEqual(float(actual.min()), 0.0, places=5)
self.assertAlmostEqual(float(actual.max()), 1.0, places=5)
+ def test_the_target_range_is_the_one_asked_for(self):
+ """[-1, 1] and [0, 10] were both unreachable."""
+ data = _multi_scale_data(seed=6)
+ for low, high in ((-1.0, 1.0), (0.0, 10.0), (5.0, 6.0)):
+ with self.subTest(target=(low, high)):
+ actual = np.asarray(
+ DistributionTransformLayer(
+ transform_type="min-max",
+ min_value=low,
+ max_value=high,
+ )(tf.constant(data)),
+ )
+ self.assertAlmostEqual(float(actual.min()), low, places=4)
+ self.assertAlmostEqual(float(actual.max()), high, places=4)
+
+ def test_each_feature_is_scaled_on_its_own_range(self):
+ """Three columns centred at -3, 50 and 1000 each span the whole range.
+
+ A single min and max across the tensor would leave the two narrow
+ columns squeezed into a sliver of it.
+ """
+ data = _multi_scale_data(seed=7)
+ actual = np.asarray(
+ DistributionTransformLayer(transform_type="min-max")(tf.constant(data)),
+ )
+ for column in range(data.shape[1]):
+ self.assertAlmostEqual(float(actual[:, column].min()), 0.0, places=4)
+ self.assertAlmostEqual(float(actual[:, column].max()), 1.0, places=4)
+
+ def test_the_order_of_the_values_is_kept(self):
+ data = _multi_scale_data(seed=8)
+ actual = np.asarray(
+ DistributionTransformLayer(transform_type="min-max")(tf.constant(data)),
+ )
+ for column in range(data.shape[1]):
+ np.testing.assert_array_equal(
+ np.argsort(actual[:, column]),
+ np.argsort(data[:, column]),
+ )
+
+ def test_a_constant_feature_does_not_divide_by_zero(self):
+ constant = np.full((40, 2), 7.0, dtype="float32")
+ actual = np.asarray(
+ DistributionTransformLayer(transform_type="min-max")(tf.constant(constant)),
+ )
+ self.assertTrue(np.isfinite(actual).all())
+
class TestGraphModeCompatibility(unittest.TestCase):
"""Every transform must build inside a functional model."""
diff --git a/test/layers/test_every_transform_matches_its_name.py b/test/layers/test_every_transform_matches_its_name.py
new file mode 100644
index 0000000..e0be3ec
--- /dev/null
+++ b/test/layers/test_every_transform_matches_its_name.py
@@ -0,0 +1,176 @@
+"""Each named transform has to compute the function it is named after.
+
+The other tests here check properties -- bounded output, per-feature scaling,
+order preserved. A transform can satisfy every one of those and still not be
+the function on the label: `min-max` returned its input untouched and passed a
+bounded-output test, because the input happened to be bounded.
+
+So each one is compared against its definition, written out in numpy.
+"""
+
+import unittest
+
+import numpy as np
+import pytest
+import tensorflow as tf
+
+from kdp.layers.distribution_transform_layer import DistributionTransformLayer
+
+EPSILON = 1e-10
+
+
+def _positive(seed=0, rows=400):
+ return (
+ np.abs(np.random.default_rng(seed).gamma(2.0, 2.0, (rows, 1))) + 0.5
+ ).astype(
+ "float32",
+ )
+
+
+def _mixed(seed=1, rows=400):
+ return np.random.default_rng(seed).normal(0, 3, (rows, 1)).astype("float32")
+
+
+def _unit(seed=2, rows=400):
+ return np.random.default_rng(seed).uniform(0.05, 0.95, (rows, 1)).astype("float32")
+
+
+def _yeo_johnson(x, lam):
+ out = np.empty_like(x, dtype=np.float64)
+ positive = x >= 0
+ if lam == 0:
+ out[positive] = np.log1p(x[positive])
+ else:
+ out[positive] = ((x[positive] + 1) ** lam - 1) / lam
+ negative = ~positive
+ if lam == 2:
+ out[negative] = -np.log1p(-x[negative])
+ else:
+ out[negative] = -(((-x[negative] + 1) ** (2 - lam) - 1) / (2 - lam))
+ return out
+
+
+@pytest.mark.layers
+class TestEveryTransformMatchesItsName(unittest.TestCase):
+ """Every transform against the function it claims to be."""
+
+ def _apply(self, kind, data, **kwargs):
+ layer = DistributionTransformLayer(transform_type=kind, **kwargs)
+ return np.asarray(layer(tf.constant(data)), dtype=np.float64)
+
+ def _assert_matches(self, kind, actual, expected, tolerance=1e-3):
+ actual = np.asarray(actual, dtype=np.float64).ravel()
+ expected = np.asarray(expected, dtype=np.float64).ravel()
+ self.assertEqual(actual.shape, expected.shape, kind)
+ relative = np.max(np.abs(actual - expected) / (np.abs(expected) + 1e-6))
+ self.assertLess(
+ relative, tolerance, f"{kind}: max relative error {relative:.2e}"
+ )
+
+ def test_none_is_the_identity(self):
+ data = _mixed()
+ self._assert_matches("none", self._apply("none", data), data)
+
+ def test_log(self):
+ data = _positive()
+ self._assert_matches("log", self._apply("log", data), np.log(data + EPSILON))
+
+ def test_sqrt(self):
+ data = _positive()
+ self._assert_matches("sqrt", self._apply("sqrt", data), np.sqrt(data))
+
+ def test_arcsinh(self):
+ data = _mixed()
+ self._assert_matches("arcsinh", self._apply("arcsinh", data), np.arcsinh(data))
+
+ def test_cube_root(self):
+ data = _mixed()
+ self._assert_matches(
+ "cube-root",
+ self._apply("cube-root", data),
+ np.sign(data) * np.abs(data) ** (1 / 3),
+ )
+
+ def test_logit(self):
+ data = _unit()
+ self._assert_matches(
+ "logit",
+ self._apply("logit", data),
+ np.log(data / (1 - data)),
+ )
+
+ def test_box_cox(self):
+ data = _positive()
+ self._assert_matches(
+ "box-cox (lambda 0)",
+ self._apply("box-cox", data, lambda_param=0.0),
+ np.log(data + EPSILON),
+ )
+ for lam in (0.5, 1.0, 2.0):
+ self._assert_matches(
+ f"box-cox (lambda {lam})",
+ self._apply("box-cox", data, lambda_param=lam),
+ (data**lam - 1) / lam,
+ )
+
+ def test_yeo_johnson(self):
+ data = _mixed()
+ for lam in (0.0, 0.5, 1.0):
+ self._assert_matches(
+ f"yeo-johnson (lambda {lam})",
+ self._apply("yeo-johnson", data, lambda_param=lam),
+ _yeo_johnson(data, lam),
+ )
+
+ def test_robust_scale(self):
+ data = _mixed()
+ iqr = np.percentile(data, 75) - np.percentile(data, 25)
+ self._assert_matches(
+ "robust-scale",
+ self._apply("robust-scale", data),
+ (data - np.median(data)) / iqr,
+ tolerance=1e-2,
+ )
+
+ def test_min_max(self):
+ """The one this file was written for: it returned its input."""
+ data = _mixed()
+ low, high = -1.0, 1.0
+ scaled = (data - data.min()) / (data.max() - data.min())
+ self._assert_matches(
+ "min-max",
+ self._apply("min-max", data, min_value=low, max_value=high),
+ low + scaled * (high - low),
+ tolerance=1e-2,
+ )
+
+ def test_quantile_maps_ranks_onto_the_documented_interval(self):
+ """Ranks to [-1, 1], which is what the implementation says it does."""
+ data = _mixed()
+ actual = self._apply("quantile", data)
+ ranks = np.argsort(np.argsort(data.ravel()))
+ expected = 2.0 * ((ranks + 0.5) / data.size) - 1.0
+ self._assert_matches("quantile", actual, expected, tolerance=1e-2)
+
+ def test_every_named_transform_is_covered_here(self):
+ """A transform added later must be measured too, not just listed."""
+ layer = DistributionTransformLayer()
+ named = set(layer._valid_transforms) - {"auto"}
+ covered = {
+ "none",
+ "log",
+ "sqrt",
+ "box-cox",
+ "yeo-johnson",
+ "arcsinh",
+ "cube-root",
+ "logit",
+ "quantile",
+ "robust-scale",
+ "min-max",
+ }
+ self.assertEqual(
+ named - covered,
+ set(),
+ "these transforms have no test against their definition",
+ )
diff --git a/test/test_calendar_features_can_be_built.py b/test/test_calendar_features_can_be_built.py
new file mode 100644
index 0000000..c2fd701
--- /dev/null
+++ b/test/test_calendar_features_can_be_built.py
@@ -0,0 +1,137 @@
+"""A calendar time series feature has to survive the statistics pass.
+
+The documented way to ask for calendar features is a `TimeSeriesFeature` whose
+`calendar_feature_config` names the components to extract. Its column holds
+dates, and everything it produces is derived from the string by
+`CalendarFeatureLayer` -- there is no mean or variance to take. The statistics
+pass fed it to the numeric accumulator anyway and died inside a `tf.function`
+with "Cast string to double is not supported", so that whole documented
+configuration could not be built at all.
+
+The layer itself was covered; the path from `PreprocessingModel` to it was not.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.features import TimeSeriesFeature
+from kdp.processor import OutputModeOptions
+
+ROWS = 70
+COMPONENTS = [
+ "year",
+ "month",
+ "day",
+ "day_of_week",
+ "day_of_year",
+ "week_of_year",
+ "is_weekend",
+ "is_month_start",
+ "is_month_end",
+]
+
+
+def _frame() -> pd.DataFrame:
+ # Nearly four years, so every component below actually varies.
+ stamps = pd.date_range("2021-01-01", periods=ROWS, freq="20D")
+ return pd.DataFrame(
+ {
+ "when": stamps.strftime("%Y-%m-%d"),
+ "sales": np.arange(ROWS, dtype=float),
+ },
+ )
+
+
+@pytest.mark.time_series
+class TestCalendarFeaturesCanBeBuilt(unittest.TestCase):
+ """From `features_specs` all the way to a tensor."""
+
+ def setUp(self) -> None:
+ self.directory = Path(tempfile.mkdtemp())
+ self.frame = _frame()
+ self.data = self.directory / "data.csv"
+ self.frame.to_csv(self.data, index=False)
+
+ def _run(self, **config) -> tuple[np.ndarray, int]:
+ feature = TimeSeriesFeature(name="when", calendar_feature_config=config)
+ preprocessor = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / f"stats_{len(config)}.json"),
+ features_specs={"when": feature, "sales": FeatureType.FLOAT_NORMALIZED},
+ overwrite_stats=True,
+ output_mode=OutputModeOptions.DICT,
+ )
+ preprocessor.build_preprocessor()
+ produced = np.asarray(
+ preprocessor.model(
+ {
+ "when": tf.constant(self.frame["when"].values.reshape(-1, 1)),
+ "sales": tf.constant(
+ self.frame["sales"].values.reshape(-1, 1),
+ dtype=tf.float32,
+ ),
+ },
+ )["when"],
+ )
+ return produced, feature.get_output_dim()
+
+ def test_the_documented_configuration_builds_and_runs(self) -> None:
+ produced, declared = self._run(features=COMPONENTS)
+ self.assertEqual(produced.shape, (ROWS, len(COMPONENTS)))
+ self.assertEqual(produced.shape[1], declared)
+ self.assertTrue(np.isfinite(produced).all())
+
+ def test_no_component_comes_back_constant(self) -> None:
+ produced, _ = self._run(features=COMPONENTS)
+ constant = [
+ name
+ for index, name in enumerate(COMPONENTS)
+ if produced[:, index].std() < 1e-12
+ ]
+ self.assertEqual(constant, [], f"constant calendar columns: {constant}")
+
+ def test_the_raw_components_are_the_ones_pandas_reads(self) -> None:
+ produced, _ = self._run(features=COMPONENTS, normalize=False)
+ stamps = pd.to_datetime(self.frame["when"])
+ expected = {
+ "year": stamps.dt.year,
+ "month": stamps.dt.month,
+ "day": stamps.dt.day,
+ "day_of_week": stamps.dt.dayofweek,
+ "day_of_year": stamps.dt.dayofyear,
+ "week_of_year": stamps.dt.isocalendar().week.astype(int),
+ "is_weekend": stamps.dt.dayofweek.isin([5, 6]).astype(int),
+ "is_month_start": stamps.dt.is_month_start.astype(int),
+ "is_month_end": stamps.dt.is_month_end.astype(int),
+ }
+ for index, name in enumerate(COMPONENTS):
+ with self.subTest(component=name):
+ np.testing.assert_allclose(
+ produced[:, index],
+ expected[name].to_numpy().astype(float),
+ atol=1e-4,
+ )
+
+ def test_normalize_puts_them_in_the_unit_interval(self) -> None:
+ produced, _ = self._run(features=COMPONENTS, normalize=True)
+ self.assertGreaterEqual(produced.min(), 0.0)
+ self.assertLessEqual(produced.max(), 1.0)
+
+ def test_the_cyclical_components_can_be_asked_for(self) -> None:
+ produced, declared = self._run(
+ features=["month_sin", "month_cos", "day_of_week_sin", "day_of_week_cos"],
+ )
+ self.assertEqual(produced.shape, (ROWS, 4))
+ self.assertEqual(produced.shape[1], declared)
+ self.assertTrue(np.isfinite(produced).all())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_dates_read_the_formats_they_accept.py b/test/test_dates_read_the_formats_they_accept.py
new file mode 100644
index 0000000..f3842ea
--- /dev/null
+++ b/test/test_dates_read_the_formats_they_accept.py
@@ -0,0 +1,140 @@
+"""A date column has to be read, or refused where the caller can see why.
+
+The parser reads year, then month, then day, with `-` or `/` between them. It
+never read the format string it was given, so a day-first or month-first format
+was accepted at construction, ignored, and met again at the first batch as an
+assertion buried inside a TensorFlow graph error -- far from the line that
+caused it.
+
+A time after the date is the other half: timestamps are what a great many date
+columns actually hold, and "2021-06-15 13:45:00" had no way through this layer
+at all.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.features import DateFeature
+from kdp.layers.date_parsing_layer import DateParsingLayer
+from kdp.processor import OutputModeOptions
+
+READABLE = ("%Y-%m-%d", "%Y/%m/%d", "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S")
+UNREADABLE = ("%d-%m-%Y", "%m/%d/%Y", "%d/%m/%Y", "nonsense")
+
+
+@pytest.mark.unit
+class TestDateFormatsAreCheckedWhereTheyAreWritten(unittest.TestCase):
+ """The format is judged at construction, not at the first batch."""
+
+ def test_readable_formats_are_accepted(self) -> None:
+ for date_format in READABLE:
+ with self.subTest(date_format=date_format):
+ DateFeature(
+ name="d",
+ feature_type=FeatureType.DATE,
+ format=date_format,
+ )
+
+ def test_unreadable_formats_are_refused_by_name(self) -> None:
+ for date_format in UNREADABLE:
+ with self.subTest(date_format=date_format):
+ with self.assertRaises(ValueError) as raised:
+ DateFeature(
+ name="signed_on",
+ feature_type=FeatureType.DATE,
+ format=date_format,
+ )
+ message = str(raised.exception)
+ self.assertIn("signed_on", message)
+ self.assertIn(date_format, message)
+
+ def test_date_format_is_the_same_option_as_format(self) -> None:
+ """Both spellings appear in the wild; only one was ever read."""
+ feature = DateFeature(
+ name="d",
+ feature_type=FeatureType.DATE,
+ date_format="%Y/%m/%d",
+ )
+ self.assertEqual(feature.kwargs.get("format"), "%Y/%m/%d")
+
+ with self.assertRaises(ValueError):
+ DateFeature(
+ name="d",
+ feature_type=FeatureType.DATE,
+ date_format="%d/%m/%Y",
+ )
+
+
+@pytest.mark.layers
+class TestTheParserReadsTimestamps(unittest.TestCase):
+ """A time after the date is dropped, not rejected."""
+
+ def test_a_date_with_a_time_parses_to_the_same_components(self) -> None:
+ layer = DateParsingLayer()
+ plain = np.asarray(layer(tf.constant([["2021-06-15"]])))
+ spaced = np.asarray(layer(tf.constant([["2021-06-15 13:45:00"]])))
+ iso = np.asarray(layer(tf.constant([["2021-06-15T13:45:00"]])))
+ slashed = np.asarray(layer(tf.constant([["2021/06/15 13:45:00"]])))
+
+ np.testing.assert_array_equal(plain, spaced)
+ np.testing.assert_array_equal(plain, iso)
+ np.testing.assert_array_equal(plain, slashed)
+ # year, month, day, day-of-week
+ np.testing.assert_array_equal(plain[0][:3], [2021, 6, 15])
+
+ def test_a_column_of_timestamps_goes_through_a_model(self) -> None:
+ directory = Path(tempfile.mkdtemp())
+ stamps = pd.date_range("2020-01-01 06:00:00", periods=60, freq="37h")
+ frame = pd.DataFrame(
+ {
+ "seen_at": stamps.strftime("%Y-%m-%d %H:%M:%S"),
+ "n": np.arange(60.0),
+ },
+ )
+ data = directory / "data.csv"
+ frame.to_csv(data, index=False)
+
+ preprocessor = PreprocessingModel(
+ path_data=str(data),
+ features_stats_path=str(directory / "stats.json"),
+ features_specs={
+ "seen_at": DateFeature(
+ name="seen_at",
+ feature_type=FeatureType.DATE,
+ format="%Y-%m-%d %H:%M:%S",
+ ),
+ "n": FeatureType.FLOAT_NORMALIZED,
+ },
+ overwrite_stats=True,
+ output_mode=OutputModeOptions.DICT,
+ )
+ preprocessor.build_preprocessor()
+ result = np.asarray(
+ preprocessor.model(
+ {
+ "seen_at": tf.constant(frame["seen_at"].values.reshape(-1, 1)),
+ "n": tf.constant(
+ frame["n"].values.reshape(-1, 1),
+ dtype=tf.float32,
+ ),
+ },
+ )["seen_at"],
+ )
+ self.assertEqual(result.shape, (60, 8))
+ self.assertTrue(np.isfinite(result).all())
+ self.assertGreater(
+ len(np.unique(result, axis=0)),
+ 1,
+ "every timestamp encoded to the same row",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_every_configuration_reloads.py b/test/test_every_configuration_reloads.py
new file mode 100644
index 0000000..2f32fb8
--- /dev/null
+++ b/test/test_every_configuration_reloads.py
@@ -0,0 +1,298 @@
+"""A saved preprocessor has to come back with the numbers it went in with.
+
+Serialization is where this library has lost data silently: the model saves
+without complaint, and the loss shows up only when someone needs the model
+back. `output_mode="tf_idf"` was the last case -- Keras writes a `tf_idf`
+vectorizer's IDF weights as layer variables and, on load, assigns them to a
+layer that has no vocabulary yet and therefore no such variable, which surfaces
+as `object of type 'bool' has no len()`. It happens with `TextVectorization`
+alone, with no KDP in the picture; passing the vocabulary and the weights to
+the constructor puts both in the config, where loading reads them.
+
+Rather than testing that one case, this walks every option that changes what
+gets built and checks the round trip is exact.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import keras
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.features import (
+ CategoricalFeature,
+ DateFeature,
+ NumericalFeature,
+ PassthroughFeature,
+ TextFeature,
+)
+from kdp.processor import OutputModeOptions
+
+ROWS = 120
+
+BASE = {
+ "num1": FeatureType.FLOAT_NORMALIZED,
+ "num2": FeatureType.FLOAT_RESCALED,
+ "cat1": FeatureType.STRING_CATEGORICAL,
+ "cat2": FeatureType.INTEGER_CATEGORICAL,
+ "txt": FeatureType.TEXT,
+ "date": FeatureType.DATE,
+}
+
+# (name, feature overrides, model options)
+CONFIGURATIONS = [
+ ("plain", {}, {}),
+ (
+ "one_hot",
+ {
+ "cat1": CategoricalFeature(
+ name="cat1",
+ feature_type=FeatureType.STRING_CATEGORICAL,
+ category_encoding="ONE_HOT_ENCODING",
+ ),
+ },
+ {},
+ ),
+ (
+ "hashing",
+ {
+ "cat1": CategoricalFeature(
+ name="cat1",
+ feature_type=FeatureType.STRING_CATEGORICAL,
+ category_encoding="HASHING",
+ hash_bucket_size=8,
+ salt=7,
+ ),
+ },
+ {},
+ ),
+ (
+ "discretized",
+ {
+ "num1": NumericalFeature(
+ name="num1",
+ feature_type=FeatureType.FLOAT_DISCRETIZED,
+ bin_boundaries=[-1.0, 0.0, 1.0],
+ ),
+ },
+ {},
+ ),
+ (
+ "numeric_embedding",
+ {
+ "num1": NumericalFeature(
+ name="num1",
+ feature_type=FeatureType.FLOAT_NORMALIZED,
+ use_embedding=True,
+ embedding_dim=6,
+ ),
+ },
+ {},
+ ),
+ (
+ "distribution_aware",
+ {
+ "num1": NumericalFeature(
+ name="num1",
+ feature_type=FeatureType.FLOAT_NORMALIZED,
+ use_distribution_aware=True,
+ ),
+ },
+ {},
+ ),
+ (
+ "text_multi_hot",
+ {"txt": TextFeature(name="txt", output_mode="multi_hot")},
+ {},
+ ),
+ ("text_tf_idf", {"txt": TextFeature(name="txt", output_mode="tf_idf")}, {}),
+ (
+ "text_ngrams",
+ {"txt": TextFeature(name="txt", output_mode="multi_hot", ngrams=2)},
+ {},
+ ),
+ (
+ "text_capped",
+ {"txt": TextFeature(name="txt", output_mode="multi_hot", max_tokens=4)},
+ {},
+ ),
+ ("date_season", {"date": DateFeature(name="date", add_season=True)}, {}),
+ ("passthrough", {"num2": PassthroughFeature(name="num2", dtype=tf.float32)}, {}),
+ ("crosses", {}, {"feature_crosses": [("cat1", "cat2", 8)]}),
+ (
+ "attention",
+ {},
+ {
+ "tabular_attention": True,
+ "tabular_attention_heads": 2,
+ "tabular_attention_dim": 16,
+ },
+ ),
+ (
+ "transformer_categorical",
+ {},
+ {
+ "transfo_nr_blocks": 2,
+ "transfo_nr_heads": 2,
+ "transfo_ff_units": 8,
+ "transfo_placement": "categorical",
+ },
+ ),
+ (
+ "transformer_all_features",
+ {},
+ {
+ "transfo_nr_blocks": 2,
+ "transfo_nr_heads": 2,
+ "transfo_ff_units": 8,
+ "transfo_placement": "all_features",
+ },
+ ),
+ (
+ "feature_selection",
+ {},
+ {"feature_selection_placement": "all_features", "feature_selection_units": 8},
+ ),
+ ("feature_moe", {}, {"use_feature_moe": True, "feature_moe_num_experts": 3}),
+ (
+ "global_numerical_embedding",
+ {},
+ {"use_global_numerical_embedding": True, "global_embedding_dim": 6},
+ ),
+ ("dict_mode", {}, {"output_mode": OutputModeOptions.DICT}),
+ (
+ "dict_mode_feature_moe",
+ {},
+ {
+ "output_mode": OutputModeOptions.DICT,
+ "use_feature_moe": True,
+ "feature_moe_num_experts": 3,
+ },
+ ),
+]
+
+
+def _frame() -> pd.DataFrame:
+ generator = np.random.default_rng(41)
+ return pd.DataFrame(
+ {
+ "num1": generator.normal(0.0, 1.0, ROWS),
+ "num2": generator.uniform(0.0, 10.0, ROWS),
+ "cat1": generator.choice(["a", "b", "c"], ROWS),
+ "cat2": generator.integers(0, 5, ROWS),
+ "txt": generator.choice(["red car fast", "blue sky wide"], ROWS),
+ "date": pd.date_range("2020-01-01", periods=ROWS, freq="3D").strftime(
+ "%Y-%m-%d",
+ ),
+ },
+ )
+
+
+def _feed(frame: pd.DataFrame) -> dict:
+ batch = {}
+ for column in frame.columns:
+ values = frame[column].to_numpy()
+ if values.dtype.kind == "f":
+ batch[column] = tf.constant(values.reshape(-1, 1), dtype=tf.float32)
+ elif values.dtype.kind in "iu":
+ batch[column] = tf.constant(values.reshape(-1, 1), dtype=tf.int32)
+ else:
+ batch[column] = tf.constant(values.reshape(-1, 1))
+ return batch
+
+
+def _flatten(result) -> np.ndarray:
+ if isinstance(result, dict):
+ return np.concatenate(
+ [np.asarray(result[key]).reshape(ROWS, -1) for key in sorted(result)],
+ axis=1,
+ )
+ return np.asarray(result)
+
+
+@pytest.mark.integration
+class TestEveryConfigurationReloads(unittest.TestCase):
+ """Save, load, and compare the numbers, for every option that builds."""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ cls.frame = _frame()
+ cls.data = cls.directory / "data.csv"
+ cls.frame.to_csv(cls.data, index=False)
+
+ def _round_trip(self, name: str, overrides: dict, options: dict) -> None:
+ specs = dict(BASE)
+ specs.update(overrides)
+
+ keras.backend.clear_session()
+ preprocessor = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / f"stats_{name}.json"),
+ features_specs=specs,
+ overwrite_stats=True,
+ **options,
+ )
+ preprocessor.build_preprocessor()
+
+ batch = _feed(self.frame)
+ before = _flatten(preprocessor.model(batch))
+
+ path = self.directory / f"{name}.keras"
+ preprocessor.model.save(path)
+ after = _flatten(keras.models.load_model(path)(batch))
+
+ self.assertEqual(before.shape, after.shape, name)
+ np.testing.assert_allclose(before, after, atol=1e-5, err_msg=name)
+
+ def test_every_configuration_round_trips(self) -> None:
+ for name, overrides, options in CONFIGURATIONS:
+ with self.subTest(configuration=name):
+ self._round_trip(name, overrides, options)
+
+
+@pytest.mark.unit
+class TestTfIdfKeepsItsWeights(unittest.TestCase):
+ """The rebuilt vectorizer has to compute what the adapted one computed."""
+
+ def test_it_matches_a_keras_layer_adapted_on_the_same_data(self) -> None:
+ directory = Path(tempfile.mkdtemp())
+ sentences = [
+ "red car fast",
+ "blue sky wide",
+ "red red red",
+ "green grass here",
+ ]
+ frame = pd.DataFrame(
+ {"txt": np.random.default_rng(41).choice(sentences, 200)},
+ )
+ data = directory / "data.csv"
+ frame.to_csv(data, index=False)
+
+ preprocessor = PreprocessingModel(
+ path_data=str(data),
+ features_stats_path=str(directory / "stats.json"),
+ features_specs={"txt": TextFeature(name="txt", output_mode="tf_idf")},
+ overwrite_stats=True,
+ output_mode=OutputModeOptions.DICT,
+ )
+ preprocessor.build_preprocessor()
+
+ probe = tf.constant(np.array(sentences).reshape(-1, 1))
+ ours = np.asarray(preprocessor.model({"txt": probe})["txt"])
+
+ reference = keras.layers.TextVectorization(output_mode="tf_idf")
+ reference.adapt(tf.constant(frame["txt"].to_numpy()))
+ theirs = np.asarray(reference(tf.reshape(probe, [-1])))
+
+ self.assertEqual(ours.shape, theirs.shape)
+ np.testing.assert_allclose(ours, theirs, atol=1e-4)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_feature_crosses_reach_the_output.py b/test/test_feature_crosses_reach_the_output.py
new file mode 100644
index 0000000..0819386
--- /dev/null
+++ b/test/test_feature_crosses_reach_the_output.py
@@ -0,0 +1,210 @@
+"""A cross has to appear in the output, and mean what a hashed cross means.
+
+`feature_crosses` built every cross it was given and then dropped it:
+`_group_features_by_type` looks each processed feature up in `features_specs`,
+a cross is configured with `feature_crosses` rather than by declaring a column,
+so the lookup found nothing and skipped it. In the default output mode the
+option was inert -- the model built, ran, and had exactly the width it would
+have had with no crosses at all.
+
+The existing tests asserted that the model built and that its output was not
+None, which was true throughout. These measure the cross itself: that it widens
+the output, that its value depends on both columns and on nothing else, and
+that it stays inside the number of bins it was given.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.processor import OutputModeOptions
+
+COLOURS = ["red", "green", "blue"]
+SHAPES = ["circle", "square"]
+
+SPECS = {
+ "colour": FeatureType.STRING_CATEGORICAL,
+ "shape": FeatureType.STRING_CATEGORICAL,
+ "size": FeatureType.FLOAT_NORMALIZED,
+}
+
+
+def _frame(rows: int = 60) -> pd.DataFrame:
+ return pd.DataFrame(
+ {
+ "colour": [COLOURS[i % len(COLOURS)] for i in range(rows)],
+ "shape": [SHAPES[i % len(SHAPES)] for i in range(rows)],
+ "size": np.linspace(0.0, 10.0, rows),
+ },
+ )
+
+
+def _call(model: tf.keras.Model, frame: pd.DataFrame) -> np.ndarray:
+ return np.asarray(
+ model({name: tf.constant(frame[name].values.reshape(-1, 1)) for name in frame}),
+ )
+
+
+@pytest.mark.unit
+class TestFeatureCrossesReachTheOutput(unittest.TestCase):
+ """The cross has to survive as far as the tensor the caller receives."""
+
+ def setUp(self) -> None:
+ self.directory = Path(tempfile.mkdtemp())
+ self.frame = _frame()
+ self.data = self.directory / "data.csv"
+ self.frame.to_csv(self.data, index=False)
+
+ def _build(self, **kwargs) -> tf.keras.Model:
+ model = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / f"stats_{len(kwargs)}.json"),
+ features_specs=SPECS,
+ overwrite_stats=True,
+ **kwargs,
+ )
+ return model.build_preprocessor()["model"]
+
+ def test_a_cross_makes_the_output_wider(self) -> None:
+ """One crossed column per cross, on top of the declared features."""
+ plain = _call(self._build(), self.frame).shape[1]
+ crossed = _call(
+ self._build(feature_crosses=[("colour", "shape", 16)]),
+ self.frame,
+ ).shape[1]
+ self.assertEqual(
+ crossed,
+ plain + 1,
+ "the cross never reached the concatenated output",
+ )
+
+ def test_two_crosses_add_two_columns(self) -> None:
+ plain = _call(self._build(), self.frame).shape[1]
+ crossed = _call(
+ self._build(
+ feature_crosses=[("colour", "shape", 16), ("shape", "colour", 8)],
+ ),
+ self.frame,
+ ).shape[1]
+ self.assertEqual(crossed, plain + 2)
+
+ def test_the_cross_is_a_function_of_both_columns_and_of_nothing_else(self) -> None:
+ """Every distinct pair gets its own bin; other columns do not move it."""
+ width = _call(self._build(), self.frame).shape[1]
+ model = self._build(feature_crosses=[("colour", "shape", 64)])
+ probe = pd.DataFrame(
+ {
+ "colour": ["red", "red", "green", "green", "red"],
+ "shape": ["circle", "square", "circle", "square", "circle"],
+ # The last row repeats the first pair with a different size.
+ "size": [1.0, 1.0, 1.0, 1.0, 9.0],
+ },
+ )
+ crossed = _call(model, probe)[:, width:]
+
+ self.assertEqual(
+ len({tuple(row) for row in crossed[:4]}),
+ 4,
+ "distinct (colour, shape) pairs collided into one bin",
+ )
+ np.testing.assert_allclose(
+ crossed[0],
+ crossed[4],
+ err_msg="the cross moved when a column it does not cross changed",
+ )
+
+ def test_the_cross_stays_inside_the_number_of_bins(self) -> None:
+ rows = 120
+ frame = pd.DataFrame(
+ {
+ "colour": [f"colour_{index}" for index in range(rows)],
+ "shape": [f"shape_{index % 7}" for index in range(rows)],
+ "size": np.linspace(0.0, 1.0, rows),
+ },
+ )
+ data = self.directory / "wide.csv"
+ frame.to_csv(data, index=False)
+
+ def build(**kwargs) -> tf.keras.Model:
+ model = PreprocessingModel(
+ path_data=str(data),
+ features_stats_path=str(self.directory / f"wide_{len(kwargs)}.json"),
+ features_specs=SPECS,
+ overwrite_stats=True,
+ **kwargs,
+ )
+ return model.build_preprocessor()["model"]
+
+ width = _call(build(), frame).shape[1]
+ crossed = _call(build(feature_crosses=[("colour", "shape", 5)]), frame)[
+ :,
+ width:,
+ ]
+ self.assertGreaterEqual(crossed.min(), 0.0)
+ self.assertLess(crossed.max(), 5.0)
+ self.assertGreater(
+ len(np.unique(crossed)),
+ 1,
+ "every pair hashed to the same bin",
+ )
+
+ def test_the_cross_is_a_key_in_dict_mode(self) -> None:
+ model = self._build(
+ feature_crosses=[("colour", "shape", 16)],
+ output_mode=OutputModeOptions.DICT,
+ )
+ outputs = model(
+ {
+ name: tf.constant(self.frame[name].values.reshape(-1, 1))
+ for name in self.frame
+ },
+ )
+ self.assertIn("colour_x_shape", outputs)
+
+
+@pytest.mark.unit
+class TestFeatureCrossesRefuseWhatTheyCannotDo(unittest.TestCase):
+ """A cross that cannot run has to say so while the config is still in view."""
+
+ def setUp(self) -> None:
+ self.directory = Path(tempfile.mkdtemp())
+ self.data = self.directory / "data.csv"
+ _frame().to_csv(self.data, index=False)
+
+ def _build(self, crosses: list) -> tf.keras.Model:
+ model = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / "stats.json"),
+ features_specs=SPECS,
+ overwrite_stats=True,
+ feature_crosses=crosses,
+ )
+ return model.build_preprocessor()["model"]
+
+ def test_crossing_a_float_column_is_refused_at_build_time(self) -> None:
+ """`HashedCrossing` takes integers and strings; floats used to build.
+
+ The model came back looking correct -- summary, output width, the lot --
+ and raised on the first batch it was ever given.
+ """
+ with self.assertRaises(ValueError) as raised:
+ self._build([("colour", "size", 8)])
+ message = str(raised.exception)
+ self.assertIn("size", message)
+ self.assertIn("float32", message)
+
+ def test_crossing_an_undeclared_feature_names_it(self) -> None:
+ with self.assertRaises(ValueError) as raised:
+ self._build([("colour", "not_a_column", 8)])
+ self.assertIn("not_a_column", str(raised.exception))
+ self.assertIn("features_specs", str(raised.exception))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_float32_collapse_is_reported.py b/test/test_float32_collapse_is_reported.py
new file mode 100644
index 0000000..1a57008
--- /dev/null
+++ b/test/test_float32_collapse_is_reported.py
@@ -0,0 +1,142 @@
+"""A column float32 cannot hold has to be called out, not silently flattened.
+
+Everything after the CSV reader is float32, which carries about seven
+significant digits. A column whose values sit far from zero and vary only
+slightly loses that variation on the way in: Unix timestamps in seconds are 128
+apart in float32 near 1.6e9, so a column spread over half a minute arrives as
+one or two distinct numbers. Normalization then works perfectly on what is
+left, the reported standard deviation is 1.0, and the feature the model sees is
+a constant.
+
+The numbers are not wrong -- they describe the values the file really produced.
+The loss is upstream and cannot be undone here, so this is a warning naming the
+column, and it has to fire for exactly the columns that are hurt.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+from loguru import logger
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.stats import DatasetStatistics
+
+ROWS = 3000
+
+
+def _frame() -> pd.DataFrame:
+ generator = np.random.default_rng(7)
+ return pd.DataFrame(
+ {
+ # Unix seconds, spread over well under a minute.
+ "epoch_seconds": generator.normal(1.6e9, 30.0, ROWS),
+ # Days since the epoch: far from zero, but coarse enough to survive.
+ "epoch_days": generator.normal(19_000.0, 300.0, ROWS),
+ # Money in cents, up to ten million.
+ "price_cents": generator.integers(0, 10**7, ROWS).astype(float),
+ "ordinary": generator.normal(0.0, 1.0, ROWS),
+ "constant": np.full(ROWS, 42.0),
+ },
+ )
+
+
+class _Captured:
+ """Collect loguru warnings raised while the block runs."""
+
+ def __init__(self) -> None:
+ self.messages: list[str] = []
+ self._handler: int | None = None
+
+ def __enter__(self) -> "_Captured":
+ self._handler = logger.add(
+ lambda message: self.messages.append(str(message)),
+ level="WARNING",
+ )
+ return self
+
+ def __exit__(self, *_) -> None:
+ if self._handler is not None:
+ logger.remove(self._handler)
+
+
+@pytest.mark.unit
+class TestFloat32CollapseIsReported(unittest.TestCase):
+ """The warning has to name the column that is lost, and only that one."""
+
+ def setUp(self) -> None:
+ self.directory = Path(tempfile.mkdtemp())
+ self.frame = _frame()
+ self.data = self.directory / "data.csv"
+ self.frame.to_csv(self.data, index=False)
+
+ def _run_statistics(self) -> _Captured:
+ with _Captured() as captured:
+ DatasetStatistics(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / "stats.json"),
+ features_specs={
+ name: FeatureType.FLOAT_NORMALIZED for name in self.frame
+ },
+ overwrite_stats=True,
+ ).main()
+ return captured
+
+ def test_the_column_float32_destroys_is_named(self) -> None:
+ warnings = [m for m in self._run_statistics().messages if "float32" in m]
+ self.assertEqual(len(warnings), 1, warnings)
+ self.assertIn("epoch_seconds", warnings[0])
+
+ def test_columns_float32_can_hold_are_left_alone(self) -> None:
+ warnings = [m for m in self._run_statistics().messages if "float32" in m]
+ joined = " ".join(warnings)
+ for name in ("epoch_days", "price_cents", "ordinary", "constant"):
+ self.assertNotIn(name, joined)
+
+
+@pytest.mark.unit
+class TestTheWarningDescribesWhatActuallyHappens(unittest.TestCase):
+ """The claim in the message has to hold end to end."""
+
+ def test_the_named_column_really_does_collapse(self) -> None:
+ directory = Path(tempfile.mkdtemp())
+ frame = _frame()
+ data = directory / "data.csv"
+ frame.to_csv(data, index=False)
+
+ preprocessor = PreprocessingModel(
+ path_data=str(data),
+ features_stats_path=str(directory / "stats.json"),
+ features_specs={name: FeatureType.FLOAT_NORMALIZED for name in frame},
+ overwrite_stats=True,
+ output_mode="dict",
+ )
+ preprocessor.build_preprocessor()
+ outputs = preprocessor.model(
+ {
+ name: tf.constant(frame[name].values.reshape(-1, 1), dtype=tf.float32)
+ for name in frame
+ },
+ )
+
+ collapsed = np.unique(np.asarray(outputs["epoch_seconds"]))
+ self.assertLess(
+ len(collapsed),
+ 10,
+ "the warning claims this column collapses, and it did not",
+ )
+ for name in ("epoch_days", "price_cents", "ordinary"):
+ kept = np.unique(np.asarray(outputs[name]))
+ self.assertGreater(
+ len(kept),
+ ROWS // 2,
+ f"{name} was not warned about but lost most of its values",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_max_tokens_caps_the_vocabulary.py b/test/test_max_tokens_caps_the_vocabulary.py
new file mode 100644
index 0000000..e361970
--- /dev/null
+++ b/test/test_max_tokens_caps_the_vocabulary.py
@@ -0,0 +1,112 @@
+"""`max_tokens` has to cap the vocabulary, which is the only reason to set it.
+
+The statistics pass collects every distinct word in the column and the layer was
+handed all of them alongside the cap. Keras refuses that outright -- "Attempted
+to set a vocabulary larger than the maximum vocab size" -- so asking for a
+vocabulary smaller than the data holds could not build at all, while asking for
+a larger one worked and did nothing. The option was usable only where it had no
+effect.
+
+The statistics carry no word counts to choose the survivors with, so when the
+cap bites the vectorizer adapts on the column instead, which is how `max_tokens`
+picks its words everywhere else in Keras: by frequency.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.features import TextFeature
+from kdp.processor import OutputModeOptions
+
+ROWS = 150
+SENTENCES = [
+ "red car fast",
+ "blue sky wide open",
+ "green grass",
+ "red red red",
+]
+
+
+@pytest.mark.unit
+class TestMaxTokensCapsTheVocabulary(unittest.TestCase):
+ """The output width has to follow the cap, above and below the data."""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ generator = np.random.default_rng(13)
+ cls.frame = pd.DataFrame({"txt": generator.choice(SENTENCES, ROWS)})
+ cls.data = cls.directory / "data.csv"
+ cls.frame.to_csv(cls.data, index=False)
+ # "red car fast blue sky wide open green grass" -- nine distinct words.
+ cls.distinct_words = len(
+ {word for sentence in SENTENCES for word in sentence.split()},
+ )
+
+ def _width(self, **kwargs) -> int:
+ feature = TextFeature(
+ name="txt",
+ feature_type=FeatureType.TEXT,
+ output_mode="multi_hot",
+ **kwargs,
+ )
+ preprocessor = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / f"stats_{kwargs}.json"),
+ features_specs={"txt": feature},
+ overwrite_stats=True,
+ output_mode=OutputModeOptions.DICT,
+ )
+ preprocessor.build_preprocessor()
+ result = preprocessor.model(
+ {"txt": tf.constant(self.frame["txt"].values[:5].reshape(-1, 1))},
+ )
+ return int(np.asarray(result["txt"]).shape[1])
+
+ def test_a_cap_below_the_data_is_honoured(self) -> None:
+ for cap in (3, 5, 7):
+ with self.subTest(max_tokens=cap):
+ self.assertLessEqual(self._width(max_tokens=cap), cap)
+
+ def test_a_cap_below_the_data_still_produces_a_usable_vocabulary(self) -> None:
+ self.assertGreater(self._width(max_tokens=4), 1)
+
+ def test_a_cap_above_the_data_keeps_every_word(self) -> None:
+ width = self._width(max_tokens=500)
+ self.assertGreaterEqual(width, self.distinct_words)
+
+ def test_the_int_mode_reserves_its_two_slots(self) -> None:
+ """`int` keeps a padding slot as well as the out-of-vocabulary one."""
+ feature = TextFeature(
+ name="txt",
+ feature_type=FeatureType.TEXT,
+ output_mode="int",
+ output_sequence_length=6,
+ max_tokens=4,
+ )
+ preprocessor = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / "stats_int.json"),
+ features_specs={"txt": feature},
+ overwrite_stats=True,
+ output_mode=OutputModeOptions.DICT,
+ )
+ preprocessor.build_preprocessor()
+ result = np.asarray(
+ preprocessor.model(
+ {"txt": tf.constant(self.frame["txt"].values[:5].reshape(-1, 1))},
+ )["txt"],
+ )
+ self.assertEqual(result.shape[1], 6)
+ self.assertLess(result.max(), 4, "a token index landed outside max_tokens")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_path_data_selects_one_file.py b/test/test_path_data_selects_one_file.py
new file mode 100644
index 0000000..47018f0
--- /dev/null
+++ b/test/test_path_data_selects_one_file.py
@@ -0,0 +1,187 @@
+"""`path_data` pointing at a file has to mean that file.
+
+The path was replaced with `/*.csv`, so naming one CSV computed the
+statistics over every CSV beside it. Every example in the documentation --
+`path_data="data/my_data.csv"` in the README included -- passes a single file,
+and a project laid out as `data/train.csv` and `data/test.csv` therefore had its
+test set folded into the training statistics. Nothing raised: the numbers came
+back, drawn from the wrong rows.
+"""
+
+import json
+import zlib
+import tempfile
+import unittest
+from pathlib import Path
+
+import keras
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.stats import DatasetStatistics
+
+ROWS = 400
+
+
+def _write(directory, name, mean):
+ """A CSV whose column sits at a known, distinctive mean."""
+ # `hash()` on a string is salted per process, so seeding from it drew
+ # different numbers in every pytest-xdist worker.
+ frame = pd.DataFrame(
+ {
+ "x": np.random.default_rng(zlib.crc32(name.encode())).normal(
+ mean,
+ 1.0,
+ ROWS,
+ ),
+ },
+ )
+ path = Path(directory) / name
+ frame.to_csv(path, index=False)
+ return path, float(frame["x"].mean())
+
+
+@pytest.mark.unit
+class TestPathDataSelectsOneFile(unittest.TestCase):
+ """What a file, a directory and a glob each select."""
+
+ def test_naming_a_file_ignores_its_neighbours(self):
+ with tempfile.TemporaryDirectory() as directory:
+ train, train_mean = _write(directory, "train.csv", mean=10.0)
+ _write(directory, "test.csv", mean=1000.0)
+
+ stats_path = Path(directory) / "stats.json"
+ keras.backend.clear_session()
+ preprocessor = PreprocessingModel(
+ path_data=str(train),
+ features_specs={"x": FeatureType.FLOAT_NORMALIZED},
+ features_stats_path=str(stats_path),
+ overwrite_stats=True,
+ )
+ preprocessor.build_preprocessor()
+ stats = json.loads(stats_path.read_text())["numeric_stats"]["x"]
+
+ # Only train.csv: its own row count and its own mean, nowhere near the
+ # 505 that averaging the two files would give.
+ self.assertEqual(int(stats["count"]), ROWS)
+ self.assertAlmostEqual(float(stats["mean"]), train_mean, places=3)
+
+ def test_naming_a_directory_reads_every_csv_in_it(self):
+ with tempfile.TemporaryDirectory() as directory:
+ _, mean_a = _write(directory, "a.csv", mean=10.0)
+ _, mean_b = _write(directory, "b.csv", mean=20.0)
+
+ stats_path = Path(directory) / "stats.json"
+ keras.backend.clear_session()
+ preprocessor = PreprocessingModel(
+ path_data=directory,
+ features_specs={"x": FeatureType.FLOAT_NORMALIZED},
+ features_stats_path=str(stats_path),
+ overwrite_stats=True,
+ )
+ preprocessor.build_preprocessor()
+ stats = json.loads(stats_path.read_text())["numeric_stats"]["x"]
+
+ self.assertEqual(int(stats["count"]), 2 * ROWS)
+ # Against the mean the two files actually hold, not the 15.0 they were
+ # drawn around: with 400 rows each the sample mean wanders by about
+ # 0.035, which is wider than the tolerance this once asserted.
+ self.assertAlmostEqual(
+ float(stats["mean"]),
+ (mean_a + mean_b) / 2.0,
+ places=3,
+ )
+
+ def test_the_three_shapes_a_path_can_take(self):
+ with tempfile.TemporaryDirectory() as directory:
+ path, _ = _write(directory, "one.csv", mean=5.0)
+ self.assertEqual(
+ DatasetStatistics._get_csv_file_pattern(str(path)),
+ str(path),
+ )
+ self.assertEqual(
+ DatasetStatistics._get_csv_file_pattern(directory),
+ str(Path(directory) / "*.csv"),
+ )
+ glob = str(Path(directory) / "*.csv")
+ self.assertEqual(DatasetStatistics._get_csv_file_pattern(glob), glob)
+
+ def test_a_path_that_does_not_exist_is_reported(self):
+ """`tf.data` answers a missing file with an empty dataset, and the
+ statistics then come from nothing at all."""
+ with tempfile.TemporaryDirectory() as directory:
+ missing = str(Path(directory) / "absent.csv")
+ with self.assertRaises(FileNotFoundError) as caught:
+ DatasetStatistics._get_csv_file_pattern(missing)
+ self.assertIn("absent.csv", str(caught.exception))
+
+ def test_two_models_built_in_one_process_keep_their_own_statistics(self):
+ """The symptom this surfaced as: the second model's numbers were the
+ two datasets pooled together."""
+ with tempfile.TemporaryDirectory() as directory:
+ first, first_mean = _write(directory, "first.csv", mean=10.0)
+ second, second_mean = _write(directory, "second.csv", mean=900.0)
+
+ seen = []
+ for index, (path, expected) in enumerate(
+ ((first, first_mean), (second, second_mean)),
+ ):
+ stats_path = Path(directory) / f"stats{index}.json"
+ keras.backend.clear_session()
+ preprocessor = PreprocessingModel(
+ path_data=str(path),
+ features_specs={"x": FeatureType.FLOAT_NORMALIZED},
+ features_stats_path=str(stats_path),
+ overwrite_stats=True,
+ )
+ preprocessor.build_preprocessor()
+ stats = json.loads(stats_path.read_text())["numeric_stats"]["x"]
+ seen.append((int(stats["count"]), float(stats["mean"]), expected))
+
+ for count, mean, expected in seen:
+ self.assertEqual(count, ROWS)
+ self.assertAlmostEqual(mean, expected, places=3)
+
+
+@pytest.mark.unit
+class TestLargeMagnitudeColumnsSurviveNormalisation(unittest.TestCase):
+ """IDs, epoch timestamps and amounts in cents all live far from zero."""
+
+ COLUMNS = {
+ "ordinary": (50.0, 12.0),
+ "large ids": (1e8, 1e3),
+ "amounts in cents": (1e7, 5e2),
+ "epoch seconds": (1.7e9, 1e6),
+ }
+
+ def test_a_column_far_from_zero_still_standardises(self):
+ rng = np.random.default_rng(1)
+ for label, (centre, spread) in self.COLUMNS.items():
+ with self.subTest(column=label), tempfile.TemporaryDirectory() as directory:
+ column = rng.normal(centre, spread, 2000)
+ path = Path(directory) / "d.csv"
+ pd.DataFrame({"x": column}).to_csv(path, index=False)
+
+ keras.backend.clear_session()
+ preprocessor = PreprocessingModel(
+ path_data=str(path),
+ features_specs={"x": FeatureType.FLOAT_NORMALIZED},
+ features_stats_path=str(Path(directory) / "s.json"),
+ overwrite_stats=True,
+ )
+ preprocessor.build_preprocessor()
+ output = np.asarray(
+ preprocessor.model(
+ {"x": tf.constant(column.reshape(-1, 1), dtype=tf.float32)},
+ training=False,
+ ),
+ )
+
+ self.assertFalse(np.isnan(output).any(), label)
+ # The whole point of FLOAT_NORMALIZED: zero mean, unit variance.
+ # These columns used to come out as a constant.
+ self.assertLess(abs(float(output.mean())), 0.05, label)
+ self.assertAlmostEqual(float(output.std()), 1.0, delta=0.05, msg=label)
diff --git a/test/test_predict_accepts_what_it_documents.py b/test/test_predict_accepts_what_it_documents.py
new file mode 100644
index 0000000..6c4efa4
--- /dev/null
+++ b/test/test_predict_accepts_what_it_documents.py
@@ -0,0 +1,267 @@
+"""`predict()` has to take the shapes its own docstring offers.
+
+The docstring says the data "can be pandas DataFrame, dict, or TensorFlow
+dataset". A DataFrame went straight to Keras, which reads a frame as a block of
+floats and raised `could not convert string to float` on the first categorical
+column. A dict of flat lists -- the natural spelling, and exactly what
+`InferenceFormatter` produces -- converted to shape (N,) where every input is
+declared (N, 1), and failed with `as_list() is not defined on an unknown
+TensorShape`. Both carried the right values; only the container was wrong.
+
+Every accepted spelling has to give the same numbers as calling the model with
+tensors, because that is the whole claim.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+
+ROWS = 24
+
+SPECS = {
+ "age": FeatureType.FLOAT_NORMALIZED,
+ "city": FeatureType.STRING_CATEGORICAL,
+ "tier": FeatureType.INTEGER_CATEGORICAL,
+ "signed": FeatureType.DATE,
+}
+
+
+def _frame() -> pd.DataFrame:
+ generator = np.random.default_rng(0)
+ return pd.DataFrame(
+ {
+ "age": generator.normal(40.0, 8.0, ROWS),
+ "city": generator.choice(["paris", "lisbon", "oslo"], ROWS),
+ "tier": generator.integers(0, 4, ROWS),
+ "signed": pd.date_range("2021-01-01", periods=ROWS, freq="9D").strftime(
+ "%Y-%m-%d",
+ ),
+ },
+ )
+
+
+@pytest.mark.unit
+class TestPredictAcceptsWhatItDocuments(unittest.TestCase):
+ """Every documented container has to reach the same tensor."""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ cls.frame = _frame()
+ data = cls.directory / "data.csv"
+ cls.frame.to_csv(data, index=False)
+ cls.preprocessor = PreprocessingModel(
+ path_data=str(data),
+ features_stats_path=str(cls.directory / "stats.json"),
+ features_specs=SPECS,
+ overwrite_stats=True,
+ )
+ cls.preprocessor.build_preprocessor()
+ cls.reference = np.asarray(
+ cls.preprocessor.model(
+ {
+ name: tf.constant(cls.frame[name].values.reshape(-1, 1))
+ for name in cls.frame
+ },
+ ),
+ )
+
+ def _assert_matches(self, payload) -> None:
+ got = np.asarray(self.preprocessor.predict(payload, verbose=0))
+ self.assertEqual(got.shape, self.reference.shape)
+ np.testing.assert_allclose(got, self.reference, atol=1e-5)
+
+ def test_a_dataframe(self) -> None:
+ self._assert_matches(self.frame)
+
+ def test_a_dict_of_flat_lists(self) -> None:
+ self._assert_matches({name: self.frame[name].tolist() for name in self.frame})
+
+ def test_a_dict_of_column_lists(self) -> None:
+ self._assert_matches(
+ {name: [[value] for value in self.frame[name]] for name in self.frame},
+ )
+
+ def test_a_dict_of_numpy_columns(self) -> None:
+ self._assert_matches(
+ {name: self.frame[name].values.reshape(-1, 1) for name in self.frame},
+ )
+
+ def test_a_tensorflow_dataset(self) -> None:
+ dataset = tf.data.Dataset.from_tensor_slices(dict(self.frame)).batch(8)
+ self._assert_matches(dataset)
+
+ def test_a_single_row_as_a_dataframe(self) -> None:
+ got = np.asarray(self.preprocessor.predict(self.frame.iloc[:1], verbose=0))
+ np.testing.assert_allclose(got, self.reference[:1], atol=1e-5)
+
+ def test_what_the_inference_formatter_produces(self) -> None:
+ """The formatter's own output is flat, and has to be accepted."""
+ from kdp import InferenceFormatter
+
+ formatter = InferenceFormatter(self.preprocessor)
+ prepared = formatter.prepare_inference_data(self.frame, to_tensors=True)
+ self._assert_matches(prepared)
+
+
+@pytest.mark.unit
+class TestAGapInTheFrameDoesNotStopIt(unittest.TestCase):
+ """A frame with missing values is what real data looks like.
+
+ pandas gives a column mixing NaN with strings the object dtype, and Keras
+ refuses that outright with `Invalid dtype: object`. `InferenceFormatter`
+ already decided each column's type from the values actually present;
+ `predict()` did not, so the two documented ways in disagreed.
+ """
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ cls.frame = _frame()
+ data = cls.directory / "data.csv"
+ cls.frame.to_csv(data, index=False)
+ cls.preprocessor = PreprocessingModel(
+ path_data=str(data),
+ features_stats_path=str(cls.directory / "stats.json"),
+ features_specs=SPECS,
+ overwrite_stats=True,
+ )
+ cls.preprocessor.build_preprocessor()
+
+ def _with_gaps(self) -> pd.DataFrame:
+ # Not the date: that column is the one kind of gap the library refuses,
+ # and the test below is about the message it refuses with.
+ gappy = self.frame.iloc[:5].copy()
+ gappy.loc[gappy.index[0], "age"] = np.nan
+ gappy.loc[gappy.index[1], "city"] = None
+ gappy.loc[gappy.index[2], "tier"] = np.nan
+ return gappy
+
+ def test_a_frame_with_gaps_goes_through(self) -> None:
+ result = np.asarray(self.preprocessor.predict(self._with_gaps(), verbose=0))
+ self.assertEqual(result.shape[0], 5)
+
+ def test_a_missing_category_does_not_produce_a_non_finite_number(self) -> None:
+ """Only the numeric gap may carry NaN; the rest map to their OOV slot."""
+ gappy = self.frame.iloc[:4].copy()
+ gappy.loc[gappy.index[0], "city"] = None
+ gappy.loc[gappy.index[1], "tier"] = np.nan
+ result = np.asarray(self.preprocessor.predict(gappy, verbose=0))
+ self.assertTrue(np.isfinite(result).all())
+
+ def test_a_missing_date_says_what_is_wrong_and_which_value(self) -> None:
+ """A date column cannot hold nulls, and has to say so where it fails.
+
+ The formatter turns any missing string into the empty string, which the
+ date parser refuses. It used to refuse it as a TensorFlow graph error
+ naming an internal assertion node and nothing else.
+ """
+ gappy = self.frame.iloc[:3].copy()
+ gappy.loc[gappy.index[1], "signed"] = None
+ with self.assertRaises(Exception) as raised: # noqa: B017 -- tf error type
+ self.preprocessor.predict(gappy, verbose=0)
+ message = " ".join(str(raised.exception).split())
+ self.assertIn("date column cannot hold nulls", message)
+
+ def test_a_missing_number_stays_inside_its_own_column(self) -> None:
+ gappy = self.frame.iloc[:4].copy()
+ gappy.loc[gappy.index[1], "age"] = np.nan
+ result = np.asarray(self.preprocessor.predict(gappy, verbose=0))
+ rows = {int(row) for row in np.argwhere(~np.isfinite(result))[:, 0]}
+ self.assertEqual(rows, {1}, "the gap spread beyond the row it was in")
+ self.assertEqual(int((~np.isfinite(result)).sum()), 1)
+
+ def test_both_documented_paths_agree_about_a_gap(self) -> None:
+ from kdp import InferenceFormatter
+
+ gappy = self._with_gaps()
+ through_predict = np.asarray(self.preprocessor.predict(gappy, verbose=0))
+ prepared = InferenceFormatter(self.preprocessor).prepare_inference_data(
+ gappy,
+ to_tensors=True,
+ )
+ through_formatter = np.asarray(self.preprocessor.model(prepared))
+ np.testing.assert_array_equal(
+ np.isfinite(through_predict),
+ np.isfinite(through_formatter),
+ )
+ finite = np.isfinite(through_predict)
+ np.testing.assert_allclose(
+ through_predict[finite],
+ through_formatter[finite],
+ atol=1e-5,
+ )
+
+
+@pytest.mark.unit
+class TestPredictIsStableAcrossHowItIsCalled(unittest.TestCase):
+ """The same row has to come back the same, however it is fed in."""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ cls.frame = _frame()
+ data = cls.directory / "data.csv"
+ cls.frame.to_csv(data, index=False)
+ preprocessor = PreprocessingModel(
+ path_data=str(data),
+ features_stats_path=str(cls.directory / "stats.json"),
+ features_specs=SPECS,
+ overwrite_stats=True,
+ )
+ preprocessor.build_preprocessor()
+ cls.model = preprocessor.model
+ cls.reference = cls._call(cls.frame)
+
+ @classmethod
+ def _call(cls, frame: pd.DataFrame) -> np.ndarray:
+ return np.asarray(
+ cls.model(
+ {
+ name: tf.constant(frame[name].values.reshape(-1, 1))
+ for name in frame
+ },
+ ),
+ )
+
+ def test_batch_size_does_not_change_a_row(self) -> None:
+ pieces = [self._call(self.frame.iloc[i : i + 5]) for i in range(0, ROWS, 5)]
+ np.testing.assert_allclose(
+ np.concatenate(pieces),
+ self.reference,
+ atol=1e-5,
+ )
+
+ def test_row_order_does_not_change_a_row(self) -> None:
+ order = np.random.default_rng(1).permutation(ROWS)
+ shuffled = self._call(self.frame.iloc[order].reset_index(drop=True))
+ np.testing.assert_allclose(shuffled, self.reference[order], atol=1e-5)
+
+ def test_calling_twice_gives_the_same_answer(self) -> None:
+ np.testing.assert_array_equal(self._call(self.frame), self.reference)
+
+ def test_a_single_row_matches_its_place_in_the_batch(self) -> None:
+ np.testing.assert_allclose(
+ self._call(self.frame.iloc[:1]),
+ self.reference[:1],
+ atol=1e-5,
+ )
+
+ def test_values_never_seen_in_training_still_produce_finite_numbers(self) -> None:
+ unseen = self.frame.iloc[:4].copy()
+ unseen["city"] = ["atlantis", "paris", "", "oslo"]
+ unseen["tier"] = [999, 0, -7, 1]
+ result = self._call(unseen)
+ self.assertEqual(result.shape, (4, self.reference.shape[1]))
+ self.assertTrue(np.isfinite(result).all())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_processor.py b/test/test_processor.py
index 0133410..d4430e7 100644
--- a/test/test_processor.py
+++ b/test/test_processor.py
@@ -549,10 +549,21 @@ def test_date_feature_preprocessing(self):
)
test_data.to_csv(self._path_data, index=False)
- # Verify preprocessing works
- dataset = tf.data.Dataset.from_tensor_slices(dict(test_data))
- preprocessed = model.batch_predict(dataset)
- self.assertIsNotNone(preprocessed)
+ # Verify preprocessing works. `batch_predict` is a generator, so
+ # holding onto it and asserting it is not None never ran the model.
+ dataset = tf.data.Dataset.from_tensor_slices(dict(test_data)).batch(4)
+ batches = list(model.batch_predict(dataset))
+ self.assertTrue(batches)
+ # Dict output mode: each batch is a mapping of feature name to tensor.
+ rows = 0
+ for batch in batches:
+ tensors = list(batch.values()) if isinstance(batch, dict) else [batch]
+ widths = {int(np.asarray(tensor).shape[0]) for tensor in tensors}
+ self.assertEqual(len(widths), 1, f"features disagree on rows: {widths}")
+ rows += widths.pop()
+ for tensor in tensors:
+ self.assertTrue(np.isfinite(np.asarray(tensor)).all())
+ self.assertEqual(rows, len(test_data))
def test_caching_functionality(self):
"""Test the caching functionality of preprocessed features."""
@@ -695,16 +706,24 @@ def test_end_to_end_feature_combinations(self):
# Create a small batch of test data
test_data = generate_fake_data(test_case["features"], num_rows=5)
- dataset = tf.data.Dataset.from_tensor_slices(dict(test_data))
+ dataset = tf.data.Dataset.from_tensor_slices(dict(test_data)).batch(5)
- # Test preprocessing
- preprocessed = ppr.batch_predict(dataset)
- self.assertIsNotNone(preprocessed)
+ # Test preprocessing. `batch_predict` is a generator: the
+ # assertion below used to be made against the generator object
+ # rather than anything it produced, so the model never ran.
+ batches = [np.asarray(batch) for batch in ppr.batch_predict(dataset)]
+ self.assertTrue(batches)
+ preprocessed = np.concatenate(batches)
+ self.assertEqual(preprocessed.shape[0], 5)
+ self.assertTrue(np.isfinite(preprocessed).all())
# Additional checks based on feature combination
if "date1" in test_case["features"]:
date_feature = test_case["features"]["date1"]
- if getattr(date_feature, "add_season", False):
+ # `add_season` lives in the feature's kwargs, not as an
+ # attribute, so `getattr` here always answered False and
+ # this check never ran either.
+ if date_feature.kwargs.get("add_season", False):
# Check if output shape includes seasonal encoding
self.assertGreaterEqual(
preprocessed.shape[-1], 4
@@ -737,9 +756,15 @@ def test_date_feature_variations(self):
),
},
{
+ # Dates are read as year, then month, then day. This used to
+ # ask for "%m/%d/%Y" and feed month-first strings, which the
+ # parser cannot read -- the case passed because nothing ever
+ # consumed the generator below.
"name": "custom_format_date",
"config": DateFeature(
- name="date", feature_type=FeatureType.DATE, date_format="%m/%d/%Y"
+ name="date",
+ feature_type=FeatureType.DATE,
+ date_format="%Y-%m-%d %H:%M:%S",
),
},
{
@@ -771,13 +796,18 @@ def test_date_feature_variations(self):
# Test with specific date formats
if config["name"] == "custom_format_date":
- test_data = pd.DataFrame({"date": ["01/15/2023", "12/31/2022"]})
+ test_data = pd.DataFrame(
+ {"date": ["2023-01-15 08:30:00", "2022-12-31 23:59:59"]},
+ )
else:
test_data = pd.DataFrame({"date": ["2023-01-15", "2022-12-31"]})
- dataset = tf.data.Dataset.from_tensor_slices(dict(test_data))
- preprocessed = ppr.batch_predict(dataset)
- self.assertIsNotNone(preprocessed)
+ dataset = tf.data.Dataset.from_tensor_slices(dict(test_data)).batch(2)
+ batches = [np.asarray(batch) for batch in ppr.batch_predict(dataset)]
+ self.assertTrue(batches)
+ preprocessed = np.concatenate(batches)
+ self.assertEqual(preprocessed.shape[0], 2)
+ self.assertTrue(np.isfinite(preprocessed).all())
def test_preprocessor_with_tabular_attention(self):
"""Test end-to-end preprocessing with TabularAttention."""
@@ -926,7 +956,7 @@ def test_preprocessor_attention_with_feature_crosses(self):
tabular_attention_heads=4,
tabular_attention_dim=64,
tabular_attention_dropout=0.1,
- feature_crosses=[("num1", "num2", 5), ("cat1", "cat2", 5)],
+ feature_crosses=[("cat1", "cat2", 5)],
output_mode=OutputModeOptions.CONCAT,
)
@@ -1365,7 +1395,11 @@ def test_preprocessor_all_features_with_transformer_and_attention_with_distribut
self.assertIsNotNone(preprocessed)
# Check output dimensions
- self.assertEqual(len(preprocessed.shape), 3) # (batch_size, d_model)
+ # `transfo_placement="all_features"` was the one configuration that came
+ # out rank 3: a transformer block adds a sequence axis and this branch
+ # never took it off again. Every other placement, this test's own
+ # comment, and its three siblings all say (rows, width).
+ self.assertEqual(len(preprocessed.shape), 2) # (batch_size, d_model)
self.assertEqual(preprocessed.shape[-1], 23) # Example dimension
def test_preprocessor_all_features_with_transformer_and_attention_with_distribution_aware_v4(
@@ -1791,9 +1825,14 @@ def test_preprocessor_parameter_combinations(self):
if test_case["output_mode"] == OutputModeOptions.CONCAT:
if test_case["tabular_attention"]:
- # Check output dimensions for concatenated output with attention
+ # Check output dimensions for concatenated output with attention.
+ # This case is attention plus `transfo_placement="all_features"`,
+ # the one combination that used to come back rank 3
+ # because that branch never removed the sequence axis a
+ # transformer block adds. Attention on its own has
+ # always flattened, like every other placement.
self.assertEqual(
- len(preprocessed.shape), 3
+ len(preprocessed.shape), 2
) # (batch_size, d_model)
self.assertEqual(
preprocessed.shape[-1], test_case["tabular_attention_dim"]
diff --git a/test/test_stats_accumulators.py b/test/test_stats_accumulators.py
index f9c4f54..471ef34 100644
--- a/test/test_stats_accumulators.py
+++ b/test/test_stats_accumulators.py
@@ -13,7 +13,7 @@
import pytest
import tensorflow as tf
-from kdp.stats import DateAccumulator, TextAccumulator
+from kdp.stats import DateAccumulator, TextAccumulator, WelfordAccumulator
@pytest.mark.unit
@@ -136,3 +136,269 @@ def test_a_single_date_has_no_spread(self):
if __name__ == "__main__":
unittest.main()
+
+
+@pytest.mark.unit
+class TestWelfordAccumulator(unittest.TestCase):
+ """The mean and variance every normalised column is built from.
+
+ This accumulated in float32 and subtracted the running mean from each raw
+ value, so the deviations were computed at the magnitude of the values
+ themselves. A column around 1e8 came back with a variance of *minus* 1.6e6
+ against a true 1.0e6; one with a spread of 1e-4 was wrong by four orders of
+ magnitude. `Normalization` divides by the root of that number, so the
+ column reached the model as a constant and nothing raised.
+ """
+
+ CASES = {
+ "ordinary": lambda rng: rng.normal(50, 12, 5000),
+ "large magnitude": lambda rng: rng.normal(1e8, 1e3, 5000),
+ "small magnitude": lambda rng: rng.normal(1e-6, 1e-7, 5000),
+ "heavy tail": lambda rng: rng.standard_t(2, 5000) * 100,
+ "constant": lambda rng: np.full(5000, 42.0),
+ }
+
+ @staticmethod
+ def _accumulate(data, chunks=37, dtype=tf.float64):
+ accumulator = WelfordAccumulator()
+ for chunk in np.array_split(data, chunks):
+ accumulator.update(tf.constant(chunk, dtype=dtype))
+ return accumulator
+
+ def test_mean_and_variance_match_numpy(self):
+ rng = np.random.default_rng(3)
+ for label, make in self.CASES.items():
+ with self.subTest(case=label):
+ data = make(rng)
+ accumulator = self._accumulate(data)
+ mean = float(accumulator.mean.numpy())
+ variance = float(accumulator.variance.numpy())
+ self.assertAlmostEqual(
+ mean / max(abs(float(np.mean(data))), 1e-30),
+ 1.0 if np.mean(data) != 0 else mean,
+ places=6,
+ )
+ expected = float(np.var(data, ddof=1))
+ if expected == 0.0:
+ self.assertAlmostEqual(variance, 0.0, places=12)
+ else:
+ self.assertLess(abs(variance - expected) / expected, 1e-6, label)
+
+ def test_variance_is_never_negative(self):
+ """A negative variance is not a number a variance can be."""
+ rng = np.random.default_rng(11)
+ for label, make in self.CASES.items():
+ with self.subTest(case=label):
+ accumulator = self._accumulate(make(rng))
+ self.assertGreaterEqual(float(accumulator.variance.numpy()), 0.0, label)
+
+ def test_the_answer_does_not_depend_on_how_the_data_is_batched(self):
+ """One row at a time or all at once has to give the same statistics."""
+ data = np.random.default_rng(5).normal(1e7, 250, 900)
+ results = [
+ (
+ float(self._accumulate(data, chunks=c).mean.numpy()),
+ float(self._accumulate(data, chunks=c).variance.numpy()),
+ )
+ for c in (1, 7, 90, 900)
+ ]
+ for mean, variance in results[1:]:
+ self.assertAlmostEqual(mean / results[0][0], 1.0, places=10)
+ self.assertAlmostEqual(variance / results[0][1], 1.0, places=8)
+
+ def test_an_empty_batch_changes_nothing(self):
+ accumulator = WelfordAccumulator()
+ accumulator.update(tf.constant(np.arange(10.0), dtype=tf.float64))
+ before = (
+ float(accumulator.n.numpy()),
+ float(accumulator.mean.numpy()),
+ float(accumulator.variance.numpy()),
+ )
+ accumulator.update(tf.constant([], dtype=tf.float64))
+ after = (
+ float(accumulator.n.numpy()),
+ float(accumulator.mean.numpy()),
+ float(accumulator.variance.numpy()),
+ )
+ self.assertEqual(before, after)
+
+ def test_accumulators_do_not_share_state(self):
+ first = WelfordAccumulator()
+ first.update(tf.constant(np.zeros(100), dtype=tf.float64))
+ second = WelfordAccumulator()
+ second.update(tf.constant(np.full(50, 7.0), dtype=tf.float64))
+ self.assertEqual(float(first.n.numpy()), 100.0)
+ self.assertEqual(float(first.mean.numpy()), 0.0)
+ self.assertEqual(float(second.n.numpy()), 50.0)
+ self.assertAlmostEqual(float(second.mean.numpy()), 7.0, places=10)
+
+ def test_merging_accumulators_equals_accumulating_the_whole(self):
+ """Grouped statistics are pooled group by group through `merge`.
+
+ The grouped time series path replaced each group with `count` copies of
+ its own mean before combining, which keeps only the variance *between*
+ the group means. Two groups of spread 25 whose means happened to be
+ close reported 0.73 against a true 643.
+ """
+ rng = np.random.default_rng(9)
+ parts = [
+ rng.normal(10, 2, 300),
+ rng.normal(1000, 50, 700),
+ rng.normal(-5, 0.5, 120),
+ ]
+ whole = np.concatenate(parts)
+
+ single = WelfordAccumulator()
+ single.update(tf.constant(whole, dtype=tf.float64))
+
+ pooled = WelfordAccumulator()
+ for part in parts:
+ group = WelfordAccumulator()
+ group.update(tf.constant(part, dtype=tf.float64))
+ pooled.merge(group)
+
+ self.assertAlmostEqual(float(pooled.n.numpy()), float(len(whole)))
+ self.assertAlmostEqual(
+ float(pooled.mean.numpy()),
+ float(single.mean.numpy()),
+ places=8,
+ )
+ self.assertAlmostEqual(
+ float(pooled.variance.numpy()) / float(single.variance.numpy()),
+ 1.0,
+ places=9,
+ )
+ self.assertAlmostEqual(
+ float(pooled.variance.numpy()) / float(np.var(whole, ddof=1)),
+ 1.0,
+ places=9,
+ )
+
+ def test_merging_an_empty_accumulator_changes_nothing(self):
+ accumulator = WelfordAccumulator()
+ accumulator.update(tf.constant(np.arange(20.0), dtype=tf.float64))
+ before = (
+ float(accumulator.n.numpy()),
+ float(accumulator.mean.numpy()),
+ float(accumulator.variance.numpy()),
+ )
+ accumulator.merge(WelfordAccumulator())
+ self.assertEqual(
+ (
+ float(accumulator.n.numpy()),
+ float(accumulator.mean.numpy()),
+ float(accumulator.variance.numpy()),
+ ),
+ before,
+ )
+
+
+@pytest.mark.unit
+class TestTheHigherMomentsAndExtremes(unittest.TestCase):
+ """Skewness, kurtosis, min and max, against the same data in one pass.
+
+ The advisor reads all four; none were collected, so it saw skew 0 and
+ kurtosis 3 for every column and called them all normal. They are pooled the
+ same way as the mean and variance, so batching must not move them.
+ """
+
+ CASES = {
+ "normal": lambda rng: rng.normal(0.0, 1.0, 4000),
+ "right_skewed": lambda rng: rng.exponential(3.0, 4000),
+ "left_skewed": lambda rng: -rng.exponential(3.0, 4000),
+ "heavy_tailed": lambda rng: rng.standard_t(3, 4000),
+ "far_from_zero": lambda rng: rng.normal(1e6, 50.0, 4000),
+ }
+
+ @staticmethod
+ def _reference(column: np.ndarray) -> dict:
+ centered = column - column.mean()
+ second = (centered**2).mean()
+ return {
+ "skewness": (centered**3).mean() / second**1.5,
+ "kurtosis": (centered**4).mean() / second**2,
+ "min": column.min(),
+ "max": column.max(),
+ }
+
+ @staticmethod
+ def _accumulate(column: np.ndarray, batches: int) -> dict:
+ accumulator = WelfordAccumulator()
+ for chunk in np.array_split(column, batches):
+ accumulator.update(tf.constant(chunk))
+ return {
+ "skewness": float(accumulator.skewness.numpy()),
+ "kurtosis": float(accumulator.kurtosis.numpy()),
+ "min": float(accumulator.smallest.numpy()),
+ "max": float(accumulator.largest.numpy()),
+ }
+
+ def test_they_match_the_data_whatever_the_batch_size(self) -> None:
+ generator = np.random.default_rng(21)
+ for name, make in self.CASES.items():
+ column = make(generator)
+ reference = self._reference(column)
+ for batches in (1, 37, 500):
+ with self.subTest(case=name, batches=batches):
+ got = self._accumulate(column, batches)
+ for key, expected in reference.items():
+ self.assertAlmostEqual(
+ got[key] / max(abs(expected), 1e-9),
+ expected / max(abs(expected), 1e-9),
+ places=6,
+ msg=f"{name}/{key} at {batches} batches",
+ )
+
+ def test_the_shapes_are_told_apart(self) -> None:
+ """The numbers have to be different, not merely present."""
+ generator = np.random.default_rng(21)
+ normal = self._accumulate(generator.normal(0.0, 1.0, 4000), 7)
+ skewed = self._accumulate(generator.exponential(3.0, 4000), 7)
+ self.assertLess(abs(normal["skewness"]), 0.2)
+ self.assertGreater(skewed["skewness"], 1.5)
+ self.assertLess(abs(normal["kurtosis"] - 3.0), 0.4)
+ self.assertGreater(skewed["kurtosis"], 6.0)
+
+ def test_merging_gives_the_same_answer_as_one_pass(self) -> None:
+ column = np.random.default_rng(29).exponential(2.0, 3000)
+ whole = WelfordAccumulator()
+ whole.update(tf.constant(column))
+
+ head = WelfordAccumulator()
+ head.update(tf.constant(column[:1100]))
+ tail = WelfordAccumulator()
+ tail.update(tf.constant(column[1100:]))
+ head.merge(tail)
+
+ for name in ("skewness", "kurtosis", "smallest", "largest"):
+ with self.subTest(statistic=name):
+ self.assertAlmostEqual(
+ float(getattr(head, name).numpy()),
+ float(getattr(whole, name).numpy()),
+ places=6,
+ )
+
+ def test_too_few_values_answer_with_the_neutral_defaults(self) -> None:
+ empty = WelfordAccumulator()
+ self.assertEqual(float(empty.skewness.numpy()), 0.0)
+ self.assertEqual(float(empty.kurtosis.numpy()), 3.0)
+ self.assertEqual(float(empty.smallest.numpy()), 0.0)
+ self.assertEqual(float(empty.largest.numpy()), 0.0)
+
+ empty.update(tf.constant([], dtype=tf.float32))
+ self.assertEqual(float(empty.n.numpy()), 0.0)
+ self.assertEqual(float(empty.smallest.numpy()), 0.0)
+
+ one = WelfordAccumulator()
+ one.update(tf.constant([5.0]))
+ self.assertEqual(float(one.smallest.numpy()), 5.0)
+ self.assertEqual(float(one.largest.numpy()), 5.0)
+ self.assertEqual(float(one.skewness.numpy()), 0.0)
+
+ def test_a_constant_column_has_no_shape_to_report(self) -> None:
+ constant = WelfordAccumulator()
+ constant.update(tf.constant(np.full(100, 42.0)))
+ self.assertEqual(float(constant.skewness.numpy()), 0.0)
+ self.assertEqual(float(constant.kurtosis.numpy()), 3.0)
+ self.assertEqual(float(constant.smallest.numpy()), 42.0)
+ self.assertEqual(float(constant.largest.numpy()), 42.0)
diff --git a/test/test_text_vocabulary_matches_the_layer.py b/test/test_text_vocabulary_matches_the_layer.py
new file mode 100644
index 0000000..643a2be
--- /dev/null
+++ b/test/test_text_vocabulary_matches_the_layer.py
@@ -0,0 +1,172 @@
+"""The vocabulary collected from the data has to be the one the layer looks up.
+
+`TextVectorization` standardizes before it splits: by default it lowercases and
+strips punctuation. The statistics pass lowercased and split on whitespace but
+kept the punctuation, so a column of ordinary prose produced a vocabulary of
+"product," and "it!" while the layer, at inference, looked up "product" and
+"it". Every punctuated word therefore landed in the single out-of-vocabulary
+slot. The output width was right, the counts summed correctly, and roughly half
+the words in a normal English sentence were being thrown away.
+
+The same mismatch reached further: `ngrams` asks for word pairs the statistics
+never recorded, and a custom `standardize` or `split` spells tokens differently
+again. Those cases read the column and let the layer build its own vocabulary.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import keras
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+from kdp.features import TextFeature
+from kdp.processor import OutputModeOptions
+from kdp.stats import DatasetStatistics
+
+SENTENCES = [
+ "Great product, loved it!",
+ "Terrible. Would not buy.",
+ "It's okay -- nothing special",
+ "Great value!",
+]
+
+
+def _frame(repeats: int = 40) -> pd.DataFrame:
+ return pd.DataFrame({"txt": SENTENCES * repeats})
+
+
+@pytest.mark.unit
+class TestTextVocabularyMatchesTheLayer(unittest.TestCase):
+ """What the statistics collect, and what the layer expects, are one list."""
+
+ def setUp(self) -> None:
+ self.directory = Path(tempfile.mkdtemp())
+ self.frame = _frame()
+ self.data = self.directory / "data.csv"
+ self.frame.to_csv(self.data, index=False)
+
+ def test_the_statistics_vocabulary_is_what_keras_would_build(self) -> None:
+ statistics = DatasetStatistics(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / "stats.json"),
+ features_specs={"txt": FeatureType.TEXT},
+ overwrite_stats=True,
+ ).main()
+ collected = sorted(statistics["text"]["txt"]["vocab"])
+
+ vectorizer = keras.layers.TextVectorization(output_mode="count")
+ vectorizer.adapt(tf.constant(self.frame["txt"].values))
+ adapted = sorted(
+ str(word)
+ for word in vectorizer.get_vocabulary()
+ if word not in ("", "[UNK]")
+ )
+
+ self.assertEqual(collected, adapted)
+
+ def _counts(self, **kwargs) -> np.ndarray:
+ feature = TextFeature(
+ name="txt",
+ feature_type=FeatureType.TEXT,
+ output_mode="count",
+ **kwargs,
+ )
+ preprocessor = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / f"stats_{kwargs}.json"),
+ features_specs={"txt": feature},
+ overwrite_stats=True,
+ output_mode=OutputModeOptions.DICT,
+ )
+ preprocessor.build_preprocessor()
+ return np.asarray(
+ preprocessor.model(
+ {"txt": tf.constant(np.array(SENTENCES).reshape(-1, 1))},
+ )["txt"],
+ )
+
+ def test_no_word_of_the_training_text_falls_out_of_vocabulary(self) -> None:
+ """Column 0 of a `count` output is the out-of-vocabulary bucket."""
+ counts = self._counts()
+ self.assertEqual(
+ counts[:, 0].sum(),
+ 0.0,
+ f"words went missing: {counts[:, 0].tolist()}",
+ )
+
+ def test_every_word_is_accounted_for(self) -> None:
+ counts = self._counts()
+ for sentence, row in zip(SENTENCES, counts, strict=True):
+ # "--" is punctuation, and disappears under standardization.
+ expected = len([w for w in sentence.split() if w.strip("-")])
+ self.assertEqual(
+ row.sum(),
+ expected,
+ f"{sentence!r} matched {row.sum()} of {expected} tokens",
+ )
+
+
+@pytest.mark.unit
+class TestOptionsThatChangeWhatATokenIs(unittest.TestCase):
+ """`ngrams` has to reach the vocabulary, not just the tokenizer."""
+
+ def setUp(self) -> None:
+ self.directory = Path(tempfile.mkdtemp())
+ generator = np.random.default_rng(13)
+ phrases = ["red car fast", "blue sky wide open", "green grass", "red red red"]
+ self.frame = pd.DataFrame({"txt": generator.choice(phrases, 150)})
+ self.data = self.directory / "data.csv"
+ self.frame.to_csv(self.data, index=False)
+ self.phrases = phrases
+
+ def _width_and_oov(self, **kwargs) -> tuple[int, float]:
+ feature = TextFeature(
+ name="txt",
+ feature_type=FeatureType.TEXT,
+ output_mode="multi_hot",
+ **kwargs,
+ )
+ preprocessor = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / f"stats_{kwargs}.json"),
+ features_specs={"txt": feature},
+ overwrite_stats=True,
+ output_mode=OutputModeOptions.DICT,
+ )
+ preprocessor.build_preprocessor()
+ result = np.asarray(
+ preprocessor.model(
+ {"txt": tf.constant(np.array(self.phrases).reshape(-1, 1))},
+ )["txt"],
+ )
+ return int(result.shape[1]), float(result[:, 0].sum())
+
+ def test_ngrams_widen_the_vocabulary_instead_of_hitting_the_oov_slot(self) -> None:
+ unigram_width, _ = self._width_and_oov()
+ bigram_width, bigram_oov = self._width_and_oov(ngrams=2)
+ self.assertGreater(
+ bigram_width,
+ unigram_width,
+ "asking for bigrams produced the unigram vocabulary",
+ )
+ self.assertEqual(bigram_oov, 0.0, "every bigram fell out of vocabulary")
+
+ def test_the_width_matches_what_keras_builds_on_the_same_data(self) -> None:
+ for ngrams in (None, 2, 3):
+ with self.subTest(ngrams=ngrams):
+ width, _ = self._width_and_oov(ngrams=ngrams)
+ vectorizer = keras.layers.TextVectorization(
+ output_mode="multi_hot",
+ ngrams=ngrams,
+ )
+ vectorizer.adapt(tf.constant(self.frame["txt"].values))
+ self.assertEqual(width, len(vectorizer.get_vocabulary()))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_the_advisor_can_tell_distributions_apart.py b/test/test_the_advisor_can_tell_distributions_apart.py
new file mode 100644
index 0000000..77c9d32
--- /dev/null
+++ b/test/test_the_advisor_can_tell_distributions_apart.py
@@ -0,0 +1,151 @@
+"""The advisor's advice has to come from the data, and has to be runnable.
+
+`ModelAdvisor` reads `skewness`, `kurtosis`, `min` and `max` off each numeric
+feature's statistics, and the statistics carried none of them. Every column
+therefore arrived with the neutral defaults -- skew 0, kurtosis 3 -- and came
+back as "Normal distribution detected, standard normalization recommended"
+whatever shape it actually had, while the rescaling factor derived from `min`
+and `max` always worked out to exactly 1.
+
+The generated snippet had the matching problem: it built a `PreprocessingModel`
+with no `path_data`, and every configuration the advisor recommends needs the
+data, so pasting the advice raised.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+
+from kdp.features import FeatureType
+from kdp.model_advisor import recommend_model_configuration
+from kdp.stats import DatasetStatistics
+
+ROWS = 4000
+
+
+def _statistics(directory: Path, frame: pd.DataFrame) -> dict:
+ data = directory / "data.csv"
+ frame.to_csv(data, index=False)
+ return DatasetStatistics(
+ path_data=str(data),
+ features_stats_path=str(directory / "stats.json"),
+ features_specs={
+ name: FeatureType.STRING_CATEGORICAL
+ if frame[name].dtype == object
+ else FeatureType.FLOAT_NORMALIZED
+ for name in frame
+ },
+ overwrite_stats=True,
+ ).main()
+
+
+@pytest.mark.unit
+class TestTheAdvisorCanTellDistributionsApart(unittest.TestCase):
+ """Three differently shaped columns must not get one answer."""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ generator = np.random.default_rng(19)
+ cls.frame = pd.DataFrame(
+ {
+ "gaussian": generator.normal(0.0, 1.0, ROWS),
+ "heavy_tailed": generator.exponential(5.0, ROWS),
+ "uniform": generator.uniform(0.0, 1.0, ROWS),
+ "label": generator.choice(["a", "b", "c"], ROWS),
+ },
+ )
+ cls.statistics = _statistics(cls.directory, cls.frame)
+ cls.recommendation = recommend_model_configuration(cls.statistics)
+
+ def test_the_statistics_carry_the_numbers_the_advisor_reads(self) -> None:
+ numeric = self.statistics["numeric_stats"]
+ for name in ("gaussian", "heavy_tailed", "uniform"):
+ with self.subTest(feature=name):
+ for key in ("min", "max", "skewness", "kurtosis"):
+ self.assertIn(key, numeric[name])
+
+ def test_the_moments_match_the_data(self) -> None:
+ numeric = self.statistics["numeric_stats"]
+ for name in ("gaussian", "heavy_tailed", "uniform"):
+ column = self.frame[name].to_numpy()
+ centered = column - column.mean()
+ second = (centered**2).mean()
+ with self.subTest(feature=name):
+ self.assertAlmostEqual(
+ numeric[name]["skewness"],
+ (centered**3).mean() / second**1.5,
+ places=2,
+ )
+ self.assertAlmostEqual(
+ numeric[name]["kurtosis"],
+ (centered**4).mean() / second**2,
+ places=2,
+ )
+ self.assertAlmostEqual(numeric[name]["min"], column.min(), places=3)
+ self.assertAlmostEqual(numeric[name]["max"], column.max(), places=3)
+
+ def test_each_column_gets_the_advice_its_shape_calls_for(self) -> None:
+ features = self.recommendation["features"]
+ self.assertEqual(features["gaussian"]["detected_distribution"], "normal")
+ self.assertEqual(
+ features["heavy_tailed"]["detected_distribution"],
+ "heavy_tailed",
+ )
+ self.assertNotEqual(
+ features["uniform"]["detected_distribution"],
+ features["heavy_tailed"]["detected_distribution"],
+ )
+
+ def test_the_advice_is_not_the_same_sentence_for_every_column(self) -> None:
+ notes = {
+ name: tuple(self.recommendation["features"][name].get("notes", ()))
+ for name in ("gaussian", "heavy_tailed", "uniform")
+ }
+ self.assertEqual(
+ len(set(notes.values())),
+ 3,
+ f"three differently shaped columns got the same advice: {notes}",
+ )
+
+
+@pytest.mark.unit
+class TestTheGeneratedSnippetRuns(unittest.TestCase):
+ """Advice you cannot paste is not advice."""
+
+ def test_the_snippet_builds_a_preprocessor(self) -> None:
+ directory = Path(tempfile.mkdtemp())
+ generator = np.random.default_rng(23)
+ frame = pd.DataFrame(
+ {
+ "amount": generator.exponential(3.0, 500),
+ "score": generator.normal(0.0, 1.0, 500),
+ "label": generator.choice(["x", "y"], 500),
+ },
+ )
+ statistics = _statistics(directory, frame)
+ snippet = recommend_model_configuration(statistics)["code_snippet"]
+
+ self.assertIn('path_data="your_data.csv"', snippet)
+
+ runnable = snippet.replace(
+ 'path_data="your_data.csv"',
+ f'path_data="{directory / "data.csv"}"',
+ ).replace(
+ " features_specs=features,",
+ " features_specs=features,\n"
+ f' features_stats_path="{directory / "fresh.json"}",\n'
+ " overwrite_stats=True,",
+ )
+ namespace: dict = {}
+ exec(compile(runnable, "", "exec"), namespace) # noqa: S102
+ result = namespace["model"].build_preprocessor()
+ self.assertIsNotNone(result["model"])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_the_column_order_is_stable.py b/test/test_the_column_order_is_stable.py
new file mode 100644
index 0000000..edfa9f8
--- /dev/null
+++ b/test/test_the_column_order_is_stable.py
@@ -0,0 +1,137 @@
+"""Column N of the output has to be the same feature on every build.
+
+Features are preprocessed in parallel batches, and the concatenated output was
+assembled by walking `processed_features` -- a dict keyed in whichever order the
+workers happened to finish. Building the same configuration twice therefore laid
+the columns out differently: five distinct layouts in six builds of the same six
+features.
+
+Nothing raised, and the numbers were right for the layout each build chose. The
+damage lands on anyone who trains on one build and serves from another: every
+feature is read as a different one. Reloading a saved `.keras` file was always
+safe, because the order is baked into its graph.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import keras
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+
+ROWS = 200
+NUMERIC = [f"num{index}" for index in range(6)]
+
+SPECS = {name: FeatureType.FLOAT_NORMALIZED for name in NUMERIC}
+SPECS["cat_string"] = FeatureType.STRING_CATEGORICAL
+SPECS["cat_integer"] = FeatureType.INTEGER_CATEGORICAL
+
+
+def _frame() -> pd.DataFrame:
+ generator = np.random.default_rng(67)
+ frame = pd.DataFrame({name: generator.normal(0.0, 1.0, ROWS) for name in NUMERIC})
+ frame["cat_string"] = generator.choice(["a", "b", "c"], ROWS)
+ frame["cat_integer"] = generator.integers(0, 4, ROWS)
+ return frame
+
+
+class _Built:
+ """One build of the preprocessor, with a way to locate each feature."""
+
+ def __init__(self, directory: Path, first: bool, **options) -> None:
+ keras.backend.clear_session()
+ self.preprocessor = PreprocessingModel(
+ path_data=str(directory / "data.csv"),
+ features_stats_path=str(directory / "stats.json"),
+ features_specs=SPECS,
+ overwrite_stats=first,
+ **options,
+ )
+ self.preprocessor.build_preprocessor()
+
+ @property
+ def names(self) -> tuple:
+ return tuple(self.preprocessor._concat_feature_names)
+
+ def columns_moved_by(self, feature: str) -> tuple:
+ """Which output columns respond to a change in one numeric feature."""
+ baseline = {name: tf.constant(np.zeros((3, 1)), np.float32) for name in NUMERIC}
+ baseline["cat_string"] = tf.constant(np.array(["a", "a", "a"]).reshape(-1, 1))
+ baseline["cat_integer"] = tf.constant(np.zeros((3, 1), dtype=np.int32))
+ before = np.asarray(self.preprocessor.model(baseline))
+
+ bumped = dict(baseline)
+ bumped[feature] = tf.constant(np.full((3, 1), 7.0), np.float32)
+ after = np.asarray(self.preprocessor.model(bumped))
+
+ return tuple(np.where(np.abs(after - before).max(axis=0) > 1e-6)[0].tolist())
+
+
+@pytest.mark.integration
+class TestTheColumnOrderIsStable(unittest.TestCase):
+ """The same configuration has to produce the same layout every time."""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ _frame().to_csv(cls.directory / "data.csv", index=False)
+
+ def test_the_reported_order_does_not_move(self) -> None:
+ orders = {_Built(self.directory, first=(run == 0)).names for run in range(5)}
+ self.assertEqual(len(orders), 1, f"the feature order moved: {orders}")
+
+ def test_it_is_the_order_the_features_were_declared_in(self) -> None:
+ built = _Built(self.directory, first=True)
+ # Numeric features are concatenated before categorical ones, and within
+ # each block the declaration order holds.
+ self.assertEqual(built.names[: len(NUMERIC)], tuple(NUMERIC))
+ self.assertEqual(
+ set(built.names[len(NUMERIC) :]),
+ {"cat_string", "cat_integer"},
+ )
+
+ def test_each_feature_lands_in_the_same_columns_every_time(self) -> None:
+ layouts = set()
+ for run in range(4):
+ built = _Built(self.directory, first=(run == 0))
+ layouts.add(
+ tuple((name, built.columns_moved_by(name)) for name in NUMERIC),
+ )
+ self.assertEqual(len(layouts), 1, f"the column layout moved: {layouts}")
+
+ def test_a_numeric_feature_sits_at_its_declared_index(self) -> None:
+ built = _Built(self.directory, first=True)
+ for index, name in enumerate(NUMERIC):
+ with self.subTest(feature=name):
+ self.assertEqual(built.columns_moved_by(name), (index,))
+
+ def test_the_mixture_of_experts_keeps_the_same_order(self) -> None:
+ orders = {
+ _Built(
+ self.directory,
+ first=(run == 0),
+ use_feature_moe=True,
+ feature_moe_num_experts=2,
+ feature_moe_expert_dim=4,
+ ).names
+ for run in range(4)
+ }
+ self.assertEqual(len(orders), 1, f"the feature order moved: {orders}")
+
+ def test_a_cross_comes_after_the_declared_features(self) -> None:
+ built = _Built(
+ self.directory,
+ first=True,
+ feature_crosses=[("cat_string", "cat_integer", 8)],
+ )
+ self.assertEqual(built.names[-1], "cat_string_x_cat_integer")
+ self.assertEqual(built.names[: len(NUMERIC)], tuple(NUMERIC))
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_the_output_is_always_a_table.py b/test/test_the_output_is_always_a_table.py
new file mode 100644
index 0000000..65a2fd6
--- /dev/null
+++ b/test/test_the_output_is_always_a_table.py
@@ -0,0 +1,155 @@
+"""Whatever the options, a preprocessor returns one row of numbers per input row.
+
+A transformer block adds a sequence axis to a 2-D input and hands it back.
+The categorical placement flattened that away; `transfo_placement="all_features"`
+did not, so that one configuration returned (rows, 1, width). A Dense head
+bolted onto the preprocessor sees a different rank depending on a flag that has
+nothing to do with the output shape, and nothing said so.
+
+Rank is the contract every downstream model depends on, so it is checked for
+every model-level option rather than for the ones that happened to be tried.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp import FeatureType, PreprocessingModel
+
+ROWS = 80
+
+SPECS = {
+ "num1": FeatureType.FLOAT_NORMALIZED,
+ "num2": FeatureType.FLOAT_RESCALED,
+ "cat1": FeatureType.STRING_CATEGORICAL,
+ "cat2": FeatureType.INTEGER_CATEGORICAL,
+}
+
+# Every model-level switch that reshapes the concatenated block, and the
+# combinations that stack two of them.
+CONFIGURATIONS = {
+ "plain": {},
+ "attention_all": {
+ "tabular_attention": True,
+ "tabular_attention_heads": 2,
+ "tabular_attention_dim": 16,
+ "tabular_attention_placement": "all_features",
+ },
+ "attention_numeric": {
+ "tabular_attention": True,
+ "tabular_attention_heads": 2,
+ "tabular_attention_dim": 16,
+ "tabular_attention_placement": "numeric",
+ },
+ "attention_categorical": {
+ "tabular_attention": True,
+ "tabular_attention_heads": 2,
+ "tabular_attention_dim": 16,
+ "tabular_attention_placement": "categorical",
+ },
+ "attention_multi_resolution": {
+ "tabular_attention": True,
+ "tabular_attention_heads": 2,
+ "tabular_attention_dim": 16,
+ "tabular_attention_placement": "multi_resolution",
+ },
+ "transformer_categorical": {
+ "transfo_nr_blocks": 2,
+ "transfo_nr_heads": 2,
+ "transfo_ff_units": 8,
+ "transfo_placement": "categorical",
+ },
+ "transformer_all_features": {
+ "transfo_nr_blocks": 2,
+ "transfo_nr_heads": 2,
+ "transfo_ff_units": 8,
+ "transfo_placement": "all_features",
+ },
+ "transformer_and_attention": {
+ "tabular_attention": True,
+ "tabular_attention_heads": 1,
+ "tabular_attention_dim": 12,
+ "tabular_attention_placement": "all_features",
+ "transfo_nr_blocks": 1,
+ "transfo_nr_heads": 2,
+ "transfo_ff_units": 8,
+ "transfo_placement": "all_features",
+ },
+ "feature_selection": {
+ "feature_selection_placement": "all_features",
+ "feature_selection_units": 8,
+ },
+ "feature_moe": {"use_feature_moe": True, "feature_moe_num_experts": 3},
+ "global_numerical_embedding": {
+ "use_global_numerical_embedding": True,
+ "global_embedding_dim": 6,
+ },
+ "feature_crosses": {"feature_crosses": [("cat1", "cat2", 8)]},
+}
+
+
+def _frame() -> pd.DataFrame:
+ generator = np.random.default_rng(5)
+ return pd.DataFrame(
+ {
+ "num1": generator.normal(0.0, 1.0, ROWS),
+ "num2": generator.uniform(0.0, 10.0, ROWS),
+ "cat1": generator.choice(["a", "b", "c"], ROWS),
+ "cat2": generator.integers(0, 4, ROWS),
+ },
+ )
+
+
+@pytest.mark.unit
+class TestTheOutputIsAlwaysATable(unittest.TestCase):
+ """Rank 2, one row out per row in, for every option."""
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.directory = Path(tempfile.mkdtemp())
+ cls.frame = _frame()
+ cls.data = cls.directory / "data.csv"
+ cls.frame.to_csv(cls.data, index=False)
+
+ def _build_and_call(self, name: str, options: dict) -> np.ndarray:
+ preprocessor = PreprocessingModel(
+ path_data=str(self.data),
+ features_stats_path=str(self.directory / f"stats_{name}.json"),
+ features_specs=SPECS,
+ overwrite_stats=True,
+ **options,
+ )
+ preprocessor.build_preprocessor()
+ return np.asarray(
+ preprocessor.model(
+ {
+ column: tf.constant(self.frame[column].values.reshape(-1, 1))
+ for column in self.frame
+ },
+ ),
+ )
+
+ def test_every_configuration_returns_rows_by_width(self) -> None:
+ wrong = {}
+ for name, options in CONFIGURATIONS.items():
+ result = self._build_and_call(name, options)
+ if result.ndim != 2 or result.shape[0] != ROWS:
+ wrong[name] = result.shape
+ self.assertEqual(wrong, {}, f"not (rows, width): {wrong}")
+
+ def test_every_configuration_returns_finite_numbers(self) -> None:
+ wrong = [
+ name
+ for name, options in CONFIGURATIONS.items()
+ if not np.isfinite(self._build_and_call(name, options)).all()
+ ]
+ self.assertEqual(wrong, [], f"non-finite output: {wrong}")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_time_series_feature.py b/test/test_time_series_feature.py
index 30c5b5c..b1870b1 100644
--- a/test/test_time_series_feature.py
+++ b/test/test_time_series_feature.py
@@ -387,9 +387,13 @@ def test_build_layers_for_a_calendar_feature(self):
"wavelet_transform_config": {"levels": 2, "flatten_output": False},
}
],
+ # tsfresh summarizes a window, and a time series feature hands one
+ # column per row over, so it needs a config that widens the feature
+ # first -- exactly like the wavelet cases above.
[
{
"name": "sales",
+ "lag_config": {"lags": [1, 2, 3]},
"tsfresh_feature_config": {"features": ["mean", "std", "min"]},
}
],
diff --git a/test/test_transforms_stay_in_their_domain.py b/test/test_transforms_stay_in_their_domain.py
new file mode 100644
index 0000000..6901fc7
--- /dev/null
+++ b/test/test_transforms_stay_in_their_domain.py
@@ -0,0 +1,116 @@
+"""A transform outside its domain must return a number, not NaN or infinity.
+
+`log` is defined for positive values. Every neighbouring transform in the layer
+guards its own domain -- `sqrt` clamps at zero, `box-cox` at epsilon -- but
+`log` handed negatives straight to `log()` and returned NaN. A NaN in a
+preprocessor output is not a local problem: it spreads through every layer that
+touches it and through every gradient taken from it, without a message.
+
+`logit` clipped into `[epsilon, 1 - epsilon]`, which reads as correct and was
+not: the default epsilon of 1e-10 is far below the float32 gap at 1.0, so
+`1 - epsilon` rounded back to exactly 1.0, `1 - x` became 0, and any value at
+or above 1 came out as +inf.
+"""
+
+import unittest
+
+import numpy as np
+import pytest
+import tensorflow as tf
+
+from kdp.layers.distribution_transform_layer import DistributionTransformLayer
+
+# Values that sit outside the domain of at least one transform.
+OUT_OF_DOMAIN = np.array(
+ [[-1e6], [-5.0], [-0.5], [0.0], [0.5], [1.0], [5.0], [1e6]],
+ dtype=np.float32,
+)
+
+TRANSFORMS = [
+ "log",
+ "sqrt",
+ "box-cox",
+ "yeo-johnson",
+ "arcsinh",
+ "cube-root",
+ "logit",
+ "quantile",
+ "robust-scale",
+ "min-max",
+]
+
+
+@pytest.mark.layers
+class TestTransformsStayInTheirDomain(unittest.TestCase):
+ """No transform may answer with NaN or infinity."""
+
+ def test_every_transform_returns_finite_numbers(self) -> None:
+ offenders = {}
+ for name in TRANSFORMS:
+ result = np.asarray(
+ DistributionTransformLayer(transform_type=name)(
+ tf.constant(OUT_OF_DOMAIN),
+ ),
+ )
+ if not np.isfinite(result).all():
+ offenders[name] = result.reshape(-1).tolist()
+ self.assertEqual(offenders, {}, f"NaN or infinity in the output: {offenders}")
+
+ def test_log_still_matches_its_definition_where_it_is_defined(self) -> None:
+ values = np.linspace(0.05, 5.0, 40, dtype=np.float32).reshape(-1, 1)
+ result = np.asarray(
+ DistributionTransformLayer(transform_type="log")(tf.constant(values)),
+ )
+ np.testing.assert_allclose(result, np.log(values + 1e-10), atol=1e-4)
+
+ def test_logit_still_matches_its_definition_inside_the_unit_interval(self) -> None:
+ values = np.linspace(0.05, 0.95, 40, dtype=np.float32).reshape(-1, 1)
+ result = np.asarray(
+ DistributionTransformLayer(transform_type="logit")(tf.constant(values)),
+ )
+ np.testing.assert_allclose(result, np.log(values / (1 - values)), atol=5e-3)
+
+ def test_logit_is_symmetric_at_both_ends(self) -> None:
+ """The clip has to bite on both sides, not only the low one."""
+ result = np.asarray(
+ DistributionTransformLayer(transform_type="logit")(
+ tf.constant(np.array([[0.0], [1.0]], dtype=np.float32)),
+ ),
+ ).reshape(-1)
+ self.assertTrue(np.isfinite(result).all())
+ np.testing.assert_allclose(result[0], -result[1], rtol=1e-5)
+
+ def test_arcsinh_is_odd_and_matches_its_definition(self) -> None:
+ """`log(x + sqrt(x*x + 1))` lost the whole negative tail to float32.
+
+ At x = -1e6 the square is 1e12, where float32 cannot hold the +1; its
+ root is exactly 1e6, the sum is exactly zero, and the logarithm is
+ -infinity. The positive side was unaffected, so the transform was quietly
+ one-sided.
+ """
+ values = np.array(
+ [[-1e6], [-1e3], [-1.0], [0.0], [1.0], [1e3], [1e6]],
+ dtype=np.float32,
+ )
+ result = np.asarray(
+ DistributionTransformLayer(transform_type="arcsinh")(tf.constant(values)),
+ ).reshape(-1)
+ np.testing.assert_allclose(result, np.arcsinh(values.reshape(-1)), rtol=1e-5)
+ np.testing.assert_allclose(result, -result[::-1], rtol=1e-5)
+
+ def test_a_negative_column_survives_the_log_transform_in_a_model(self) -> None:
+ """The same thing, through the layer as a model would call it."""
+ column = np.linspace(-50.0, 50.0, 100, dtype=np.float32).reshape(-1, 1)
+ result = np.asarray(
+ DistributionTransformLayer(transform_type="log")(tf.constant(column)),
+ )
+ self.assertTrue(np.isfinite(result).all())
+ self.assertGreater(
+ len(np.unique(result)),
+ 10,
+ "the positive half of the column was flattened too",
+ )
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/test/test_tsfresh_needs_a_window.py b/test/test_tsfresh_needs_a_window.py
new file mode 100644
index 0000000..ee445d1
--- /dev/null
+++ b/test/test_tsfresh_needs_a_window.py
@@ -0,0 +1,118 @@
+"""Statistics over a window of one are not statistics.
+
+A `TimeSeriesFeature` hands one column per row to the layers it builds, and
+`TSFreshFeatureLayer` clamps `window_size` to the number of steps it is given.
+Over a single step the mean, the minimum and the maximum are all the value
+itself and the standard deviation is zero, so a config asking for four
+statistics returned the input column three times plus a column of zeros. The
+width matched what `get_output_dim()` declared, the values were finite, and
+none of it carried anything.
+
+The wavelet already refused the same shape of input. This checks that tsfresh
+does too, and that once it has a window its numbers are the ones pandas gets.
+"""
+
+import unittest
+
+import numpy as np
+import pandas as pd
+import pytest
+import tensorflow as tf
+
+from kdp.features import TimeSeriesFeature
+from kdp.layers.time_series.tsfresh_feature_layer import TSFreshFeatureLayer
+
+ROWS = 96
+SERIES = np.sin(np.arange(ROWS) / 6.0) * 10 + np.arange(ROWS) * 0.3 + 50
+
+
+def _through(feature: TimeSeriesFeature) -> np.ndarray:
+ data = tf.constant(SERIES.reshape(-1, 1), dtype=tf.float32)
+ for layer in feature.build_layers():
+ data = layer(data)
+ return np.asarray(data)
+
+
+@pytest.mark.time_series
+class TestTsfreshNeedsAWindow(unittest.TestCase):
+ """One step in, nothing to summarize."""
+
+ def test_a_single_step_is_refused(self) -> None:
+ with self.assertRaises(ValueError) as raised:
+ TSFreshFeatureLayer(features=["mean", "std"])(
+ tf.constant(np.arange(10.0).reshape(-1, 1), dtype=tf.float32),
+ )
+ message = str(raised.exception)
+ self.assertIn("two time steps", message)
+ self.assertIn("lag_config", message)
+
+ def test_a_time_series_feature_without_a_window_is_refused(self) -> None:
+ feature = TimeSeriesFeature(
+ name="sales",
+ sort_by="order",
+ tsfresh_feature_config={
+ "features": ["mean", "std", "min", "max"],
+ "window_size": 8,
+ },
+ )
+ with self.assertRaises(ValueError):
+ _through(feature)
+
+ def test_with_a_window_the_statistics_are_the_ones_pandas_gets(self) -> None:
+ window = 8
+ feature = TimeSeriesFeature(
+ name="sales",
+ sort_by="order",
+ lag_config={
+ "lags": list(range(1, window)),
+ "drop_na": False,
+ "fill_value": 0.0,
+ },
+ tsfresh_feature_config={
+ "features": ["mean", "std", "min", "max"],
+ "window_size": window,
+ },
+ )
+ produced = _through(feature)
+ self.assertEqual(produced.shape, (ROWS, 4))
+
+ series = pd.Series(SERIES)
+ rolling = series.rolling(window)
+ expected = {
+ 0: rolling.mean(),
+ 1: rolling.std(ddof=0),
+ 2: rolling.min(),
+ 3: rolling.max(),
+ }
+ # Only the rows with a full window: earlier ones sit on the lag padding.
+ for column, reference in expected.items():
+ with self.subTest(statistic=column):
+ np.testing.assert_allclose(
+ produced[window - 1 :, column],
+ reference.to_numpy()[window - 1 :],
+ atol=1e-2,
+ )
+
+ def test_the_statistics_are_not_four_copies_of_the_input(self) -> None:
+ """What the degenerate window produced, stated as a check."""
+ feature = TimeSeriesFeature(
+ name="sales",
+ sort_by="order",
+ lag_config={"lags": [1, 2, 3, 4, 5], "drop_na": False, "fill_value": 0.0},
+ tsfresh_feature_config={
+ "features": ["mean", "std", "min", "max"],
+ "window_size": 6,
+ },
+ )
+ produced = _through(feature)
+ self.assertGreater(
+ produced[:, 1].std(),
+ 0.0,
+ "the standard deviation column is constant",
+ )
+ distinct = {tuple(np.round(produced[:, i], 4)) for i in range(4)}
+ self.assertEqual(len(distinct), 4, "two statistics produced the same column")
+
+
+if __name__ == "__main__":
+ unittest.main()
