fix(KDP): repair silent data defects in seasons, time series options and serialization

.github/workflows/UTESTS.yml

@@ -58,6 +58,23 @@ jobs:
           python -m pip install --upgrade pip
           pip install poetry
           poetry install --no-interaction
+      - name: Every test belongs to a group the matrix runs
+        run: |
+          # The matrix selects by marker. A test carrying none of these markers
+          # is collected by nobody and never runs in CI, which is how the whole
+          # save/load round-trip suite went unexecuted.
+          orphans=$(poetry run pytest test/ --collect-only -q \
+            -m "not unit and not integration and not layers and not processor and not time_series" \
+            2>/dev/null | grep -c "::" || true)
+          if [ "$orphans" -ne 0 ]; then
+            echo "::error::$orphans test(s) match no CI marker group:"
+            poetry run pytest test/ --collect-only -q \
+              -m "not unit and not integration and not layers and not processor and not time_series"
+            exit 1
+          fi
+          echo "All collected tests belong to a marker group."
+        timeout-minutes: 3
+
       - name: Run smoke tests
         run: |
           poetry run pytest -m "micro" --maxfail=1 --tb=no -q --timeout=30
@@ -160,6 +177,32 @@ jobs:
         run: poetry run pytest -c pytest-fast.ini --timeout=180
         timeout-minutes: 15
 
+  # The docs promise an API. This job runs every self-contained python example
+  # in docs/ against data synthesised from the block's own feature spec, which
+  # catches the defects a symbol check cannot: a wrong shape, an option
+  # combination Keras rejects, or a method whose return value the prose
+  # describes incorrectly.
+  doc-examples:
+    runs-on: ubuntu-latest
+    needs: [smoke-test]
+    if: always() && (needs.smoke-test.result == 'success' || github.event_name != 'pull_request')
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+          poetry install --no-interaction
+
+      - name: Run every documented example
+        run: poetry run python scripts/verify_doc_examples.py
+        timeout-minutes: 25
+
   # Coverage job (only on main Python version)
   coverage:
     runs-on: ubuntu-latest

.gitignore

@@ -167,3 +167,6 @@ my_tests/*
 
 # derivative files
 *.csv
+
+# Images written by the example scripts
+examples/outputs/

Makefile

@@ -276,3 +276,8 @@ help:
 		printf "\n"; \
 	}' \
 	| more $(shell test $(shell uname) = Darwin && echo '--no-init --raw-control-chars')
+
+.PHONY: verify_doc_examples
+## Run every self-contained python example in docs/ against synthesised data
+verify_doc_examples:
+	poetry run python scripts/verify_doc_examples.py

README.md

@@ -24,11 +24,16 @@ KDP provides a state-of-the-art preprocessing system built on TensorFlow Keras.
 ## ✨ Key Features
 
 - 🚀 **Efficient Single-Pass Processing**: Process all features in one go, dramatically faster than alternatives
-- 🧠 **Distribution-Aware Encoding**: Automatically detects and optimally handles different data distributions
-- 👁️ **Tabular Attention**: Captures complex feature interactions for better model performance
-- 🔍 **Feature Selection**: Automatically identifies and focuses on the most important features
-- 🔄 **Feature-wise Mixture of Experts**: Specialized processing for different feature types
-- 📦 **Production-Ready**: Deploy your preprocessing along with your model as a single unit
+- 🧠 **[Distribution-Aware Encoding](docs/advanced/distribution-aware-encoding.md)**: Detects the shape of each numeric column and transforms it accordingly
+- 👁️ **[Tabular Attention](docs/advanced/tabular-attention.md)**: Inter-feature and inter-sample attention, with a multi-resolution mode that treats numeric and categorical columns separately
+- 🧮 **[Advanced Numerical Embeddings](docs/advanced/numerical-embeddings.md)**: Learned per-feature embeddings, and a global embedding across all numeric columns
+- 🔀 **[Transformer Blocks](docs/advanced/transformer-blocks.md)**: Stackable blocks over the categorical features or over everything
+- 🔄 **[Feature-wise Mixture of Experts](docs/advanced/feature-moe.md)**: Specialised expert networks per feature, routed automatically or by hand
+- ⏱️ **[Time Series Features](docs/features/time_series_features.md)**: Lags, rolling statistics, differencing, moving averages, wavelets, FFT, tsfresh statistics and calendar features
+- 📝 **[Text](docs/features/text-features.md), [Date](docs/features/date-features.md), [Categorical](docs/features/categorical-features.md), [Cross](docs/features/cross-features.md) and [Passthrough](docs/features/passthrough-features.md) features**: Every column type in one model
+- 🎛️ **[Auto-Configuration](docs/optimization/auto-configuration.md)**: Inspects your data and recommends the preprocessing to use
+- 🔍 **[Feature Selection](docs/optimization/feature-selection.md)**: Gated variable-selection layers on any subset of your features
+- 📦 **Production-Ready**: Save and reload the preprocessor with your model as a single Keras artefact
 
 ## 🚀 Quick Installation
 

docs/advanced/custom-preprocessing.md

@@ -565,7 +565,6 @@ feature = NumericalFeature(
         CustomScalingLayer,
         "Dense"
     ],
-    scaling_factor=5.0,  # For CustomScalingLayer
     units=16             # For Dense
 )
 ```
@@ -798,9 +797,12 @@ feature = NumericalFeature(
     ]
 )
 
-# The normalization layer needs to be adapted to the data
-model = PreprocessingModel(features={"height": feature})
-model.fit(data)  # This initializes the normalization statistics
+# The normalization layer is adapted while the preprocessor is built
+model = PreprocessingModel(
+    path_data="data.csv",
+    features_specs={"height": feature},
+)
+model.build_preprocessor()
 ```
 
 ### 🚀 GPU-Accelerated Custom Preprocessing
@@ -825,14 +827,20 @@ import logging
 logging.basicConfig(level=logging.DEBUG)
 
 # Create a model with your custom preprocessing
-model = PreprocessingModel(features=features)
+model = PreprocessingModel(path_data="data.csv", features_specs=features)
 
 # Inspect the model layers
-model.build_model()
+model.build_preprocessor()
 model.model.summary()
 
-# Test with small batch
-small_batch = data.head(5)
-result = model.transform(small_batch)
-print(result)
+# Test with a small batch. The built model is a Keras model over a dict of
+# columns, so feed it one tensor per feature -- `InferenceFormatter` does that
+# conversion for you when you have a DataFrame in hand.
+from kdp import InferenceFormatter
+
+batch = InferenceFormatter(model).prepare_inference_data(
+    data.head(5),
+    to_tensors=True,
+)
+print(model.model(batch))
 ```

docs/advanced/distribution-aware-encoding.md

@@ -108,6 +108,8 @@ preprocessor = PreprocessingModel(
 <div class="code-container">
 
 ```python
+from kdp import FeatureType, PreprocessingModel
+
 from kdp.features import NumericalFeature
 from kdp.layers.distribution_aware_encoder_layer import DistributionType
 
@@ -129,8 +131,6 @@ preprocessor = PreprocessingModel(
     features_specs=features_specs,
     use_distribution_aware=True,
     distribution_aware_bins=1000,
-    detect_periodicity=True,  # Enable periodic pattern detection
-    handle_sparsity=True     # Enable sparse data handling
 )
 ```
 
@@ -251,6 +251,10 @@ preprocessor = PreprocessingModel(
 
 ## ⚙️ Configuration Options
 
+### On `PreprocessingModel`
+
+These two are the whole model-level surface:
+
 <div class="table-container">
   <table class="config-table">
     <thead>
@@ -274,6 +278,88 @@ preprocessor = PreprocessingModel(
         <td>1000</td>
         <td>Number of bins for distribution analysis</td>
       </tr>
+    </tbody>
+  </table>
+</div>
+
+Per feature, set `preferred_distribution` on a `NumericalFeature` to skip
+automatic detection and force a specific distribution.
+
+!!! warning "Automatic detection is a heuristic, and it is not always right"
+    Measured over six seeded samples, detection identifies `heavy_tailed`,
+    `log_normal`, `discrete`, `periodic`, `sparse` and `beta` correctly and
+    consistently. Four shapes are consistently confused with a neighbour:
+
+    | Actual | Detected as |
+    |---|---|
+    | `normal` | `multimodal` |
+    | `uniform` | `multimodal` |
+    | `multimodal` | `periodic` |
+    | `exponential` | `log_normal` |
+
+    The encoding still works &mdash; a neighbouring distribution's transform is
+    usually a reasonable choice &mdash; but if a column's shape matters to you,
+    set `preferred_distribution` on the feature rather than relying on
+    detection. `test/layers/test_distribution_aware_encoder.py` pins this
+    behaviour, so it cannot change without being noticed.
+
+### Available transformations
+
+`transform_type` accepts these; anything else raises. `auto` is the default on
+the encoder and picks from the rest using the column's own shape &mdash;
+whether it is bounded, contains zeros, or contains negative values.
+
+<div class="table-container">
+  <table>
+    <thead>
+      <tr>
+        <th>Name</th>
+        <th>Requires</th>
+        <th>Use for</th>
+      </tr>
+    </thead>
+    <tbody>
+      <tr><td><code>auto</code></td><td>&mdash;</td><td>Let KDP choose from the column's shape.</td></tr>
+      <tr><td><code>none</code></td><td>&mdash;</td><td>Pass the values through unchanged.</td></tr>
+      <tr><td><code>log</code></td><td>Strictly positive</td><td>Long right tails, such as income.</td></tr>
+      <tr><td><code>sqrt</code></td><td>Non-negative</td><td>Milder right skew; tolerates zeros.</td></tr>
+      <tr><td><code>cube-root</code></td><td>Any sign</td><td>Skew in data that also goes negative.</td></tr>
+      <tr><td><code>arcsinh</code></td><td>Any sign</td><td>Log-like compression that accepts zero and negatives.</td></tr>
+      <tr><td><code>box-cox</code></td><td>Strictly positive</td><td>Power transform toward normality.</td></tr>
+      <tr><td><code>yeo-johnson</code></td><td>Any sign</td><td>Box-Cox for data including zero and negatives.</td></tr>
+      <tr><td><code>logit</code></td><td>Values inside (0, 1)</td><td>Proportions and rates.</td></tr>
+      <tr><td><code>min-max</code></td><td>&mdash;</td><td>Rescale to a fixed range.</td></tr>
+      <tr><td><code>robust-scale</code></td><td>&mdash;</td><td>Median and IQR per feature; resists outliers.</td></tr>
+      <tr><td><code>quantile</code></td><td>&mdash;</td><td>Rank-based, per feature; flattens any shape.</td></tr>
+    </tbody>
+  </table>
+</div>
+
+!!! tip "`auto` narrows the candidates to what the data allows"
+    A column with zeros never gets `log`, and one with negative values never
+    gets `box-cox`, so `auto` cannot produce infinities from an unsuitable
+    transform. Restrict the search yourself with
+    `auto_candidates=["log", "sqrt"]`.
+
+### On the `DistributionAwareEncoder` layer
+
+The processor builds this layer for you with `detect_periodicity=True`,
+`handle_sparsity=True`, `adaptive_binning=True` and `mixture_components=3`
+fixed. To change them, use the layer directly through a
+[custom preprocessing pipeline](custom-preprocessing.md) &mdash; passing them
+to `PreprocessingModel` raises `TypeError`.
+
+<div class="table-container">
+  <table class="config-table">
+    <thead>
+      <tr>
+        <th>Parameter</th>
+        <th>Type</th>
+        <th>Default</th>
+        <th>Description</th>
+      </tr>
+    </thead>
+    <tbody>
       <tr>
         <td><code>detect_periodicity</code></td>
         <td>bool</td>
@@ -286,6 +372,12 @@ preprocessor = PreprocessingModel(
         <td>True</td>
         <td>Special handling for sparse data</td>
       </tr>
+      <tr>
+        <td><code>adaptive_binning</code></td>
+        <td>bool</td>
+        <td>None</td>
+        <td>Learn bin boundaries from the batch rather than using fixed ones</td>
+      </tr>
       <tr>
         <td><code>embedding_dim</code></td>
         <td>int</td>
@@ -385,10 +477,7 @@ preprocessor = PreprocessingModel(
     features_specs=features_specs,
     use_distribution_aware=True,
     distribution_aware_bins=1000,
-    detect_periodicity=True,  # For daily/weekly patterns
-    handle_sparsity=True,    # For low-volume periods
     embedding_dim=32,        # Project to fixed dimension
-    add_distribution_embedding=True  # Add distribution information
 )
 ```
 
@@ -400,6 +489,8 @@ preprocessor = PreprocessingModel(
     <div class="code-container">
 
 ```python
+from kdp import DistributionType, FeatureType, NumericalFeature, PreprocessingModel
+
 features_specs = {
     "temperature": NumericalFeature(
         name="temperature",
@@ -423,8 +514,6 @@ preprocessor = PreprocessingModel(
     features_specs=features_specs,
     use_distribution_aware=True,
     distribution_aware_bins=500,  # Fewer bins for simpler distributions
-    detect_periodicity=True,      # For daily temperature cycles
-    handle_sparsity=False,       # No sparse data expected
     embedding_dim=16             # Smaller embedding for simpler patterns
 )
 ```
@@ -457,7 +546,7 @@ preprocessor = PreprocessingModel(
     <span class="topic-icon">👁️</span>
     <span class="topic-text">Tabular Attention</span>
   </a>
-  <a href="feature-selection.md" class="topic-link">
+  <a href="../optimization/feature-selection.md" class="topic-link">
     <span class="topic-icon">🎯</span>
     <span class="topic-text">Feature Selection</span>
   </a>