diff --git a/.github/templates/Python/mkdocs/action.yml b/.github/templates/Python/mkdocs/action.yml index b3df8ac..d23e02f 100644 --- a/.github/templates/Python/mkdocs/action.yml +++ b/.github/templates/Python/mkdocs/action.yml @@ -11,7 +11,7 @@ runs: using: "composite" steps: - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "${{ inputs.PYTHON_VERSION }}" cache: "pip" diff --git a/.github/templates/Python/poetry_build/action.yml b/.github/templates/Python/poetry_build/action.yml index a2d1ba7..a688d85 100644 --- a/.github/templates/Python/poetry_build/action.yml +++ b/.github/templates/Python/poetry_build/action.yml @@ -29,7 +29,7 @@ runs: using: "composite" steps: - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "${{ inputs.PYTHON_VERSION }}" diff --git a/.github/templates/Python/unittests/action.yml b/.github/templates/Python/unittests/action.yml index 78eb659..178f93b 100644 --- a/.github/templates/Python/unittests/action.yml +++ b/.github/templates/Python/unittests/action.yml @@ -14,9 +14,9 @@ inputs: runs: using: "composite" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "${{ inputs.PYTHON_VERSION }}" - name: Install dependencies diff --git a/.github/workflows/MANUAL_DOCS_PUBLISH.yml b/.github/workflows/MANUAL_DOCS_PUBLISH.yml index 692f90f..7dece34 100644 --- a/.github/workflows/MANUAL_DOCS_PUBLISH.yml +++ b/.github/workflows/MANUAL_DOCS_PUBLISH.yml @@ -21,13 +21,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: main fetch-depth: 1 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 cache: 'pip' diff --git a/.github/workflows/MANUAL_PYPI_PUBLISH.yml b/.github/workflows/MANUAL_PYPI_PUBLISH.yml index f11a05b..77fec8f 100644 --- a/.github/workflows/MANUAL_PYPI_PUBLISH.yml +++ b/.github/workflows/MANUAL_PYPI_PUBLISH.yml @@ -16,13 +16,13 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: main fetch-depth: 1 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 cache: 'pip' diff --git a/.github/workflows/PRECOMMITS.yml b/.github/workflows/PRECOMMITS.yml index bec6a09..e789af5 100644 --- a/.github/workflows/PRECOMMITS.yml +++ b/.github/workflows/PRECOMMITS.yml @@ -21,30 +21,44 @@ jobs: pre-commit-checks: runs-on: ["ubuntu-latest"] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 + - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - - name: Check Pull Request Title - uses: Slashgear/action-check-pr-title@main + + # Kept in step with the local commit-msg hook in .pre-commit-config.yaml. + # Checking it here rather than through a third-party action keeps the two + # definitions in one repository and removes an unpinned dependency. + - name: Check pull request title + if: github.event_name == 'pull_request' + env: + PR_TITLE: ${{ github.event.pull_request.title }} + run: | + PATTERN='^(break|build|ci|docs|feat|fix|perf|refactor|style|test|ops|hotfix|release|maint|init|enh|revert)\([^)]+\)!?: .+' + if ! printf '%s' "$PR_TITLE" | grep -Eq "$PATTERN"; then + echo "::error::Pull request title does not follow the conventional-commit format." + echo "Got: $PR_TITLE" + echo "Expected: type(scope): description" + echo "Types: break, build, ci, docs, feat, fix, perf, refactor, style," + echo " test, ops, hotfix, release, maint, init, enh, revert" + exit 1 + fi + echo "Pull request title is valid." + + - name: Cache pre-commit environments + uses: actions/cache@v4 with: - regexp: '(break|build|ci|docs|feat|fix|perf|refactor|style|test|ops|hotfix)\([a-z,A-Z,0-9,\-,\_,\/,:]+\)(:)\s{1}([\w\s]+)' # Regex the title should match. - - name: Getting changed files list - id: files - uses: jitterbit/get-changed-files@master - - name: Checking chaged files + path: ~/.cache/pre-commit + key: pre-commit-${{ hashFiles('.pre-commit-config.yaml') }} + + # `--all-files` rather than a changed-files list: the repository is clean + # under every hook, so this is both simpler and stricter, and it drops the + # unpinned third-party action the changed-files plumbing needed. + - name: Pre-commit checks shell: bash - run: | - echo "Changed files" - echo ${{ steps.files.outputs.all }} - echo "Github Client version" - echo $(gh --version) - - name: Pre-Commit Checks run: | python -m pip install --upgrade pip - pip install pre-commit==3.3.3 - echo "Running pre-commit scans:" - # adding log display in case of pre-commit errors - pre-commit run -v --files ${{ steps.files.outputs.all }} - shell: bash + pip install "pre-commit>=3.6,<4" + pre-commit run --all-files --show-diff-on-failure diff --git a/.github/workflows/PR_PREVIEW.yml b/.github/workflows/PR_PREVIEW.yml index 2c1a032..7df4c89 100644 --- a/.github/workflows/PR_PREVIEW.yml +++ b/.github/workflows/PR_PREVIEW.yml @@ -28,10 +28,10 @@ jobs: runs-on: ["ubuntu-latest"] steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Set up Python 3.11 - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" cache: "pip" diff --git a/.github/workflows/RELEASE.yml b/.github/workflows/RELEASE.yml index 703af3e..b9c5af4 100644 --- a/.github/workflows/RELEASE.yml +++ b/.github/workflows/RELEASE.yml @@ -32,7 +32,7 @@ jobs: release_created: ${{ steps.semantic-release.outputs.new_release_published }} steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} @@ -68,11 +68,13 @@ jobs: PYPI_PUBLISH: needs: [SEMANTIC_RELEASE] - if: ${{ needs.SEMANTIC_RELEASE.outputs.release_created == 'true' || inputs.SKIP_RELEASE }} + # `!cancelled()` is required so this still runs when SKIP_RELEASE skips + # SEMANTIC_RELEASE: a skipped dependency otherwise skips this job too. + if: ${{ !cancelled() && (needs.SEMANTIC_RELEASE.outputs.release_created == 'true' || inputs.SKIP_RELEASE) }} runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: main fetch-depth: 1 @@ -100,7 +102,7 @@ jobs: fi - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 cache: 'pip' @@ -143,11 +145,15 @@ jobs: UPDATE_DOCS: needs: [SEMANTIC_RELEASE, PYPI_PUBLISH] - if: ${{ (needs.SEMANTIC_RELEASE.outputs.release_created == 'true' && !failure('PYPI_PUBLISH')) || inputs.SKIP_RELEASE }} + # A job's result is read from `needs..result`; `failure()` takes no + # arguments, so `failure('PYPI_PUBLISH')` never tested what it looked like. + if: >- + ${{ !cancelled() && needs.PYPI_PUBLISH.result != 'failure' + && (needs.SEMANTIC_RELEASE.outputs.release_created == 'true' || inputs.SKIP_RELEASE) }} runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: main fetch-depth: 1 @@ -175,7 +181,7 @@ jobs: fi - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: 3.11 cache: 'pip' diff --git a/.github/workflows/UTESTS.yml b/.github/workflows/UTESTS.yml index 0e108b3..3547bb9 100644 --- a/.github/workflows/UTESTS.yml +++ b/.github/workflows/UTESTS.yml @@ -50,7 +50,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" - name: Install dependencies @@ -82,7 +82,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} @@ -133,6 +133,33 @@ jobs: htmlcov/ retention-days: 7 + # The package targets the Keras 3 API directly rather than the tf.keras shim, + # so it must keep working with Keras releases newer than the one TensorFlow + # happens to pin. This job catches a Keras 3 change before users do. + keras-latest: + runs-on: ubuntu-latest + needs: [smoke-test] + if: always() && (needs.smoke-test.result == 'success' || github.event_name != 'pull_request') + continue-on-error: true + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: "3.11" + + - name: Install dependencies with the newest Keras 3 + run: | + python -m pip install --upgrade pip + pip install poetry + poetry install --no-interaction + poetry run pip install --upgrade "keras>=3,<4" + poetry run python -c "import keras; print('keras', keras.__version__)" + + - name: Run tests + run: poetry run pytest -c pytest-fast.ini --timeout=180 + timeout-minutes: 15 + # Coverage job (only on main Python version) coverage: runs-on: ubuntu-latest @@ -141,7 +168,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" @@ -178,7 +205,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: "3.11" diff --git a/.isort.cfg b/.isort.cfg deleted file mode 100644 index 8eacc94..0000000 --- a/.isort.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[settings] -profile=black -multi_line_output=3 -include_trailing_comma=True -force_grid_wrap=0 -use_parentheses=True -ensure_newline_before_comments=True -line_length=88 diff --git a/.ruff.toml b/.ruff.toml deleted file mode 100644 index e69de29..0000000 diff --git a/Makefile b/Makefile index 538e2ce..c38b7e3 100644 --- a/Makefile +++ b/Makefile @@ -3,7 +3,12 @@ # ------------------------------------ HAS_POETRY := $(shell command -v poetry 2> /dev/null) -POETRY_VERSION := $(shell poetry version $(shell git describe --tags --abbrev=0)) + +# Sets the package version from the latest git tag. This MUST stay a lazily +# expanded recipe command: as a `:=` shell assignment it ran `poetry version +# ` while make was still parsing the file, so every single make target -- +# `make help` included -- silently rewrote the version in pyproject.toml. +SET_VERSION_FROM_TAG = poetry version $$(git describe --tags --abbrev=0) # ------------------------------------ # Test @@ -101,10 +106,10 @@ unittests: test .PHONY: clean_tests ## Remove pytest cache and test artifacts clean_tests: - find . -type d -name .pytest_cache -exec rm -r {} + - find . -type d -name __pycache__ -exec rm -r {} + - find . -type f -name '*junit_report.xml' -exec rm {} + - find . -type f -name '*.pyc' -exec rm {} + + find . -type d -name .pytest_cache -prune -exec rm -rf {} + 2>/dev/null || true + find . -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true + find . -type f -name '*junit_report.xml' -delete + find . -type f -name '*.pyc' -delete rm -rf htmlcov/ rm -f .coverage rm -f coverage.xml @@ -125,7 +130,7 @@ coverage: build_pkg: @echo "Start to build pkg" ifdef HAS_POETRY - @$(POETRY_VERSION) + @$(SET_VERSION_FROM_TAG) poetry build else @echo "To build the package, you need to have poetry first" @@ -139,8 +144,9 @@ build: clean_built build_pkg .PHONY: clean_built ## Remove cache, built package, and docs directories after build or installation clean_built: - find . -type d -name dist -exec rm -r {} + - find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete + rm -rf dist + find . -type f -name '*.py[co]' -delete + find . -type d -name __pycache__ -prune -exec rm -rf {} + 2>/dev/null || true # ------------------------------------ # Build doc @@ -172,7 +178,7 @@ docs_deploy: generate_all_diagrams @echo "Starting to build docs" @echo "more info: https://squidfunk.github.io/mkdocs-material/setup/setting-up-versioning/" ifdef HAS_POETRY - @$(POETRY_VERSION) + @$(SET_VERSION_FROM_TAG) poetry version -s | xargs -I {} sh -c 'echo Deploying version {} && mike deploy --push --update-aliases {} latest' else @echo "To build the docs, you need to have poetry first" @@ -223,10 +229,8 @@ clean_old_diagrams: identify_unused_diagrams .PHONY: clean ## Remove cache, built package, and docs directories after build or installation -clean: clean_old_diagrams - find . -type d -name dist -exec rm -r {} + - find . -type f -name '*.rst' ! -name 'index.rst' -delete - find . -type f -name '*.py[co]' -delete -o -type d -name __pycache__ -delete +clean: clean_tests clean_built clean_old_diagrams + @echo "Workspace cleaned" # ------------------------------------ # Default diff --git a/README.md b/README.md index 214edf1..61b90f9 100644 --- a/README.md +++ b/README.md @@ -10,8 +10,9 @@ -[![Python 3.9+](https://img.shields.io/badge/python-3.9+-blue.svg)](https://www.python.org/downloads/) +[![Python 3.10+](https://img.shields.io/badge/python-3.10+-blue.svg)](https://www.python.org/downloads/) [![TensorFlow 2.18+](https://img.shields.io/badge/tensorflow-2.18+-orange.svg)](https://www.tensorflow.org/) +[![Keras 3](https://img.shields.io/badge/keras-3-red.svg)](https://keras.io/) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![🦄 UnicoLab](https://img.shields.io/badge/UnicoLab-Enterprise%20AI-blue.svg)](https://unicolab.ai) [![Documentation](https://img.shields.io/badge/docs-latest-blue.svg)](https://unicolab.github.io/keras-data-processor/) @@ -73,29 +74,29 @@ We've built an extensive documentation system to help you get the most from KDP: ### Core Guides -- [🚀 Quick Start Guide](docs/quick_start.md) - Get up and running in minutes -- [📊 Feature Processing](docs/features.md) - Learn about all supported feature types -- [🧙‍♂️ Auto-Configuration](docs/auto_configuration.md) - Let KDP configure itself for your data +- [🚀 Quick Start Guide](docs/getting-started/quick-start.md) - Get up and running in minutes +- [📊 Feature Processing](docs/features/overview.md) - Learn about all supported feature types +- [🧙‍♂️ Auto-Configuration](docs/optimization/auto-configuration.md) - Let KDP configure itself for your data ### Advanced Topics -- [📈 Distribution-Aware Encoding](docs/distribution_aware_encoder.md) - Smart handling of different distributions -- [👁️ Tabular Attention](docs/tabular_attention.md) - Capture complex feature interactions -- [🔢 Advanced Numerical Embeddings](docs/advanced_numerical_embeddings.md) - Rich representations for numbers -- [🤖 Transformer Blocks](docs/transformer_blocks.md) - Apply transformer architecture to tabular data -- [🎯 Feature Selection](docs/feature_selection.md) - Focus on what matters in your data -- [🧠 Feature-wise Mixture of Experts](docs/feature_moe.md) - Specialized processing per feature +- [📈 Distribution-Aware Encoding](docs/advanced/distribution-aware-encoding.md) - Smart handling of different distributions +- [👁️ Tabular Attention](docs/advanced/tabular-attention.md) - Capture complex feature interactions +- [🔢 Advanced Numerical Embeddings](docs/advanced/numerical-embeddings.md) - Rich representations for numbers +- [🤖 Transformer Blocks](docs/advanced/transformer-blocks.md) - Apply transformer architecture to tabular data +- [🎯 Feature Selection](docs/optimization/feature-selection.md) - Focus on what matters in your data +- [🧠 Feature-wise Mixture of Experts](docs/advanced/feature-moe.md) - Specialized processing per feature ### Integration & Performance -- [🔗 Integration Guide](docs/integrations.md) - Use KDP with existing ML pipelines -- [🚀 Tabular Optimization](docs/tabular_optimization.md) - Supercharge your preprocessing -- [📈 Performance Tips](docs/complex_examples.md) - Handling large datasets efficiently +- [🔗 Integration Guide](docs/integrations/overview.md) - Use KDP with existing ML pipelines +- [🚀 Tabular Optimization](docs/optimization/tabular-optimization.md) - Supercharge your preprocessing +- [📈 Performance Tips](docs/examples/complex-examples.md) - Handling large datasets efficiently ### Background & Resources -- [💡 Motivation](docs/motivation.md) - Why we built KDP -- [🤝 Contributing](docs/contributing.md) - Help improve KDP +- [💡 Motivation](docs/getting-started/motivation.md) - Why we built KDP +- [🤝 Contributing](docs/contributing/overview.md) - Help improve KDP ## 🖼️ Model Architecture @@ -116,7 +117,7 @@ KDP outperforms alternative preprocessing approaches, especially as data size in ## 🤝 Contributing -We welcome contributions! Please check out our [Contributing Guide](docs/contributing.md) for guidelines on how to proceed. +We welcome contributions! Please check out our [Contributing Guide](docs/contributing/overview.md) for guidelines on how to proceed. ## 💬 Join Our Community diff --git a/docs/getting-started/installation.md b/docs/getting-started/installation.md index 7ea5acd..a9c7dc0 100644 --- a/docs/getting-started/installation.md +++ b/docs/getting-started/installation.md @@ -99,8 +99,8 @@ poetry install

Core Dependencies

    -
  • 🐍 Python 3.7+
    • -
  • 🔄 TensorFlow 2.5+
    • +
  • 🐍 Python 3.10+
    • +
  • 🔄 TensorFlow 2.18+ (Keras 3)
  • 🔢 NumPy 1.19+
  • 📊 Pandas 1.2+
diff --git a/docs/getting-started/upgrading.md b/docs/getting-started/upgrading.md new file mode 100644 index 0000000..c53123a --- /dev/null +++ b/docs/getting-started/upgrading.md @@ -0,0 +1,127 @@ +# ⬆️ Upgrading from 1.11.x + +
+
+

Upgrade Guide

+

What changes when you move off 1.11.x, and what you need to do about it.

+
+
+ +## 📋 Overview + +
+

This release repairs defects that produced wrong numbers rather than errors. Pipelines that ran without complaint on 1.11.x will keep running, but several of them will now emit different values — correct ones. Re-train any downstream model that was fitted on the old output.

+
+ +## ⚠️ Output values change + +These transforms returned incorrect values before. Nothing about your +configuration changes, but the numbers coming out of the preprocessor do. + +| Area | Before | Now | +|------|--------|-----| +| Time series layers | `DifferencingLayer`, `MovingAverageLayer`, `RollingStatsLayer`, `LagFeatureLayer` and `AutoLagSelectionLayer` returned fixed constants for certain shapes instead of computing over the input | Values are computed from the data in every case | +| `DistributionTransformLayer` — `robust-scale` | Median and IQR were taken across the whole tensor | Computed per feature, as the transform is defined | +| `DistributionTransformLayer` — `quantile` | Ranking was global | Ranked per feature | +| Distribution-aware encoding | `preferred_distribution` was accepted and then ignored, so the layer always auto-detected | The requested distribution is honoured | +| Output width | `get_output_dim()` combined dimensions additively for some feature configurations | Combined multiplicatively, matching the tensor the model actually produces | +| Time series + other features | A time series feature could not be combined with any other feature: the warm-up rows `drop_na` removes left that column shorter and `Concatenate` raised | Transforms preserve rows by default, so the documented combinations build | + +!!! warning "Re-train downstream models" + If you trained a model on top of KDP output from 1.11.x and any of the + rows above apply to your feature set, the preprocessor now feeds it + different numbers. Re-fit before you compare metrics. + +## 🗂️ Existing `features_stats.json` files + +Statistics files written by 1.11.2 and earlier stored categorical +vocabularies as byte reprs — the literal string `"b'paris'"` rather than +`paris`. Such a file loaded without complaint and produced a model in which +**every category missed the vocabulary and encoded identically to an unseen +value**. + +KDP now detects that format and recomputes from the data instead of trusting +it, logging a warning that names the file. You do not have to do anything, +but be aware that: + +- the first build after upgrading re-reads your dataset, so it takes as long + as a fresh statistics pass; +- `path_data` must still point at the data for that recomputation to happen. + +To recompute deliberately instead, pass `overwrite_stats=True` once. + +## 🐍 Requirements + +
+ +``` +Python >= 3.10 # 1.11.x declared 3.9, but never ran on it +keras >= 3.5, < 4.0 # now an explicit dependency +``` + +
+ +The package previously reached Keras through the `tf.keras` compatibility +shim. It targets the Keras 3 API directly, which means it is no longer tied +to whichever Keras version TensorFlow happens to pin, and it is tested +against the newest Keras 3 release on every pull request. The `tf-keras` +dependency is gone. + +The 3.9 requirement in 1.11.x was a metadata error rather than a supported +configuration — the source already used `X | None` type syntax, which 3.9 +cannot parse — so no working 3.9 installation exists to break. + +## ✅ Things that now work that did not before + +None of these need a change on your side; they simply stop failing. + +
+ +```python +# Saving and reloading a preprocessor. Custom layers are registered as +# serializable, so a saved model round-trips. +preprocessor.model.save("model.keras") +reloaded = keras.models.load_model("model.keras") + +# predict() with plain Python or NumPy input, including string columns. +preprocessor.predict({ + "age": [[35.0]], + "city": [["paris"]], +}) + +# Custom pipelines by layer name -- the spelling the docs lead with. +Feature( + name="value", + feature_type=FeatureType.FLOAT_NORMALIZED, + preprocessors=["Rescaling", "Dense"], + scale=2.0, + units=4, +) + +# auto_configure() returns real per-feature recommendations rather than an +# empty set, and DatasetStatistics works from features_specs alone. +auto_configure("data.csv", features_specs=specs) +``` + +
+ +Building a model whose features learn nothing from the data — hashing +categoricals with an explicit `hash_bucket_size` — no longer requires +`path_data`, because no statistics pass runs. When a feature *does* need +statistics and `path_data` is missing, the error now names the parameter +instead of surfacing as a `TypeError` from inside `tf.data`. + +## 🔀 Moved symbols + +`FeatureSpaceConverter` now lives in `kdp.features`, next to the feature +classes it converts. It is re-exported from `kdp.processor`, so existing +imports keep working: + +
+ +```python +from kdp.features import FeatureSpaceConverter # preferred +from kdp.processor import FeatureSpaceConverter # still works +``` + +
diff --git a/docs/time_series_inference.md b/docs/time_series_inference.md index 4a87fec..c13721f 100644 --- a/docs/time_series_inference.md +++ b/docs/time_series_inference.md @@ -206,7 +206,9 @@ For more complex scenarios, the formatter provides additional options: formatted_data = formatter.prepare_inference_data( data=new_data, historical_data=historical_data, - fill_missing=True, # Try to fill missing values or context + fill_missing=True, # Inert; see the API docs. Missing values inside + # sufficient history are handled by + # missing_value_config on the feature itself. to_tensors=False # Keep as Python/NumPy types for inspection ) diff --git a/examples/advanced_factory_methods_test.py b/examples/advanced_factory_methods_test.py index dd7e5ef..7963ca7 100644 --- a/examples/advanced_factory_methods_test.py +++ b/examples/advanced_factory_methods_test.py @@ -12,7 +12,7 @@ n_samples = 1000 # Log-normal distribution (right-skewed) log_normal_data = np.random.lognormal(mean=0, sigma=1, size=(n_samples, 1)).astype( - np.float32 + np.float32, ) # Bimodal distribution bimodal_data = np.concatenate( @@ -30,13 +30,17 @@ # Create distribution transform layers with different transformations log_transform = PreprocessorLayerFactory.distribution_transform_layer( - transform_type="log", name="log_transform" + transform_type="log", + name="log_transform", ) box_cox_transform = PreprocessorLayerFactory.distribution_transform_layer( - transform_type="box-cox", lambda_param=0.5, name="box_cox_transform" + transform_type="box-cox", + lambda_param=0.5, + name="box_cox_transform", ) auto_transform = PreprocessorLayerFactory.distribution_transform_layer( - transform_type="auto", name="auto_transform" + transform_type="auto", + name="auto_transform", ) # Apply transformations @@ -49,16 +53,16 @@ print(f"Box-Cox transformed data shape: {box_cox_transformed.shape}") print(f"Auto-transformed data shape: {auto_transformed.shape}") print( - f"Original data range: [{np.min(log_normal_data):.2f}, {np.max(log_normal_data):.2f}]" + f"Original data range: [{np.min(log_normal_data):.2f}, {np.max(log_normal_data):.2f}]", ) print( - f"Log-transformed range: [{np.min(log_transformed):.2f}, {np.max(log_transformed):.2f}]" + f"Log-transformed range: [{np.min(log_transformed):.2f}, {np.max(log_transformed):.2f}]", ) print( - f"Box-Cox transformed range: [{np.min(box_cox_transformed):.2f}, {np.max(box_cox_transformed):.2f}]" + f"Box-Cox transformed range: [{np.min(box_cox_transformed):.2f}, {np.max(box_cox_transformed):.2f}]", ) print( - f"Auto-transformed range: [{np.min(auto_transformed):.2f}, {np.max(auto_transformed):.2f}]" + f"Auto-transformed range: [{np.min(auto_transformed):.2f}, {np.max(auto_transformed):.2f}]", ) print("\n2. Testing Numerical Embedding Layer") @@ -86,7 +90,8 @@ # Create multi-feature data multi_feature_data = np.concatenate( - [log_normal_data, uniform_data, bimodal_data], axis=1 + [log_normal_data, uniform_data, bimodal_data], + axis=1, ) print(f"Multi-feature data shape: {multi_feature_data.shape}") @@ -121,7 +126,9 @@ # Create gated residual network layer grn = PreprocessorLayerFactory.gated_residual_network_layer( - units=8, dropout_rate=0.2, name="grn" + units=8, + dropout_rate=0.2, + name="grn", ) # Apply GRN diff --git a/examples/custom_preprocessing_example.py b/examples/custom_preprocessing_example.py index 87f4af6..434bdc0 100644 --- a/examples/custom_preprocessing_example.py +++ b/examples/custom_preprocessing_example.py @@ -6,8 +6,11 @@ """ # ruff: noqa: E402 +import keras import os import sys +import tempfile +from pathlib import Path # Add the project root to the Python path to allow module imports sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -40,7 +43,7 @@ ################################################# -class LogTransformLayer(tf.keras.layers.Layer): +class LogTransformLayer(keras.layers.Layer): """ Custom layer that applies log(x + offset) transformation. Useful for handling right-skewed data. @@ -59,7 +62,7 @@ def get_config(self): return config -class ClippingLayer(tf.keras.layers.Layer): +class ClippingLayer(keras.layers.Layer): """ Custom layer that clips values between min_value and max_value. Useful for handling outliers. @@ -79,7 +82,7 @@ def get_config(self): return config -class StringCleaningLayer(tf.keras.layers.Layer): +class StringCleaningLayer(keras.layers.Layer): """ Custom layer that performs basic text cleaning operations. - Converts to lowercase @@ -101,9 +104,7 @@ def call(self, inputs): # Trim whitespace and replace multiple spaces with a single space text = tf.strings.regex_replace(text, r"\s+", " ") - text = tf.strings.strip(text) - - return text + return tf.strings.strip(text) def get_config(self): config = super().get_config() @@ -116,6 +117,45 @@ def get_config(self): ################################################# +JOB_DESCRIPTION_SECTORS = [ + "tech", + "finance", + "healthcare", + "education", + "manufacturing", +] +JOB_DESCRIPTION_SKILLS = [ + "programming", + "analysis", + "management", + "communication", + "design", +] +# The words a hand-built TextVectorization has to know about. KDP's own text +# path derives this from the computed statistics; a custom pipeline supplies it. +JOB_DESCRIPTION_VOCABULARY = [ + "this", + "job", + "in", + "requires", + "skills", + "and", + "experience", + "level", + *JOB_DESCRIPTION_SECTORS, + *JOB_DESCRIPTION_SKILLS, +] + +JOB_SECTORS = [ + "Technology", + "Finance", + "Healthcare", + "Education", + "Manufacturing", + "Other", +] + + def generate_sample_data(n_samples=1000): """Generate a synthetic dataset for demonstration.""" @@ -127,23 +167,21 @@ def generate_sample_data(n_samples=1000): # Categorical features education = np.random.choice( - ["High School", "Bachelor's", "Master's", "PhD", "Other"], n_samples - ) - job_sector = np.random.choice( - ["Technology", "Finance", "Healthcare", "Education", "Manufacturing", "Other"], + ["High School", "Bachelor's", "Master's", "PhD", "Other"], n_samples, ) + job_sector = np.random.choice(JOB_SECTORS, n_samples) # Text feature with some variations job_descriptions = [] - sectors = ["tech", "finance", "healthcare", "education", "manufacturing"] - skills = ["programming", "analysis", "management", "communication", "design"] + sectors = JOB_DESCRIPTION_SECTORS + skills = JOB_DESCRIPTION_SKILLS for _ in range(n_samples): sector = np.random.choice(sectors) skill1, skill2 = np.random.choice(skills, 2, replace=False) job_descriptions.append( f"This job in {sector} requires skills in {skill1} and {skill2}. " - f"Experience level: {np.random.randint(1, 10)} years." + f"Experience level: {np.random.randint(1, 10)} years.", ) # Target variable: synthetic credit score @@ -161,7 +199,7 @@ def generate_sample_data(n_samples=1000): credit_score = np.clip(credit_score, 300, 850) # Create DataFrame - data = pd.DataFrame( + return pd.DataFrame( { "age": age, "salary": salary, @@ -171,11 +209,9 @@ def generate_sample_data(n_samples=1000): "job_sector": job_sector, "job_description": job_descriptions, "credit_score": credit_score, - } + }, ) - return data - ################################################# # Define Features with Custom Preprocessing @@ -185,7 +221,7 @@ def generate_sample_data(n_samples=1000): def define_features(): """Define features with custom preprocessing pipelines.""" - features = { + return { # Age: Standard numerical feature with normalization "age": NumericalFeature(name="age", feature_type=FeatureType.FLOAT_NORMALIZED), # Salary: Log-transform to handle skewness, then normalize @@ -206,14 +242,18 @@ def define_features(): max_value=20, # for ClippingLayer scale=0.1, # for Rescaling ), - # Expense ratio: Beta distribution, apply special binning + # Expense ratio: Beta distribution, apply special binning. + # Note that the keyword arguments below are offered to *every* layer in + # the pipeline, and each takes the ones its constructor accepts. Both + # Discretization and CategoryEncoding accept `output_mode`, so chaining + # them here would one-hot the values twice; Discretization alone already + # emits the one-hot encoding. "expense_ratio": NumericalFeature( name="expense_ratio", feature_type=FeatureType.FLOAT, - preprocessors=["Discretization", "CategoryEncoding"], + preprocessors=["Discretization"], bin_boundaries=[0.1, 0.2, 0.3, 0.4, 0.5], # for Discretization - output_mode="one_hot", # for CategoryEncoding - num_tokens=6, # 5 bins + 1 for values outside bins + output_mode="one_hot", # for Discretization ), # Education: Standard categorical with embedding "education": CategoricalFeature( @@ -227,7 +267,12 @@ def define_features(): name="job_sector", feature_type=FeatureType.STRING_CATEGORICAL, preprocessors=["StringLookup", "Embedding", "Dense"], + # A raw StringLookup in a custom pipeline needs its vocabulary up + # front: KDP's own categorical path fills this in from the computed + # statistics, but a hand-built pipeline has to supply it. + vocabulary=JOB_SECTORS, # for StringLookup num_oov_indices=1, # for StringLookup + input_dim=len(JOB_SECTORS) + 1, # for Embedding: vocabulary + OOV output_dim=12, # for Embedding units=8, # for Dense activation="relu", @@ -243,14 +288,15 @@ def define_features(): "GlobalAveragePooling1D", ], patterns_to_remove=["[0-9]+", "years"], # for StringCleaningLayer - max_tokens=5000, # for TextVectorization + vocabulary=JOB_DESCRIPTION_VOCABULARY, # for TextVectorization output_sequence_length=30, # for TextVectorization + # Embedding must cover the vocabulary plus the padding and OOV slots + # TextVectorization prepends. + input_dim=len(JOB_DESCRIPTION_VOCABULARY) + 2, # for Embedding output_dim=32, # for Embedding ), } - return features - ################################################# # Main Execution @@ -258,6 +304,7 @@ def define_features(): def main(): + """Build a preprocessor from custom pipelines and train a model on its output.""" # Generate sample data print("Generating sample data...") data = generate_sample_data() @@ -275,31 +322,47 @@ def main(): print("\nDefining features with custom preprocessing pipelines...") features = define_features() - # Create preprocessing model + # KDP computes its statistics from data on disk, so the training split is + # written out first and the preprocessor is pointed at that file. + work_dir = Path(tempfile.mkdtemp(prefix="kdp_custom_preprocessing_")) + train_csv = work_dir / "train.csv" + X_train.to_csv(train_csv, index=False) + print("Creating preprocessing model...") - preprocessor = PreprocessingModel(features=features, output_mode="concat") + preprocessor = PreprocessingModel( + path_data=str(train_csv), + features_specs=features, + features_stats_path=str(work_dir / "features_stats.json"), + overwrite_stats=True, + output_mode="concat", + ) - # Fit preprocessing model - print("Fitting preprocessing model...") - preprocessor.fit(X_train) + print("Building preprocessing model...") + preprocessor.build_preprocessor() + + def to_model_inputs(frame: pd.DataFrame) -> dict: + """Turn a DataFrame into the per-feature tensors the model expects.""" + return { + column: tf.constant(frame[column].to_numpy().reshape(-1, 1)) + for column in features + } - # Transform data print("Transforming data with custom preprocessing...") - X_train_processed = preprocessor.transform(X_train) - X_test_processed = preprocessor.transform(X_test) + X_train_processed = np.asarray(preprocessor.model(to_model_inputs(X_train))) + X_test_processed = np.asarray(preprocessor.model(to_model_inputs(X_test))) print(f"Processed training data shape: {X_train_processed.shape}") print(f"Processed testing data shape: {X_test_processed.shape}") # Build prediction model using the preprocessed data print("\nBuilding prediction model...") - model = tf.keras.Sequential( + model = keras.Sequential( [ - tf.keras.layers.Dense(64, activation="relu"), - tf.keras.layers.Dropout(0.2), - tf.keras.layers.Dense(32, activation="relu"), - tf.keras.layers.Dense(1), - ] + keras.layers.Dense(64, activation="relu"), + keras.layers.Dropout(0.2), + keras.layers.Dense(32, activation="relu"), + keras.layers.Dense(1), + ], ) # Compile the model @@ -326,11 +389,11 @@ def main(): # Save the preprocessing model and prediction model print("\nSaving models...") - preprocessor.save("custom_preprocessing_model") - model.save("credit_score_prediction_model") + preprocessor.save_model(str(work_dir / "custom_preprocessing_model")) + model.save(str(work_dir / "credit_score_prediction_model.keras")) print( - "Done! The preprocessor with custom pipelines and prediction model are saved." + "Done! The preprocessor with custom pipelines and prediction model are saved.", ) # Example of how to use the saved models for inference @@ -344,13 +407,13 @@ def main(): "education": ["Master's"], "job_sector": ["Technology"], "job_description": [ - "This job in tech requires skills in programming and management." + "This job in tech requires skills in programming and management.", ], - } + }, ) # Preprocess the new sample - new_sample_processed = preprocessor.transform(new_sample) + new_sample_processed = np.asarray(preprocessor.model(to_model_inputs(new_sample))) # Make prediction prediction = model.predict(new_sample_processed) diff --git a/examples/dynamic_pipeline_examples.py b/examples/dynamic_pipeline_examples.py index e335381..1a6c487 100644 --- a/examples/dynamic_pipeline_examples.py +++ b/examples/dynamic_pipeline_examples.py @@ -6,6 +6,7 @@ """ # ruff: noqa: E402 +import keras import numpy as np import tensorflow as tf import logging @@ -28,7 +29,7 @@ # Example 1: Basic Custom Layers -class ScalingLayer(tf.keras.layers.Layer): +class ScalingLayer(keras.layers.Layer): """Custom layer to scale numeric input by a factor.""" def __init__(self, scaling_factor=2.0, **kwargs): @@ -44,7 +45,7 @@ def get_config(self): return config -class NormalizationLayer(tf.keras.layers.Layer): +class NormalizationLayer(keras.layers.Layer): """Custom layer to normalize input to have mean 0 and std 1.""" def __init__(self, **kwargs): @@ -59,7 +60,7 @@ def get_config(self): return super().get_config() -class LogTransformLayer(tf.keras.layers.Layer): +class LogTransformLayer(keras.layers.Layer): """Custom layer to apply log transformation to input.""" def __init__(self, offset=1.0, **kwargs): @@ -90,7 +91,7 @@ def example_1_basic_pipeline(): data = np.array([[1.0], [2.0], [3.0], [4.0], [5.0]], dtype=np.float32) # Input features need to have keys matching layer names for processing dataset = tf.data.Dataset.from_tensor_slices( - {"scaling": data, "normalization": data} + {"scaling": data, "normalization": data}, ).batch(2) # Process the data @@ -126,7 +127,7 @@ def example_2_dependency_pipeline(): dataset = tf.data.Dataset.from_tensor_slices( { "scaling": data, - } + }, ).batch(2) # Process the data @@ -142,7 +143,7 @@ def example_2_dependency_pipeline(): # Example 3: Processing Multiple Features -class EncodingLayer(tf.keras.layers.Layer): +class EncodingLayer(keras.layers.Layer): """Custom layer for encoding categorical features using one-hot encoding.""" def __init__(self, vocabulary=None, **kwargs): @@ -151,8 +152,10 @@ def __init__(self, vocabulary=None, **kwargs): def build(self, input_shape): # Create a lookup table for categorical values - self.lookup_table = tf.keras.layers.StringLookup( - vocabulary=self.vocabulary, mask_token=None, num_oov_indices=1 + self.lookup_table = keras.layers.StringLookup( + vocabulary=self.vocabulary, + mask_token=None, + num_oov_indices=1, ) super().build(input_shape) @@ -178,7 +181,8 @@ def example_3_multiple_features(): # Create layers for categorical features categorical_encoding = EncodingLayer( - vocabulary=["A", "B", "C"], name="categorical_encoding" + vocabulary=["A", "B", "C"], + name="categorical_encoding", ) categorical_pipeline = DynamicPreprocessingPipeline([categorical_encoding]) @@ -190,11 +194,11 @@ def example_3_multiple_features(): # Create separate datasets for each feature type numeric_dataset = tf.data.Dataset.from_tensor_slices( - {"numeric_scaling": numeric_data} + {"numeric_scaling": numeric_data}, ).batch(2) categorical_dataset = tf.data.Dataset.from_tensor_slices( - {"categorical_encoding": categorical_data} + {"categorical_encoding": categorical_data}, ).batch(2) # Process each dataset through its respective pipeline @@ -232,15 +236,15 @@ def example_4_keras_integration(): # Create preprocessing pipeline preprocess_pipeline = DynamicPreprocessingPipeline( - [scaling_layer, normalization_layer] + [scaling_layer, normalization_layer], ) # Create a simple Keras model - inputs = tf.keras.Input(shape=(1,), name="model_input") - dense1 = tf.keras.layers.Dense(10, activation="relu")(inputs) - dense2 = tf.keras.layers.Dense(5, activation="relu")(dense1) - outputs = tf.keras.layers.Dense(1)(dense2) - model = tf.keras.Model(inputs=inputs, outputs=outputs) + inputs = keras.Input(shape=(1,), name="model_input") + dense1 = keras.layers.Dense(10, activation="relu")(inputs) + dense2 = keras.layers.Dense(5, activation="relu")(dense1) + outputs = keras.layers.Dense(1)(dense2) + model = keras.Model(inputs=inputs, outputs=outputs) # Compile the model model.compile(optimizer="adam", loss="mse") @@ -251,7 +255,7 @@ def example_4_keras_integration(): # Create dataset and preprocess dataset = tf.data.Dataset.from_tensor_slices( - {"scaling": data, "normalization": data, "y": targets} + {"scaling": data, "normalization": data, "y": targets}, ).batch(2) processed_dataset = preprocess_pipeline.process(dataset) @@ -279,7 +283,7 @@ def data_generator(): # Make a prediction with preprocessing test_data = np.array([[6.0]], dtype=np.float32) test_dataset = tf.data.Dataset.from_tensor_slices( - {"scaling": test_data, "normalization": test_data} + {"scaling": test_data, "normalization": test_data}, ).batch(1) processed_test = preprocess_pipeline.process(test_dataset) @@ -301,12 +305,13 @@ def example_5_normalize_transform(): data = np.random.lognormal(mean=0, sigma=1, size=(1000, 1)).astype(np.float32) # Create a normalization layer - normalize_layer = tf.keras.layers.Normalization(name="normalize") + normalize_layer = keras.layers.Normalization(name="normalize") normalize_layer.adapt(data) # Create a log transform layer using our factory log_transform = PreprocessorLayerFactory.distribution_transform_layer( - transform_type="log", name="log_transform" + transform_type="log", + name="log_transform", ) # Create our pipeline with both layers diff --git a/examples/time_series_features_example.py b/examples/time_series_features_example.py index fdba488..097a364 100644 --- a/examples/time_series_features_example.py +++ b/examples/time_series_features_example.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """ Example of using the new time series feature layers in keras-data-processor. @@ -10,8 +9,8 @@ import numpy as np import matplotlib.pyplot as plt -from tensorflow.keras.models import Model -from tensorflow.keras.layers import Input, Dense, Concatenate +from keras.models import Model +from keras.layers import Input, Dense, Concatenate from kdp.layers.time_series import ( WaveletTransformLayer, @@ -52,17 +51,16 @@ def generate_sample_data(n_samples=1000, n_features=1): signals.append(variation) signal = np.column_stack(signals) + # Target variable: the next value in the series. Predicting one step ahead + # means the final row has no target, so it is dropped from the inputs too -- + # otherwise X and y differ by one row and Keras rejects the pair. + targets = signal[1:, 0] if n_features > 1 else signal[1:] + features = signal[:-1] + # Create test/train split - train_size = int(0.8 * n_samples) - X_train = signal[:train_size] - X_test = signal[train_size:] - - # Create target variable (for regression task) - # We'll predict the next value in the series - y_train = ( - signal[1 : train_size + 1, 0] if n_features > 1 else signal[1 : train_size + 1] - ) - y_test = signal[train_size + 1 :, 0] if n_features > 1 else signal[train_size + 1 :] + train_size = int(0.8 * len(features)) + X_train, X_test = features[:train_size], features[train_size:] + y_train, y_test = targets[:train_size], targets[train_size:] return X_train, y_train, X_test, y_test @@ -73,7 +71,9 @@ def build_model_with_feature_layers(input_shape): # 1. Extract wavelet transform features wavelet_features = WaveletTransformLayer( - levels=3, window_sizes=[4, 8, 16], flatten_output=True + levels=3, + window_sizes=[4, 8, 16], + flatten_output=True, )(inputs) # 2. Extract statistical features using TSFreshFeatureLayer @@ -90,7 +90,7 @@ def build_model_with_feature_layers(input_shape): # Combine all features combined_features = Concatenate()( - [wavelet_features, tsfresh_features, lag_features] + [wavelet_features, tsfresh_features, lag_features], ) # Dense layers for prediction @@ -108,7 +108,8 @@ def main(): """Run the example.""" # Generate sample data X_train, y_train, X_test, y_test = generate_sample_data( - n_samples=1000, n_features=2 + n_samples=1000, + n_features=2, ) print(f"X_train shape: {X_train.shape}") diff --git a/examples/time_series_inference.py b/examples/time_series_inference.py index 5b03032..84ec5be 100644 --- a/examples/time_series_inference.py +++ b/examples/time_series_inference.py @@ -1,11 +1,13 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """ Example script showing how to use the InferenceDataFormatter to prepare data for time series inference. This demonstrates how to handle single-point inference, batch inference, forecasting, etc. """ +import tempfile +from pathlib import Path + import numpy as np import pandas as pd import matplotlib.pyplot as plt @@ -42,12 +44,12 @@ def generate_sample_data(num_stores=3, days_per_store=30, add_noise=True): date = base_date + timedelta(days=day) # Calculate sales based on pattern - if store_id < 2: - # Linear pattern - sales = base_sales + (day * growth) - else: - # Sinusoidal pattern - sales = base_sales + 50 * np.sin(day * 0.2) + # Linear pattern for the first stores, sinusoidal for the rest. + sales = ( + base_sales + (day * growth) + if store_id < 2 + else base_sales + 50 * np.sin(day * 0.2) + ) # Add noise if requested if add_noise: @@ -58,7 +60,7 @@ def generate_sample_data(num_stores=3, days_per_store=30, add_noise=True): "date": date.strftime("%Y-%m-%d"), "store_id": f"Store_{store_id}", "sales": sales, - } + }, ) return pd.DataFrame(all_data) @@ -85,10 +87,18 @@ def train_preprocessor(train_data): "store_id": FeatureType.STRING_CATEGORICAL, } + # KDP computes its statistics from data on disk, so `path_data` is a file + # path rather than a DataFrame; write the frame out first. + work_dir = Path(tempfile.mkdtemp(prefix="kdp_time_series_inference_")) + train_csv = work_dir / "train.csv" + train_data.to_csv(train_csv, index=False) + # Create a preprocessor with dict output to see results preprocessor = PreprocessingModel( - path_data=train_data, + path_data=str(train_csv), features_specs=features_specs, + features_stats_path=str(work_dir / "features_stats.json"), + overwrite_stats=True, output_mode="dict", ) @@ -98,7 +108,7 @@ def train_preprocessor(train_data): return preprocessor -def example_single_point_inference_failure(preprocessor, formatter): +def example_single_point_inference_failure(formatter): """Example showing how single-point inference fails with time series features.""" print("\n=== Single-Point Inference with Time Series Features ===") @@ -193,10 +203,9 @@ def example_multi_step_forecast(preprocessor, formatter, train_data): prediction = preprocessor.predict(formatted_data) # Extract the prediction value (last value in the sales array) - if isinstance(prediction, dict): - predicted_value = prediction["sales"][-1] - else: - predicted_value = prediction[-1] + predicted_value = ( + prediction["sales"][-1] if isinstance(prediction, dict) else prediction[-1] + ) # Create a result row for the forecast forecast_row = { @@ -297,11 +306,12 @@ def example_batch_inference(preprocessor, formatter, train_data): if store_indices[store]: last_idx = store_indices[store][-1] print( - f"Predicted sales for {store}: {prediction['sales'][last_idx].numpy()}" + f"Predicted sales for {store}: {prediction['sales'][last_idx].numpy()}", ) else: print( - "Prediction result:", prediction[-3:] + "Prediction result:", + prediction[-3:], ) # Last 3 values are the predictions @@ -318,7 +328,7 @@ def main(): formatter = TimeSeriesInferenceFormatter(preprocessor) # Example 1: Single-point inference (will fail, showing why we need the formatter) - example_single_point_inference_failure(preprocessor, formatter) + example_single_point_inference_failure(formatter) # Example 2: Inference with historical context example_with_historical_context(preprocessor, formatter, train_data) diff --git a/examples/time_series_inference_simple.py b/examples/time_series_inference_simple.py index 77b279d..dfe114a 100644 --- a/examples/time_series_inference_simple.py +++ b/examples/time_series_inference_simple.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# -*- coding: utf-8 -*- """ Simplified example showing how to use the TimeSeriesInferenceFormatter to prepare data for time series inference. @@ -41,12 +40,12 @@ def generate_sample_data(num_stores=3, days_per_store=30, add_noise=True): date = base_date + timedelta(days=day) # Calculate sales based on pattern - if store_id < 2: - # Linear pattern - sales = base_sales + (day * growth) - else: - # Sinusoidal pattern - sales = base_sales + 50 * np.sin(day * 0.2) + # Linear pattern for the first stores, sinusoidal for the rest. + sales = ( + base_sales + (day * growth) + if store_id < 2 + else base_sales + 50 * np.sin(day * 0.2) + ) # Add noise if requested if add_noise: @@ -57,7 +56,7 @@ def generate_sample_data(num_stores=3, days_per_store=30, add_noise=True): "date": date.strftime("%Y-%m-%d"), "store_id": f"Store_{store_id}", "sales": sales, - } + }, ) return pd.DataFrame(all_data) @@ -85,7 +84,7 @@ def create_preprocessor(train_data): } # Create a preprocessor - preprocessor = PreprocessingModel( + return PreprocessingModel( path_data=train_data, features_specs=features_specs, ) @@ -93,10 +92,8 @@ def create_preprocessor(train_data): # We don't need to build the preprocessor for this simplified example # We just need the features_specs and validation methods - return preprocessor - -def example_single_point_inference_failure(preprocessor, formatter): +def example_single_point_inference_failure(formatter): """Example showing how single-point inference fails with time series features.""" print("\n=== Single-Point Inference with Time Series Features ===") @@ -145,7 +142,7 @@ def example_with_historical_context(formatter, train_data): # In a real example, we would now call preprocessor.predict(formatted_data) # but for this simplified example, we'll just show that the data is ready for prediction print( - f"Data is ready for prediction! Last data point date: {formatted_data['date'][-1]}" + f"Data is ready for prediction! Last data point date: {formatted_data['date'][-1]}", ) @@ -213,7 +210,7 @@ def main(): formatter = TimeSeriesInferenceFormatter(preprocessor) # Example 1: Single-point inference (will fail, showing why we need the formatter) - example_single_point_inference_failure(preprocessor, formatter) + example_single_point_inference_failure(formatter) # Example 2: Inference with historical context example_with_historical_context(formatter, train_data) diff --git a/kdp/auto_config.py b/kdp/auto_config.py index 3b70f22..cef897e 100644 --- a/kdp/auto_config.py +++ b/kdp/auto_config.py @@ -1,10 +1,9 @@ -""" -Automatic model configuration module that provides a simple interface for +"""Automatic model configuration module that provides a simple interface for analyzing datasets and generating optimal preprocessing configurations. """ from pathlib import Path -from typing import Dict, Any, Optional, Union +from typing import Any from loguru import logger @@ -13,15 +12,14 @@ def auto_configure( - data_path: Union[str, Path], - features_specs: Optional[Dict[str, Any]] = None, + data_path: str | Path, + features_specs: dict[str, Any] | None = None, batch_size: int = 50_000, save_stats: bool = True, - stats_path: Optional[Union[str, Path]] = None, + stats_path: str | Path | None = None, overwrite_stats: bool = False, -) -> Dict[str, Any]: - """ - Automatically analyze a dataset and generate optimal preprocessing configurations. +) -> dict[str, Any]: + """Automatically analyze a dataset and generate optimal preprocessing configurations. This is a high-level function that handles all the complexity of analyzing your dataset and recommending the best preprocessing strategies. It will: @@ -53,10 +51,7 @@ def auto_configure( """ # Convert paths to Path objects data_path = Path(data_path) - if stats_path is None: - stats_path = Path("features_stats.json") - else: - stats_path = Path(stats_path) + stats_path = Path("features_stats.json") if stats_path is None else Path(stats_path) # Initialize statistics calculator stats_calculator = DatasetStatistics( diff --git a/kdp/dynamic_pipeline.py b/kdp/dynamic_pipeline.py index 13fc882..bdc0be5 100644 --- a/kdp/dynamic_pipeline.py +++ b/kdp/dynamic_pipeline.py @@ -2,83 +2,124 @@ class DynamicPreprocessingPipeline: - """ - Dynamically initializes and manages a sequence of Keras preprocessing layers, with selective retention of outputs - based on dependencies among layers, and supports streaming data through the pipeline. + """Chains Keras preprocessing layers over a dictionary of features. + + Every layer is addressed by its ``name``. When the pipeline runs, a layer + reads its input from the entry that shares its name if that entry exists, + and otherwise from the output of the layer that precedes it. The result is + always written back under the layer's own name, so intermediate results + stay available to later layers and to the caller. + + That single rule covers both common shapes: + + * supplying one entry per layer runs the layers independently over their + own inputs; + * supplying only the first layer's entry chains the layers, each one + consuming what the previous layer produced. + + Example: + ```python + pipeline = DynamicPreprocessingPipeline( + [ScalingLayer(name="scaling"), LogLayer(name="log")] + ) + # "log" is absent from the data, so it consumes the scaling output. + out = pipeline.transform({"scaling": tf.constant([[1.0], [2.0]])}) + out["scaling"], out["log"] + ``` """ - def __init__(self, layers): - """ - Initializes the pipeline with a list of preprocessing layers. + def __init__(self, layers: list) -> None: + """Initializes the pipeline with a list of preprocessing layers. Args: - layers (list): A list of TensorFlow preprocessing layers. + layers (list): A list of TensorFlow preprocessing layers. Each layer + must have a unique ``name``, which doubles as the key it reads + from and writes to. + + Raises: + ValueError: If two layers share the same name. """ self.layers = layers self.dependency_map = self._analyze_dependencies() - def _analyze_dependencies(self): - """ - Analyzes and determines the dependencies of each layer on the outputs of previous layers. + def _analyze_dependencies(self) -> dict: + """Determines which key each layer consumes. Returns: - dict: A dictionary mapping each layer's name to the set of layer outputs it depends on. + dict: A mapping of each layer's name to the name of the entry it + reads when the pipeline runs. A layer reads its own key when the + data provides it, otherwise it falls back to the previous + layer's output. The first layer always reads its own key. + + Raises: + ValueError: If two layers share the same name. """ dependencies = {} - all_outputs = set() - for i, layer in enumerate(self.layers): - # If the layer has an input_spec (which is common in Keras layers) we inspect it. - if hasattr(layer, "input_spec") and layer.input_spec is not None: - # Use a safe getter so that if an element does not have a 'name' attribute, we get None. - # Then filter out the Nones. - required_inputs = set( - [ - name - for name in tf.nest.flatten( - tf.nest.map_structure( - lambda x: getattr(x, "name", None), layer.input_spec - ) - ) - if name is not None - ] + previous_name = None + for layer in self.layers: + if layer.name in dependencies: + raise ValueError( + f"Duplicate layer name {layer.name!r} in pipeline. Each layer " + "needs a unique name because names are used as data keys.", ) - else: - # Otherwise, assume that the layer depends on all outputs seen so far. - required_inputs = all_outputs - dependencies[layer.name] = required_inputs - all_outputs.update(required_inputs) - all_outputs.add(layer.name) + # The key is only known at run time (it depends on what the data + # actually contains), so record both candidates in priority order. + dependencies[layer.name] = ( + (layer.name,) if previous_name is None else (layer.name, previous_name) + ) + previous_name = layer.name return dependencies - def process(self, dataset): + def transform(self, features: dict) -> dict: + """Applies every layer to the feature dictionary. + + Args: + features (dict): Mapping of feature names to tensors. It is not + modified; a new dictionary is returned. + + Returns: + dict: The input entries plus one entry per layer, holding that + layer's output. + + Raises: + KeyError: If a layer has no entry to read from, i.e. neither its own + key nor the previous layer's output is available. """ - Processes the dataset through the pipeline using tf.data API. + current_data = dict(features) + for layer in self.layers: + candidates = self.dependency_map[layer.name] + source = next((key for key in candidates if key in current_data), None) + if source is None: + raise KeyError( + f"Layer {layer.name!r} has no input: none of {list(candidates)} " + f"is present in the data (available keys: " + f"{sorted(current_data)}).", + ) + current_data[layer.name] = layer(current_data[source]) + return current_data + + def initialize_and_transform(self, features: dict) -> dict: + """Applies every layer to the feature dictionary. + + Kept as an alias of :meth:`transform` for backwards compatibility. Args: - dataset (tf.data.Dataset): The dataset where each element is a dictionary of features. + features (dict): Mapping of feature names to tensors. Returns: - tf.data.Dataset: The processed dataset with outputs of each layer stored by key. + dict: The input entries plus one entry per layer. """ + return self.transform(features) - def _apply_transformations(features): - current_data = features - for i, layer in enumerate(self.layers): - # Get the required input keys for the current layer. - required_keys = self.dependency_map[layer.name] - # Prepare the input by selecting the keys if they exist in the current data. - current_input = { - k: current_data[k] for k in required_keys if k in current_data - } - # Process each required input through the layer. - # Here we assume the layer accepts one tensor per key. - transformed_output = { - layer.name: layer(current_input[k]) - for k in required_keys - if k in current_input - } - # Merge transformed output into the working data dictionary. - current_data.update(transformed_output) - return current_data - - return dataset.map(_apply_transformations) + def process(self, dataset: tf.data.Dataset) -> tf.data.Dataset: + """Processes the dataset through the pipeline using the tf.data API. + + Args: + dataset (tf.data.Dataset): A dataset whose elements are dictionaries + of features. + + Returns: + tf.data.Dataset: The processed dataset, with each layer's output + stored under the layer's name. + """ + return dataset.map(self.transform) diff --git a/kdp/features.py b/kdp/features.py index d3a1b0b..2e4eb1a 100644 --- a/kdp/features.py +++ b/kdp/features.py @@ -1,6 +1,7 @@ from enum import Enum, auto from typing import Any +import keras import tensorflow as tf from loguru import logger @@ -165,9 +166,21 @@ def __init__( self.embedding_dim = embedding_dim self.num_bins = num_bins - def get_embedding_layer(self, input_shape: tuple) -> tf.keras.layers.Layer: - """Creates and returns an NumericalEmbedding layer configured for this feature.""" - # TODO: check why to use input_shape ? + def get_embedding_layer( + self, + input_shape: tuple | None = None, # noqa: ARG002 - kept for API compatibility + ) -> keras.layers.Layer: + """Creates and returns a NumericalEmbedding layer configured for this feature. + + Args: + input_shape: Unused. `NumericalEmbedding` derives the feature count + in its own `build`, so nothing here depends on the shape. The + parameter is kept, and optional, so existing callers that pass + it keep working. + + Returns: + A `NumericalEmbedding` layer built from this feature's settings. + """ from kdp.layers.numerical_embedding_layer import NumericalEmbedding return NumericalEmbedding( @@ -223,7 +236,10 @@ class TextFeature(Feature): """TextFeature with dynamic kwargs passing.""" def __init__( - self, name: str, feature_type: FeatureType = FeatureType.TEXT, **kwargs + self, + name: str, + feature_type: FeatureType = FeatureType.TEXT, + **kwargs, ) -> None: """Initializes a TextFeature instance. @@ -241,7 +257,10 @@ class DateFeature(Feature): """TextFeature with dynamic kwargs passing.""" def __init__( - self, name: str, feature_type: FeatureType = FeatureType.DATE, **kwargs + self, + name: str, + feature_type: FeatureType = FeatureType.DATE, + **kwargs, ) -> None: """Initializes a DateFeature instance. @@ -389,12 +408,49 @@ def __init__( "is_target": self.is_target, "exclude_from_input": self.exclude_from_input, "input_type": self.input_type, - } + }, ) - def build_layers(self): + def _resolve_drop_na( + self, + config: dict, + config_name: str, + row_preserving: bool, + ) -> bool: + """Decide whether a transform may drop its warm-up rows. + + Args: + config: The transform's configuration dictionary. + config_name: Name of the configuration, used in the warning. + row_preserving: Whether the caller needs the row count preserved. + + Returns: + The drop_na value the layer should be built with. + """ + drop_na = config.get("drop_na", True) + if row_preserving and drop_na: + if "drop_na" in config: + logger.warning( + f"{config_name} for feature '{self.name}' requested " + "drop_na=True, but a preprocessing model must keep one output " + "row per input row or the features cannot be concatenated. " + "Building with drop_na=False; the warm-up rows are padded " + "instead, and can be discarded downstream.", + ) + return False + return drop_na + + def build_layers(self, row_preserving: bool = True) -> list: """Build the appropriate layers for this time series feature based on configuration. + Args: + row_preserving: When True (the default) the layers keep every input + row, padding the warm-up positions instead of dropping them. + A preprocessing model lays features out side by side, so a layer + that removes its feature's leading rows leaves that column + shorter than every other one and the concatenation fails. Pass + False only when driving the returned layers directly. + Returns: list: List of TensorFlow layers for time series preprocessing """ @@ -411,7 +467,11 @@ def build_layers(self): # Add lag layer if configured if self.lag_config and "lags" in self.lag_config: lags = self.lag_config.get("lags", [1]) - drop_na = self.lag_config.get("drop_na", True) + drop_na = self._resolve_drop_na( + self.lag_config, + "lag_config", + row_preserving, + ) keep_original = self.lag_config.get("keep_original", True) fill_value = self.lag_config.get("fill_value", 0.0) @@ -422,7 +482,7 @@ def build_layers(self): keep_original=keep_original, fill_value=fill_value, name=f"{self.name}_lag", - ) + ), ) # Add rolling stats layer if configured @@ -430,7 +490,11 @@ def build_layers(self): window_size = self.rolling_stats_config.get("window_size") statistics = self.rolling_stats_config.get("statistics") window_stride = self.rolling_stats_config.get("window_stride", 1) - drop_na = self.rolling_stats_config.get("drop_na", True) + drop_na = self._resolve_drop_na( + self.rolling_stats_config, + "rolling_stats_config", + row_preserving, + ) keep_original = self.rolling_stats_config.get("keep_original", True) pad_value = self.rolling_stats_config.get("pad_value", 0.0) @@ -443,13 +507,17 @@ def build_layers(self): keep_original=keep_original, pad_value=pad_value, name=f"{self.name}_rolling_stats", - ) + ), ) # Add differencing layer if configured if self.differencing_config and "order" in self.differencing_config: order = self.differencing_config.get("order", 1) - drop_na = self.differencing_config.get("drop_na", True) + drop_na = self._resolve_drop_na( + self.differencing_config, + "differencing_config", + row_preserving, + ) keep_original = self.differencing_config.get("keep_original", True) fill_value = self.differencing_config.get("fill_value", 0.0) @@ -460,13 +528,17 @@ def build_layers(self): keep_original=keep_original, fill_value=fill_value, name=f"{self.name}_differencing", - ) + ), ) # Add moving average layer if configured if self.moving_average_config and "periods" in self.moving_average_config: periods = self.moving_average_config.get("periods", [7]) - drop_na = self.moving_average_config.get("drop_na", True) + drop_na = self._resolve_drop_na( + self.moving_average_config, + "moving_average_config", + row_preserving, + ) keep_original = self.moving_average_config.get("keep_original", True) pad_value = self.moving_average_config.get("pad_value", 0.0) @@ -477,7 +549,7 @@ def build_layers(self): keep_original=keep_original, pad_value=pad_value, name=f"{self.name}_moving_average", - ) + ), ) # Add wavelet transform layer if configured @@ -486,7 +558,11 @@ def build_layers(self): window_sizes = self.wavelet_transform_config.get("window_sizes", None) keep_levels = self.wavelet_transform_config.get("keep_levels", "all") flatten_output = self.wavelet_transform_config.get("flatten_output", True) - drop_na = self.wavelet_transform_config.get("drop_na", True) + drop_na = self._resolve_drop_na( + self.wavelet_transform_config, + "wavelet_transform_config", + row_preserving, + ) layers.append( WaveletTransformLayer( @@ -496,17 +572,22 @@ def build_layers(self): flatten_output=flatten_output, drop_na=drop_na, name=f"{self.name}_wavelet", - ) + ), ) # Add TSFresh feature layer if configured if self.tsfresh_feature_config: features = self.tsfresh_feature_config.get( - "features", ["mean", "std", "min", "max", "median"] + "features", + ["mean", "std", "min", "max", "median"], ) window_size = self.tsfresh_feature_config.get("window_size", None) stride = self.tsfresh_feature_config.get("stride", 1) - drop_na = self.tsfresh_feature_config.get("drop_na", True) + drop_na = self._resolve_drop_na( + self.tsfresh_feature_config, + "tsfresh_feature_config", + row_preserving, + ) normalize = self.tsfresh_feature_config.get("normalize", False) layers.append( @@ -517,13 +598,14 @@ def build_layers(self): drop_na=drop_na, normalize=normalize, name=f"{self.name}_tsfresh", - ) + ), ) # Add calendar feature layer if configured if self.calendar_feature_config: features = self.calendar_feature_config.get( - "features", ["month", "day", "day_of_week", "is_weekend"] + "features", + ["month", "day", "day_of_week", "is_weekend"], ) cyclic_encoding = self.calendar_feature_config.get("cyclic_encoding", True) input_format = self.calendar_feature_config.get("input_format", "%Y-%m-%d") @@ -536,157 +618,94 @@ def build_layers(self): input_format=input_format, normalize=normalize, name=f"{self.name}_calendar", - ) + ), ) return layers - def get_output_dim(self): + def get_output_dim(self) -> int: """Calculate the output dimension of this feature after all transformations. + The layers built by :meth:`build_layers` are applied in sequence, and each + one that keeps its originals passes them through alongside its new + columns. The widths therefore compose multiplicatively, not additively: + two lags on a single column give 3 columns, and differencing that while + keeping the originals gives 6, not 4. + Returns: - int: The output dimension + int: The number of columns the layer stack produces. """ - # Handle special cases for combined configurations to match test expectations - - # All configs case (test_output_dim test) - if ( - self.lag_config - and "lags" in self.lag_config - and self.rolling_stats_config - and "statistics" in self.rolling_stats_config - and self.differencing_config - and "order" in self.differencing_config - and self.moving_average_config - and "periods" in self.moving_average_config - ): - lags = self.lag_config.get("lags", [1]) - stats = self.rolling_stats_config.get("statistics", []) - order = self.differencing_config.get("order", 1) - periods = self.moving_average_config.get("periods", []) - - # Original + lags + stats + diff + MA - return 1 + len(lags) + len(stats) + order + len(periods) - - # Lag + differencing case (test_output_dim_parameterized_6) - if ( - self.lag_config - and "lags" in self.lag_config - and self.differencing_config - and "order" in self.differencing_config - ): - lags = self.lag_config.get("lags", [1]) - order = self.differencing_config.get("order", 1) - # Special case that matches the test: lag with 2 indices (original + 2 lags) + diff order 1 = 5 - if len(lags) == 2 and order == 1: - return 5 - - # Standard calculation logic dim = 1 - # Add dimensions for lag features if self.lag_config and "lags" in self.lag_config: lags = self.lag_config.get("lags", [1]) keep_original = self.lag_config.get("keep_original", True) + dim *= len(lags) + 1 if keep_original else len(lags) - if keep_original: - dim = 1 + len(lags) - else: - dim = len(lags) - - # Add dimensions for rolling statistics if self.rolling_stats_config and "statistics" in self.rolling_stats_config: statistics = self.rolling_stats_config.get("statistics", []) keep_original = self.rolling_stats_config.get("keep_original", True) + dim *= len(statistics) + 1 if keep_original else len(statistics) - if ( - keep_original and dim == 1 - ): # Only apply if we're starting from the original - dim += len(statistics) - else: - # Apply per value (original + lags) - dim = dim + len(statistics) - - # Add dimensions for differencing if self.differencing_config and "order" in self.differencing_config: - order = self.differencing_config.get("order", 1) keep_original = self.differencing_config.get("keep_original", True) - + # Successive orders collapse into a single differenced block, so + # keeping the originals doubles the width whatever the order is. if keep_original: - dim += order - else: - dim = order + dim *= 2 - # Add dimensions for moving averages if self.moving_average_config and "periods" in self.moving_average_config: periods = self.moving_average_config.get("periods", [7]) keep_original = self.moving_average_config.get("keep_original", True) + dim *= len(periods) + 1 if keep_original else len(periods) - if keep_original: - dim += len(periods) - else: - dim = len(periods) - - # Add dimensions for wavelet transform + # The remaining transforms append their columns to whatever came before. if self.wavelet_transform_config: levels = self.wavelet_transform_config.get("levels", 3) keep_levels = self.wavelet_transform_config.get("keep_levels", "all") flatten_output = self.wavelet_transform_config.get("flatten_output", True) - if flatten_output: - # If all levels, we have coefficients for each level plus the original - if keep_levels == "all": - wavelet_dims = levels - else: - # Count the specific levels to keep - if isinstance(keep_levels, list): - wavelet_dims = len(keep_levels) - else: - wavelet_dims = 1 # Default to 1 if not properly specified - - dim += wavelet_dims + if not flatten_output: + wavelet_dims = 1 + elif keep_levels == "all": + wavelet_dims = levels + elif isinstance(keep_levels, list): + wavelet_dims = len(keep_levels) else: - # If not flattened, output keeps original dimensions - # but we just treat it as one feature for dimensionality estimation - dim += 1 + wavelet_dims = 1 + dim += wavelet_dims - # Add dimensions for TSFresh features if self.tsfresh_feature_config: features = self.tsfresh_feature_config.get( - "features", ["mean", "std", "min", "max", "median"] + "features", + ["mean", "std", "min", "max", "median"], ) - # Each feature type adds one dimension dim += len(features) - # Add dimensions for calendar features if self.calendar_feature_config: features = self.calendar_feature_config.get( - "features", ["month", "day", "day_of_week", "is_weekend"] + "features", + ["month", "day", "day_of_week", "is_weekend"], ) cyclic_encoding = self.calendar_feature_config.get("cyclic_encoding", True) - - # For cyclic features (month, day, day_of_week), we use sin/cos encoding which doubles dimensions - cyclic_features = [ + # Cyclic components are encoded as a sin/cos pair. + cyclic_features = { "month", "day", "day_of_week", "quarter", "hour", "minute", - ] - + } if cyclic_encoding: for feature in features: - if feature in cyclic_features: - dim += 2 # sin and cos components - else: - dim += 1 # binary or scalar features + dim += 2 if feature in cyclic_features else 1 else: - dim += len(features) # one-hot or scalar for each feature + dim += len(features) return dim - def to_dict(self): + def to_dict(self) -> dict: """Convert the feature configuration to a dictionary. Returns: @@ -711,7 +730,7 @@ def to_dict(self): } @classmethod - def from_dict(cls, feature_dict): + def from_dict(cls, feature_dict) -> "TimeSeriesFeature": """Create a TimeSeriesFeature from a dictionary representation. Args: @@ -743,3 +762,134 @@ def from_dict(cls, feature_dict): # Create and return the feature return cls(**constructor_args) + + +class FeatureSpaceConverter: + def __init__(self) -> None: + """Initialize a feature space converter.""" + self.features_space = {} + self.numeric_features = [] + self.categorical_features = [] + self.text_features = [] + self.date_features = [] + self.passthrough_features = [] + self.time_series_features = [] # Add time series features list + + def _init_features_specs( + self, + features_specs: dict[str, FeatureType | str], + ) -> dict[str, Feature]: + """Format the features space into a dictionary. + + Args: + features_specs: A dictionary with the features and their types, + where types can be specified as either FeatureType enums, + class instances (NumericalFeature, CategoricalFeature, TextFeature, DateFeature), + or strings. + + Returns: + A dictionary with feature names as keys and Feature objects as values. + """ + for name, spec in features_specs.items(): + # Direct instance check for standard pipelines + if isinstance( + spec, + NumericalFeature + | CategoricalFeature + | TextFeature + | DateFeature + | PassthroughFeature + | TimeSeriesFeature, # Add TimeSeriesFeature to direct instance check + ): + feature_instance = spec + else: + # handling custom features pipelines + if isinstance(spec, Feature): + feature_type = spec.feature_type + else: + # Convert string to FeatureType if necessary + feature_type = ( + FeatureType[spec.upper()] if isinstance(spec, str) else spec + ) + + # Creating feature objects based on type + if feature_type in { + FeatureType.FLOAT, + FeatureType.FLOAT_NORMALIZED, + FeatureType.FLOAT_RESCALED, + FeatureType.FLOAT_DISCRETIZED, + }: + # Get preferred_distribution from kwargs if provided + preferred_distribution = ( + spec.kwargs.get("preferred_distribution") + if isinstance(spec, Feature) + else None + ) + feature_instance = NumericalFeature( + name=name, + feature_type=feature_type, + preferred_distribution=preferred_distribution, + ) + elif feature_type in { + FeatureType.INTEGER_CATEGORICAL, + FeatureType.STRING_CATEGORICAL, + }: + feature_instance = CategoricalFeature( + name=name, + feature_type=feature_type, + ) + elif feature_type == FeatureType.TEXT: + feature_instance = TextFeature(name=name, feature_type=feature_type) + elif feature_type == FeatureType.DATE: + feature_instance = DateFeature(name=name, feature_type=feature_type) + elif feature_type == FeatureType.TIME_SERIES: + # Create TimeSeriesFeature instance + feature_instance = TimeSeriesFeature( + name=name, + feature_type=feature_type, + ) + elif feature_type == FeatureType.PASSTHROUGH: + # Get dtype from kwargs if provided + dtype = ( + spec.kwargs.get("dtype", tf.float32) + if isinstance(spec, Feature) + else tf.float32 + ) + feature_instance = PassthroughFeature( + name=name, + feature_type=feature_type, + dtype=dtype, + ) + else: + raise ValueError( + f"Unsupported feature type for feature '{name}': {spec}", + ) + + # Adding custom pipelines + if isinstance(spec, Feature): + logger.info( + f"Adding custom preprocessors to the object: {spec.preprocessors}", + ) + feature_instance.preprocessors = spec.preprocessors + feature_instance.kwargs = spec.kwargs + + # Categorize feature based on its class + if isinstance(feature_instance, NumericalFeature): + self.numeric_features.append(name) + elif isinstance(feature_instance, CategoricalFeature): + self.categorical_features.append(name) + elif isinstance(feature_instance, TextFeature): + self.text_features.append(name) + elif isinstance(feature_instance, DateFeature): + self.date_features.append(name) + elif isinstance(feature_instance, TimeSeriesFeature): + # Add to time series features + self.time_series_features.append(name) + elif isinstance(feature_instance, PassthroughFeature): + # Add to passthrough features + self.passthrough_features.append(name) + + # Adding formatted spec to the features_space dictionary + self.features_space[name] = feature_instance + + return self.features_space diff --git a/kdp/inference/base.py b/kdp/inference/base.py index 1e7681d..cd535d3 100644 --- a/kdp/inference/base.py +++ b/kdp/inference/base.py @@ -1,7 +1,23 @@ import pandas as pd import numpy as np import tensorflow as tf -from typing import Dict, Union + + +def _is_missing(value) -> bool: + """Report whether a single value stands for "no data". + + Args: + value: A scalar taken from a feature column. + + Returns: + True for None, NaN and NaT. + """ + if value is None: + return True + try: + return bool(pd.isna(value)) + except (TypeError, ValueError): + return False class InferenceFormatter: @@ -23,8 +39,10 @@ def __init__(self, preprocessor): self.preprocessor = preprocessor def prepare_inference_data( - self, data: Union[Dict, pd.DataFrame], to_tensors: bool = False - ) -> Union[Dict, Dict[str, tf.Tensor]]: + self, + data: dict | pd.DataFrame, + to_tensors: bool = False, + ) -> dict | dict[str, tf.Tensor]: """Prepare data for inference based on preprocessor requirements. Args: @@ -43,7 +61,7 @@ def prepare_inference_data( return inference_data - def _convert_to_dict(self, data: Union[Dict, pd.DataFrame]) -> Dict: + def _convert_to_dict(self, data: dict | pd.DataFrame) -> dict: """Convert data to dictionary format required by the preprocessor. Args: @@ -61,13 +79,13 @@ def _convert_to_dict(self, data: Union[Dict, pd.DataFrame]) -> Dict: elif isinstance(data, dict): # Ensure all values are lists/arrays for key, value in data.items(): - if not isinstance(value, (list, np.ndarray)): + if not isinstance(value, list | np.ndarray): data[key] = [value] # Convert single values to lists return data else: raise ValueError(f"Unsupported data type: {type(data)}") - def _convert_to_tensors(self, data: Dict) -> Dict[str, tf.Tensor]: + def _convert_to_tensors(self, data: dict) -> dict[str, tf.Tensor]: """Convert dictionary data to TensorFlow tensors. Args: @@ -78,18 +96,30 @@ def _convert_to_tensors(self, data: Dict) -> Dict[str, tf.Tensor]: """ tf_data = {} for key, value in data.items(): - # Infer the type from the values - if ( - len(value) > 0 - and isinstance(value[0], (int, float, np.number, type(None))) - or any( - isinstance(v, (int, float, np.number)) or pd.isna(v) for v in value + values = list(value) + # Decide the column's type from the values that are actually there. + # The previous check treated any column containing a missing value + # as numeric, so a categorical column with one gap was handed to + # `tf.constant(..., tf.float32)` and raised "mixed types". + present = [item for item in values if not _is_missing(item)] + is_numeric = bool(present) and all( + isinstance(item, int | float | np.number) and not isinstance(item, bool) + for item in present + ) + + if is_numeric: + tf_data[key] = tf.constant( + [ + float("nan") if _is_missing(item) else float(item) + for item in values + ], + dtype=tf.float32, ) - ): - # Numerical features as float32 - tf_data[key] = tf.constant(value, dtype=tf.float32) else: - # Everything else as string - tf_data[key] = tf.constant(value) + # Missing categories become the empty string, which every + # vocabulary layer maps to its out-of-vocabulary slot. + tf_data[key] = tf.constant( + ["" if _is_missing(item) else str(item) for item in values], + ) return tf_data diff --git a/kdp/layers/__init__.py b/kdp/layers/__init__.py new file mode 100644 index 0000000..fe6d95b --- /dev/null +++ b/kdp/layers/__init__.py @@ -0,0 +1,73 @@ +"""Custom Keras layers used by KDP preprocessing pipelines. + +Importing this package registers every KDP layer with the Keras +serialization registry, which is what makes ``keras.models.load_model`` +able to rebuild a saved preprocessing model without an explicit +``custom_objects`` mapping. +""" + +from kdp.layers.cast_to_float import CastToFloat32Layer +from kdp.layers.date_encoding_layer import DateEncodingLayer +from kdp.layers.date_parsing_layer import DateParsingLayer +from kdp.layers.distribution_aware_encoder_layer import ( + DistributionAwareEncoder, + DistributionType, +) +from kdp.layers.distribution_transform_layer import DistributionTransformLayer +from kdp.layers.gated_linear_unit_layer import GatedLinearUnit +from kdp.layers.gated_residual_network_layer import GatedResidualNetwork +from kdp.layers.global_numerical_embedding_layer import GlobalNumericalEmbedding +from kdp.layers.multi_resolution_tabular_attention_layer import ( + MultiResolutionTabularAttention, +) +from kdp.layers.numerical_embedding_layer import NumericalEmbedding +from kdp.layers.preserve_dtype import PreserveDtypeLayer +from kdp.layers.season_layer import SeasonLayer +from kdp.layers.tabular_attention_layer import TabularAttention +from kdp.layers.text_preprocessing_layer import TextPreprocessingLayer +from kdp.layers.time_series import ( + AutoLagSelectionLayer, + CalendarFeatureLayer, + DifferencingLayer, + FFTFeatureLayer, + LagFeatureLayer, + MissingValueHandlerLayer, + MovingAverageLayer, + RollingStatsLayer, + SeasonalDecompositionLayer, + TSFreshFeatureLayer, + WaveletTransformLayer, +) +from kdp.layers.transformer_block_layer import TransformerBlock +from kdp.layers.variable_selection_layer import VariableSelection + +__all__ = [ + "AutoLagSelectionLayer", + "CalendarFeatureLayer", + "CastToFloat32Layer", + "DateEncodingLayer", + "DateParsingLayer", + "DifferencingLayer", + "DistributionAwareEncoder", + "DistributionTransformLayer", + "DistributionType", + "FFTFeatureLayer", + "GatedLinearUnit", + "GatedResidualNetwork", + "GlobalNumericalEmbedding", + "LagFeatureLayer", + "MissingValueHandlerLayer", + "MovingAverageLayer", + "MultiResolutionTabularAttention", + "NumericalEmbedding", + "PreserveDtypeLayer", + "RollingStatsLayer", + "SeasonLayer", + "SeasonalDecompositionLayer", + "TSFreshFeatureLayer", + "TabularAttention", + "TextPreprocessingLayer", + "TransformerBlock", + "VariableSelection", + "WaveletTransformLayer", +] diff --git a/kdp/layers/cast_to_float.py b/kdp/layers/cast_to_float.py index f50a0b1..df58c6d 100644 --- a/kdp/layers/cast_to_float.py +++ b/kdp/layers/cast_to_float.py @@ -1,8 +1,8 @@ import tensorflow as tf -from tensorflow import keras +import keras -@tf.keras.utils.register_keras_serializable(package="kdp.layers") +@keras.saving.register_keras_serializable(package="kdp.layers") class CastToFloat32Layer(keras.layers.Layer): """Custom Keras layer that casts input tensors to float32. @@ -14,7 +14,7 @@ def __init__(self, **kwargs): """Initialize the layer.""" super().__init__(**kwargs) - def call(self, inputs, **kwargs): + def call(self, inputs, **kwargs) -> tf.Tensor: """Cast the input tensor to float32. Args: @@ -26,17 +26,16 @@ def call(self, inputs, **kwargs): """ return tf.cast(inputs, tf.float32) - def get_config(self): + def get_config(self) -> dict: """Return the config dictionary for serialization. Returns: A dictionary with the layer configuration """ - config = super().get_config() - return config + return super().get_config() @classmethod - def from_config(cls, config): + def from_config(cls, config) -> "CastToFloat32Layer": """Create a new instance from the serialized configuration. Args: diff --git a/kdp/layers/date_encoding_layer.py b/kdp/layers/date_encoding_layer.py index b265e88..78dedc6 100644 --- a/kdp/layers/date_encoding_layer.py +++ b/kdp/layers/date_encoding_layer.py @@ -1,8 +1,10 @@ import math +import keras import tensorflow as tf -class DateEncodingLayer(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class DateEncodingLayer(keras.layers.Layer): def __init__(self, **kwargs): """Initializing DateEncodingLayer.""" super().__init__(**kwargs) @@ -15,7 +17,9 @@ def normalize_year(self, year: tf.Tensor) -> tf.Tensor: @tf.function def cyclic_encoding( - self, value: tf.Tensor, period: float + self, + value: tf.Tensor, + period: float, ) -> tuple[tf.Tensor, tf.Tensor]: """Encode a value as a cyclical feature using sine and cosine transformations. @@ -67,13 +71,15 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor: year_sin, year_cos = self.cyclic_encoding(year_float, period=1.0) month_sin, month_cos = self.cyclic_encoding(month_float, period=12.0) day_of_month_sin, day_of_month_cos = self.cyclic_encoding( - day_of_month_float, period=31.0 + day_of_month_float, + period=31.0, ) day_of_week_sin, day_of_week_cos = self.cyclic_encoding( - day_of_week_float, period=7.0 + day_of_week_float, + period=7.0, ) - encoded = tf.stack( + return tf.stack( [ year_sin, year_cos, @@ -87,8 +93,6 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor: axis=-1, ) - return encoded - def compute_output_shape(self, input_shape: tf.TensorShape) -> tf.TensorShape: """Compute the output shape after cyclic encoding. diff --git a/kdp/layers/date_parsing_layer.py b/kdp/layers/date_parsing_layer.py index d0fd2cd..1683399 100644 --- a/kdp/layers/date_parsing_layer.py +++ b/kdp/layers/date_parsing_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -class DateParsingLayer(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class DateParsingLayer(keras.layers.Layer): def __init__(self, date_format: str = "YYYY-MM-DD", **kwargs) -> None: """Initializing DateParsingLayer. @@ -91,15 +93,21 @@ def parse_date(date_str: str) -> tf.Tensor: day_of_month + ((13 * (m + 1)) // 5) + k + (k // 4) + (j // 4) - (2 * j) ) % 7 day_of_week = tf.where( - h == 0, 6, h - 1 + h == 0, + 6, + h - 1, ) # Adjust to 0-6 range where 0 is Sunday return tf.stack([year, month, day_of_month, day_of_week]) - parsed_dates = tf.map_fn( - parse_date, tf.squeeze(inputs), fn_output_signature=tf.int32 + # `tf.reshape(..., [-1])` rather than `tf.squeeze`: squeezing a + # single-row batch of shape (1, 1) collapses it to a scalar, which + # `tf.map_fn` cannot map over -- so predicting on one row raised. + return tf.map_fn( + parse_date, + tf.reshape(inputs, [-1]), + fn_output_signature=tf.int32, ) - return parsed_dates def compute_output_shape(self, input_shape: int) -> int: """Getting output shape.""" diff --git a/kdp/layers/distribution_aware_encoder_layer.py b/kdp/layers/distribution_aware_encoder_layer.py index 79aaaf6..aa54231 100644 --- a/kdp/layers/distribution_aware_encoder_layer.py +++ b/kdp/layers/distribution_aware_encoder_layer.py @@ -1,5 +1,4 @@ -""" -Distribution-aware encoder layer for TensorFlow models. +"""Distribution-aware encoder layer for TensorFlow models. This implementation automatically detects various data distributions and applies appropriate transformations for better model performance. It is built on Keras @@ -11,23 +10,27 @@ Example usage: ```python - import tensorflow as tf + import keras + from kdp.layers import DistributionAwareEncoder # Creating a model with automatic distribution detection - inputs = tf.keras.Input(shape=(10,)) + inputs = keras.Input(shape=(10,)) encoded = DistributionAwareEncoder(embedding_dim=16)(inputs) - outputs = tf.keras.layers.Dense(1)(encoded) - model = tf.keras.Model(inputs, outputs) + outputs = keras.layers.Dense(1)(encoded) + model = keras.Model(inputs, outputs) # Save and load model with custom objects model.save("my_model.keras") custom_objects = DistributionAwareEncoder.get_custom_objects() - loaded_model = tf.keras.models.load_model("my_model", custom_objects=custom_objects) + loaded_model = keras.saving.load_model("my_model", custom_objects=custom_objects) ``` """ from enum import Enum +from typing import Any + +import keras import tensorflow as tf from loguru import logger import inspect @@ -40,7 +43,7 @@ from ..distribution_transform_layer import DistributionTransformLayer except ImportError: raise ImportError( - "Could not import DistributionTransformLayer. Please ensure it's in the correct path." + "Could not import DistributionTransformLayer. Please ensure it's in the correct path.", ) @@ -65,7 +68,8 @@ class DistributionType(str, Enum): ORDINAL = "ordinal" # For ordered categorical data -class DistributionAwareEncoder(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class DistributionAwareEncoder(keras.layers.Layer): """An advanced layer that adapts its encoding based on the input distribution. This layer automatically detects and handles various distribution types using @@ -148,6 +152,7 @@ def __init__( mixture_components: Number of mixture components. Not used in current implementation. prefered_distribution: Legacy way to specify distribution_type. If provided, auto_detect will be set to False and this value will be used as distribution_type. + **kwargs: Additional arguments forwarded to `keras.layers.Layer`. Note on output dimensions: - If detect_periodicity=True and periodic features are detected/forced: @@ -157,9 +162,7 @@ def __init__( - If add_distribution_embedding=True: output_dim += 8 (distribution embedding dimension) """ - # Call parent's __init__ with proper parameters - # Fix: Use the full class name instead of just super() - tf.keras.layers.Layer.__init__(self, name=name, trainable=trainable, **kwargs) + super().__init__(name=name, trainable=trainable, **kwargs) # Initialize class attributes to avoid attribute errors self._added_periodic_features = False @@ -172,7 +175,7 @@ def __init__( self.distribution_type = prefered_distribution self.auto_detect = False logger.info( - f"Using specified distribution type: {prefered_distribution} (legacy parameter)" + f"Using specified distribution type: {prefered_distribution} (legacy parameter)", ) else: self.distribution_type = distribution_type @@ -201,14 +204,14 @@ def __init__( and self.distribution_type != "unknown" ): logger.warning( - f"Unknown distribution type: {self.distribution_type}. Defaulting to 'normal'" + f"Unknown distribution type: {self.distribution_type}. Defaulting to 'normal'", ) self.distribution_type = "normal" # Initialize layers self._init_layers() - def _init_layers(self): + def _init_layers(self) -> None: """Initialize internal layers.""" # Check accepted parameters for DistributionTransformLayer transform_kwargs = {} @@ -218,23 +221,29 @@ def _init_layers(self): transform_signature = inspect.signature(DistributionTransformLayer.__init__) if "epsilon" in transform_signature.parameters: transform_kwargs["epsilon"] = self.epsilon - except Exception: - pass # If we can't inspect, just use default params + except (TypeError, ValueError) as exc: + logger.debug( + f"Could not inspect DistributionTransformLayer signature ({exc}); " + "falling back to its default parameters.", + ) # The DistributionTransformLayer handles most of the distribution transformations self.distribution_transform = DistributionTransformLayer( - **transform_kwargs, name=f"{self.name}_transform" + **transform_kwargs, + name=f"{self.name}_transform", ) # Projection layer for embedding dimension if specified if self.embedding_dim is not None: - self.projection = tf.keras.layers.Dense( - self.embedding_dim, activation="relu", name=f"{self.name}_projection" + self.projection = keras.layers.Dense( + self.embedding_dim, + activation="relu", + name=f"{self.name}_projection", ) else: self.projection = None - def build(self, input_shape): + def build(self, input_shape) -> None: """Build the layer. Args: @@ -284,14 +293,14 @@ def build(self, input_shape): # Build the projection layer if needed if self.projection is not None: transformed_shape = self.distribution_transform.compute_output_shape( - input_shape + input_shape, ) self.projection.build(transformed_shape) # Fix the super().build call - tf.keras.layers.Layer.build(self, input_shape) + super().build(input_shape) - def _calculate_statistics(self, x): + def _calculate_statistics(self, x) -> dict: """Calculate statistics for distribution detection. Args: @@ -325,7 +334,8 @@ def _calculate_statistics(self, x): # Check for discreteness unique_values, _ = tf.unique(tf.reshape(x, [-1])) unique_ratio = tf.cast(tf.size(unique_values), tf.float32) / tf.cast( - tf.size(x), tf.float32 + tf.size(x), + tf.float32, ) return { @@ -340,11 +350,12 @@ def _calculate_statistics(self, x): "unique_ratio": unique_ratio, "has_negative": tf.reduce_any(x < 0), "is_bounded_01": tf.logical_and( - tf.reduce_all(x >= 0), tf.reduce_all(x <= 1) + tf.reduce_all(x >= 0), + tf.reduce_all(x <= 1), ), } - def _detect_distribution(self, x): + def _detect_distribution(self, x) -> tf.Tensor: """Detect the distribution type of the data. Args: @@ -380,33 +391,44 @@ def _detect_distribution(self, x): # Sparse score - Use tf.cond dist_scores["sparse"] = tf.cond( - zero_ratio > 0.5, lambda: tf.constant(100.0), lambda: tf.constant(0.0) + zero_ratio > 0.5, + lambda: tf.constant(100.0), + lambda: tf.constant(0.0), ) # Periodic score - Use tf.cond for tensor values dist_scores["periodic"] = tf.cond( - is_periodic, lambda: tf.constant(90.0), lambda: tf.constant(0.0) + is_periodic, + lambda: tf.constant(90.0), + lambda: tf.constant(0.0), ) # Discrete score - Use tf.cond dist_scores["discrete"] = tf.cond( - unique_ratio < 0.1, lambda: tf.constant(80.0), lambda: tf.constant(0.0) + unique_ratio < 0.1, + lambda: tf.constant(80.0), + lambda: tf.constant(0.0), ) # Beta score bounded_and_skewed = tf.logical_and(is_bounded_01, skewness > 0.5) dist_scores["beta"] = tf.cond( - bounded_and_skewed, lambda: tf.constant(75.0), lambda: tf.constant(0.0) + bounded_and_skewed, + lambda: tf.constant(75.0), + lambda: tf.constant(0.0), ) # Uniform score dist_scores["uniform"] = tf.cond( - is_bounded_01, lambda: tf.constant(70.0), lambda: tf.constant(0.0) + is_bounded_01, + lambda: tf.constant(70.0), + lambda: tf.constant(0.0), ) # Log-normal score positive_and_very_skewed = tf.logical_and( - tf.logical_not(has_negative), skewness > 2.0 + tf.logical_not(has_negative), + skewness > 2.0, ) dist_scores["log_normal"] = tf.cond( positive_and_very_skewed, @@ -416,39 +438,53 @@ def _detect_distribution(self, x): # Exponential score positive_and_skewed = tf.logical_and( - tf.logical_not(has_negative), skewness > 1.0 + tf.logical_not(has_negative), + skewness > 1.0, ) dist_scores["exponential"] = tf.cond( - positive_and_skewed, lambda: tf.constant(60.0), lambda: tf.constant(0.0) + positive_and_skewed, + lambda: tf.constant(60.0), + lambda: tf.constant(0.0), ) # Gamma score positive_and_mild_skew = tf.logical_and( - tf.logical_not(has_negative), skewness > 0.5 + tf.logical_not(has_negative), + skewness > 0.5, ) dist_scores["gamma"] = tf.cond( - positive_and_mild_skew, lambda: tf.constant(55.0), lambda: tf.constant(0.0) + positive_and_mild_skew, + lambda: tf.constant(55.0), + lambda: tf.constant(0.0), ) # Cauchy score dist_scores["cauchy"] = tf.cond( - kurtosis > 10.0, lambda: tf.constant(50.0), lambda: tf.constant(0.0) + kurtosis > 10.0, + lambda: tf.constant(50.0), + lambda: tf.constant(0.0), ) # Heavy-tailed score dist_scores["heavy_tailed"] = tf.cond( - kurtosis > 4.0, lambda: tf.constant(45.0), lambda: tf.constant(0.0) + kurtosis > 4.0, + lambda: tf.constant(45.0), + lambda: tf.constant(0.0), ) # Multimodal score - Use tf.cond for tensor values dist_scores["multimodal"] = tf.cond( - is_multimodal, lambda: tf.constant(40.0), lambda: tf.constant(0.0) + is_multimodal, + lambda: tf.constant(40.0), + lambda: tf.constant(0.0), ) # Zero-inflated score moderate_zeros = tf.logical_and(zero_ratio > 0.3, zero_ratio <= 0.5) dist_scores["zero_inflated"] = tf.cond( - moderate_zeros, lambda: tf.constant(35.0), lambda: tf.constant(0.0) + moderate_zeros, + lambda: tf.constant(35.0), + lambda: tf.constant(0.0), ) # Normal score - default option if nothing else fits well @@ -457,7 +493,7 @@ def _detect_distribution(self, x): normal_like, lambda: tf.constant(30.0), lambda: tf.constant( - 20.0 + 20.0, ), # Give some baseline score to normal distribution ) @@ -493,12 +529,12 @@ def _detect_distribution(self, x): logger.debug( f"Detected distribution: {selected_dist} with " f"skewness={skewness.numpy():.2f}, kurtosis={kurtosis.numpy():.2f}, " - f"zero_ratio={zero_ratio.numpy():.2f}, unique_ratio={unique_ratio.numpy():.2f}" + f"zero_ratio={zero_ratio.numpy():.2f}, unique_ratio={unique_ratio.numpy():.2f}", ) return dist_idx - def _check_periodicity(self, x, max_lag=20, threshold=0.2): + def _check_periodicity(self, x, max_lag=20, threshold=0.2) -> tf.Tensor: """Check if the input data has periodicity by analyzing autocorrelation. This method detects periodic patterns in the data by: @@ -532,7 +568,7 @@ def _check_periodicity(self, x, max_lag=20, threshold=0.2): lambda: self._compute_periodicity(x_flat, n, max_lag, threshold), ) - def _compute_periodicity(self, x_flat, n, max_lag, threshold): + def _compute_periodicity(self, x_flat, n, max_lag, threshold) -> tf.Tensor: """Helper function to compute periodicity for graph mode compatibility.""" # Center the data x_centered = x_flat - tf.reduce_mean(x_flat) @@ -545,11 +581,22 @@ def _compute_periodicity(self, x_flat, n, max_lag, threshold): tf.less(variance, self.epsilon), lambda: tf.constant(False), lambda: self._compute_autocorrelation( - x_centered, n, max_lag, variance, threshold + x_centered, + n, + max_lag, + variance, + threshold, ), ) - def _compute_autocorrelation(self, x_centered, n, max_lag, variance, threshold): + def _compute_autocorrelation( + self, + x_centered, + n, + max_lag, + variance, + threshold, + ) -> tf.Tensor: """Helper function to compute autocorrelation.""" # Use TensorArray to collect autocorrelation values acf_array = tf.TensorArray(tf.float32, size=max_lag) @@ -558,10 +605,10 @@ def _compute_autocorrelation(self, x_centered, n, max_lag, variance, threshold): i = tf.constant(0) # Use while_loop for graph compatibility - def loop_cond(i, *args): + def loop_cond(i, *args: Any) -> tf.Tensor: return tf.logical_and(tf.less(i, max_lag), tf.less(i, n // 3)) - def loop_body(i, acf_array): + def loop_body(i, acf_array) -> list: # Make sure we don't go out of bounds lag = i + 1 # Start from lag 1 @@ -592,7 +639,7 @@ def loop_body(i, acf_array): lambda: self._find_peaks(acf_tensor, threshold), ) - def _find_peaks(self, acf_tensor, threshold): + def _find_peaks(self, acf_tensor, threshold) -> tf.Tensor: """Find peaks in autocorrelation values.""" # Need at least 3 points for meaningful peaks return tf.cond( @@ -601,7 +648,7 @@ def _find_peaks(self, acf_tensor, threshold): lambda: self._check_significant_peaks(acf_tensor, threshold), ) - def _check_significant_peaks(self, acf_tensor, threshold): + def _check_significant_peaks(self, acf_tensor, threshold) -> tf.Tensor: """Check if there are significant peaks in the autocorrelation.""" # Create is_greater_left and is_greater_right tensors is_greater_left = tf.concat([[True], acf_tensor[1:] > acf_tensor[:-1]], axis=0) @@ -615,7 +662,7 @@ def _check_significant_peaks(self, acf_tensor, threshold): # Consider periodic if at least one significant peak return num_peaks >= 1 - def _check_multimodality(self, x, stats, num_bins=100): + def _check_multimodality(self, x, stats, num_bins=100) -> tf.Tensor: """Check if the data has a multimodal distribution by analyzing its histogram. This method detects multimodality by: @@ -639,9 +686,21 @@ def _check_multimodality(self, x, stats, num_bins=100): # Flatten the input x_flat = tf.reshape(x, [-1]) - # Create histogram + # `histogram_fixed_width` requires value_range[0] < value_range[1]. A + # constant batch -- every single-row prediction, and any feature with no + # spread -- makes min == max, and a fixed epsilon vanishes into float32 + # rounding at large magnitudes (50000 + 1e-6 == 50000). Widen the upper + # bound by an epsilon scaled to the magnitude so the range is always + # representable. + lower = tf.cast(stats["min"], tf.float32) + upper = tf.cast(stats["max"], tf.float32) + min_span = tf.maximum(tf.abs(lower) * 1e-5, self.epsilon) + upper = tf.maximum(upper, lower + min_span) + hist = tf.histogram_fixed_width( - x_flat, [stats["min"], stats["max"] + self.epsilon], nbins=num_bins + x_flat, + [lower, upper], + nbins=num_bins, ) # Smooth the histogram with a simple average filter @@ -665,10 +724,12 @@ def _check_multimodality(self, x, stats, num_bins=100): # Find peaks (local maxima) # A peak is where the value is greater than its neighbors is_greater_left = tf.concat( - [[True], smoothed_hist[1:] > smoothed_hist[:-1]], axis=0 + [[True], smoothed_hist[1:] > smoothed_hist[:-1]], + axis=0, ) is_greater_right = tf.concat( - [smoothed_hist[:-1] > smoothed_hist[1:], [True]], axis=0 + [smoothed_hist[:-1] > smoothed_hist[1:], [True]], + axis=0, ) is_peak = tf.logical_and(is_greater_left, is_greater_right) @@ -680,7 +741,7 @@ def _check_multimodality(self, x, stats, num_bins=100): # Consider multimodal if at least two significant peaks return num_peaks >= 2 - def _apply_distribution_specific_transform(self, x, dist_type): + def _apply_distribution_specific_transform(self, x, dist_type) -> tf.Tensor: """Apply distribution-specific transformations to the input tensor. This method applies different transformations depending on the detected or specified @@ -767,7 +828,7 @@ def _apply_distribution_specific_transform(self, x, dist_type): # For other distribution types, keep the transformation from DistributionTransformLayer return x - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Apply distribution-aware encoding to the inputs. This method: @@ -807,7 +868,7 @@ def call(self, inputs, training=None): is_training = training if training is not None else True # Use tf.cond for graph mode compatibility - def detect_distribution_fn(): + def detect_distribution_fn() -> tf.Tensor: # During training or first call, detect the distribution dist_idx = self._detect_distribution(x) @@ -819,14 +880,15 @@ def detect_distribution_fn(): # Ensure consistent type (int32) return tf.cast(dist_idx, tf.int32) - def use_stored_distribution_fn(): + def use_stored_distribution_fn() -> tf.Tensor: # During inference, use the stored distribution if hasattr(self, "detected_distribution"): dist_idx = self.detected_distribution[0] else: # Default to normal if no stored distribution dist_idx = tf.constant( - self._valid_distributions.index("normal"), dtype=tf.int32 + self._valid_distributions.index("normal"), + dtype=tf.int32, ) return tf.cast(dist_idx, tf.int32) # Ensure consistent type @@ -840,7 +902,9 @@ def use_stored_distribution_fn(): ) dist_idx = tf.cond( - should_detect, detect_distribution_fn, use_stored_distribution_fn + should_detect, + detect_distribution_fn, + use_stored_distribution_fn, ) # Mark as initialized @@ -855,7 +919,8 @@ def use_stored_distribution_fn(): dist_type = self.distribution_type # Get the index for consistency - ensure it's int32 dist_idx = tf.constant( - self._valid_distributions.index(dist_type), dtype=tf.int32 + self._valid_distributions.index(dist_type), + dtype=tf.int32, ) # Store the current distribution type for use in compute_output_shape @@ -892,11 +957,12 @@ def use_stored_distribution_fn(): # Concat along feature dimension transformed = tf.concat( - [transformed, sin_feature, cos_feature], axis=-1 + [transformed, sin_feature, cos_feature], + axis=-1, ) else: # In graph mode, use tf.cond with carefully managed shapes - def add_periodic_features(): + def add_periodic_features() -> tf.Tensor: mean = tf.reduce_mean(transformed) std = tf.math.reduce_std(transformed) + self.epsilon normalized = ( @@ -914,7 +980,7 @@ def add_periodic_features(): # Concat with the transformed features return tf.concat([transformed, sin_feature, cos_feature], axis=-1) - def keep_original(): + def keep_original() -> tf.Tensor: return transformed # Use tf.cond to conditionally add periodic features @@ -938,7 +1004,7 @@ def keep_original(): return transformed - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute the output shape of the layer based on input shape and layer configuration. Args: @@ -980,11 +1046,11 @@ def compute_output_shape(self, input_shape): return output_shape - def get_config(self): + def get_config(self) -> dict: """Get the layer configuration for serialization. This method enables serialization and deserialization of the layer via - `tf.keras.models.save_model()` and `tf.keras.models.load_model()`. + `keras.saving.save_model()` and `keras.saving.load_model()`. Returns: Configuration dictionary containing all parameters needed to reconstruct the layer. @@ -996,10 +1062,12 @@ def get_config(self): ```python model.save("my_model.keras") custom_objects = get_custom_objects() - loaded_model = tf.keras.models.load_model("my_model", custom_objects=custom_objects) + loaded_model = keras.saving.load_model( + "my_model", custom_objects=custom_objects + ) ``` """ - config = tf.keras.layers.Layer.get_config(self) + config = super().get_config() config.update( { "embedding_dim": self.embedding_dim, @@ -1015,7 +1083,7 @@ def get_config(self): "adaptive_binning": self.adaptive_binning, "mixture_components": self.mixture_components, "prefered_distribution": self.prefered_distribution, - } + }, ) return config @@ -1025,7 +1093,7 @@ def get_config(self): # add those dependencies separately. -def get_custom_objects(): +def get_custom_objects() -> dict: """Return custom objects dictionary needed for model loading. This function returns a dictionary of custom objects that need to be provided @@ -1044,11 +1112,10 @@ def get_custom_objects(): # Load the model custom_objects = get_custom_objects() - loaded_model = tf.keras.models.load_model("my_model", custom_objects=custom_objects) + loaded_model = keras.saving.load_model("my_model", custom_objects=custom_objects) ``` """ - custom_objects = { + return { "DistributionAwareEncoder": DistributionAwareEncoder, "DistributionTransformLayer": DistributionTransformLayer, } - return custom_objects diff --git a/kdp/layers/distribution_transform_layer.py b/kdp/layers/distribution_transform_layer.py index f74247d..4cfa76e 100644 --- a/kdp/layers/distribution_transform_layer.py +++ b/kdp/layers/distribution_transform_layer.py @@ -1,16 +1,17 @@ -""" -This module implements a DistributionTransformLayer that applies various transformations +"""This module implements a DistributionTransformLayer that applies various transformations to make data more normally distributed or to handle specific distribution types better. It's particularly useful for preprocessing data before anomaly detection or other statistical analyses. """ +import keras from typing import Any from loguru import logger from keras import ops, KerasTensor import tensorflow as tf -class DistributionTransformLayer(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class DistributionTransformLayer(keras.layers.Layer): """Layer for transforming data distributions to improve anomaly detection. This layer applies various transformations to make data more normally distributed @@ -64,7 +65,9 @@ class DistributionTransformLayer(tf.keras.layers.Layer): a = arcsinh_transform(x) # Apply min-max scaling to range [0, 1] - min_max = DistributionTransformLayer(transform_type="min-max", min_value=0.0, max_value=1.0) + min_max = DistributionTransformLayer( + transform_type="min-max", min_value=0.0, max_value=1.0 + ) b = min_max(x) # Use automatic transformation selection @@ -86,7 +89,12 @@ def __init__( **kwargs: Any, ) -> None: # Call parent's __init__ first - tf.keras.layers.Layer.__init__(self, name=name, **kwargs) + """Initialize the DistributionTransformLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ + super().__init__(name=name, **kwargs) # Set private attributes self._transform_type = transform_type @@ -141,7 +149,7 @@ def _validate_params(self) -> None: if self.transform_type not in self._valid_transforms: raise ValueError( f"transform_type must be one of {self._valid_transforms}, " - f"got {self.transform_type}" + f"got {self.transform_type}", ) if self.epsilon <= 0: @@ -150,7 +158,7 @@ def _validate_params(self) -> None: if self.min_value >= self.max_value: raise ValueError( f"min_value must be less than max_value, got min_value={self.min_value}, " - f"max_value={self.max_value}" + f"max_value={self.max_value}", ) if self.transform_type == "auto" and self.auto_candidates: @@ -161,7 +169,7 @@ def _validate_params(self) -> None: ]: raise ValueError( f"Invalid transformation candidate: {candidate}. " - f"Candidates must be valid transformations excluding 'auto' and 'none'." + f"Candidates must be valid transformations excluding 'auto' and 'none'.", ) def build(self, input_shape: tuple[int, ...]) -> None: @@ -192,50 +200,60 @@ def build(self, input_shape: tuple[int, ...]) -> None: logger.debug( f"DistributionTransformLayer built with transform_type={self.transform_type}, " - f"lambda_param={self.lambda_param}" + f"lambda_param={self.lambda_param}", ) - tf.keras.layers.Layer.build(self, input_shape) + super().build(input_shape) + + def _percentile(self, x_sorted: tf.Tensor, q: float) -> tf.Tensor: + """Compute a percentile of a pre-sorted tensor along its first axis. + + Uses linear interpolation between the two neighbouring ranks, matching + NumPy's default ``percentile`` behaviour. The computation stays free of + Python-level branching so that it also works in graph mode, where the + batch size is a tensor rather than an ``int``. + + Args: + x_sorted: Tensor already sorted along axis 0. + q: Percentile to compute, expressed as a fraction in [0, 1]. + + Returns: + Tensor of per-feature percentiles, with the batch axis removed. + """ + n = tf.shape(x_sorted)[0] + position = tf.cast(q, tf.float32) * tf.cast(n - 1, tf.float32) + lower_idx = tf.cast(tf.floor(position), tf.int32) + upper_idx = tf.minimum(lower_idx + 1, n - 1) + fraction = position - tf.floor(position) + + lower = tf.gather(x_sorted, lower_idx, axis=0) + upper = tf.gather(x_sorted, upper_idx, axis=0) + return lower + (upper - lower) * tf.cast(fraction, x_sorted.dtype) - def _compute_statistics( - self, x: KerasTensor + def _compute_robust_statistics( + self, + x: KerasTensor, ) -> tuple[KerasTensor, KerasTensor, KerasTensor, KerasTensor]: - """Compute statistics for the input tensor. + """Compute order statistics for the input tensor. Args: x: Input tensor Returns: - tuple of (min, max, median, interquartile_range) + tuple of (min, max, median, interquartile_range), each computed + per feature along the batch axis. """ # Compute min and max along each feature dimension x_min = ops.min(x, axis=0, keepdims=True) x_max = ops.max(x, axis=0, keepdims=True) - # For median and IQR, we need to sort the values - # This is an approximation since Keras doesn't have direct percentile functions - x_sorted = ops.sort(x, axis=0) - n = ops.shape(x)[0] - - # Compute median (50th percentile) - median_idx = n // 2 - if n % 2 == 0: - # Even number of elements, average the middle two - median = (x_sorted[median_idx - 1] + x_sorted[median_idx]) / 2.0 - else: - # Odd number of elements, take the middle one - median = x_sorted[median_idx] - - # Compute 25th and 75th percentiles for IQR - q1_idx = n // 4 - q3_idx = (3 * n) // 4 - q1 = x_sorted[q1_idx] - q3 = x_sorted[q3_idx] - - # Compute IQR - iqr = q3 - q1 + # Median and quartiles are read off the sorted values. + x_sorted = tf.sort(x, axis=0) + median = self._percentile(x_sorted, 0.5) + q1 = self._percentile(x_sorted, 0.25) + q3 = self._percentile(x_sorted, 0.75) # Add small epsilon to IQR to avoid division by zero - iqr = ops.maximum(iqr, self.epsilon) + iqr = ops.maximum(q3 - q1, self.epsilon) return x_min, x_max, median, iqr @@ -327,7 +345,7 @@ def _is_bounded_01(self, x: KerasTensor) -> bool: return min_gt_zero & max_lt_one # Using & operator for element-wise logical AND - def _compute_statistics(self, x): + def _compute_statistics(self, x) -> dict: """Compute basic statistics for a tensor in a graph-mode compatible way. Args: @@ -512,7 +530,7 @@ def _select_best_transformation(self, x: KerasTensor) -> tuple[str, float]: return transform_type, lambda_param - def _apply_transform(self, x): + def _apply_transform(self, x) -> tf.Tensor: """Apply the selected transformation to the input tensor in a graph-compatible way. Args: @@ -524,18 +542,18 @@ def _apply_transform(self, x): # Use tf.case to select the transformation type # Define functions for each transformation type - def apply_none(): + def apply_none() -> tf.Tensor: return x - def apply_log(): + def apply_log() -> tf.Tensor: # Add epsilon to avoid log(0) return tf.math.log(x + self.epsilon) - def apply_sqrt(): + def apply_sqrt() -> tf.Tensor: # Ensure non-negative values return tf.sqrt(tf.maximum(x, 0.0) + self.epsilon) - def apply_box_cox(): + def apply_box_cox() -> tf.Tensor: # Box-Cox transformation: (x^lambda - 1)/lambda if lambda != 0, log(x) if lambda == 0 # Ensure x is positive x_pos = tf.maximum(x, self.epsilon) @@ -547,7 +565,7 @@ def apply_box_cox(): / self.lambda_param, # Standard Box-Cox formula ) - def apply_yeo_johnson(): + def apply_yeo_johnson() -> tf.Tensor: # Yeo-Johnson works for both positive and negative values lambda_p = self.lambda_param @@ -574,15 +592,17 @@ def apply_yeo_johnson(): # Combine results return tf.where( - pos_mask, pos_values, tf.where(neg_mask, neg_values, tf.zeros_like(x)) + pos_mask, + pos_values, + tf.where(neg_mask, neg_values, tf.zeros_like(x)), ) - def apply_arcsinh(): + def apply_arcsinh() -> tf.Tensor: # Inverse hyperbolic sine transformation # Works well for both positive and negative values with heavy tails return tf.math.log(x + tf.sqrt(tf.square(x) + 1.0)) - def apply_cube_root(): + def apply_cube_root() -> tf.Tensor: # Cube root transformation # Works well for both positive and negative values pos_mask = x >= 0 @@ -596,16 +616,18 @@ def apply_cube_root(): # Combine results return tf.where( - pos_mask, pos_values, tf.where(neg_mask, neg_values, tf.zeros_like(x)) + pos_mask, + pos_values, + tf.where(neg_mask, neg_values, tf.zeros_like(x)), ) - def apply_logit(): + def apply_logit() -> tf.Tensor: # Logit transformation: log(x / (1 - x)) # Clip values to the valid range with a small epsilon x_clipped = tf.clip_by_value(x, self.epsilon, 1.0 - self.epsilon) return tf.math.log(x_clipped / (1.0 - x_clipped)) - def apply_min_max(): + def apply_min_max() -> tf.Tensor: # Min-Max scaling to [0, 1] if self.clip_values: # Use predefined min/max values @@ -622,43 +644,25 @@ def apply_min_max(): return (x - min_val) / denom - def apply_robust_scale(): - # Robust scaling using median and interquartile range - # For TensorFlow, approximate with percentiles - median = tf.reduce_mean(x) - q1 = tf.reduce_mean( - tf.where( - tf.argsort(x) < tf.reduce_mean(tf.shape(x)), x, tf.zeros_like(x) - ) - ) - q3 = tf.reduce_mean( - tf.where( - tf.argsort(x) > tf.reduce_mean(tf.shape(x)), x, tf.zeros_like(x) - ) - ) - iqr = q3 - q1 + def apply_robust_scale() -> tf.Tensor: + # Robust scaling: centre on the per-feature median and divide by the + # per-feature interquartile range, so outliers do not dominate. + _, _, median, iqr = self._compute_robust_statistics(x) - # Avoid division by zero + # Avoid division by zero for constant features. iqr = tf.where(tf.abs(iqr) < self.epsilon, tf.ones_like(iqr), iqr) return (x - median) / iqr - def apply_quantile(): - # Quantile transformation (approximation) - # Simple quantile normalization for now - original_shape = tf.shape(x) - flattened = tf.reshape(x, [-1]) - ranks = tf.argsort(tf.argsort(flattened)) - n = tf.shape(flattened)[0] + def apply_quantile() -> tf.Tensor: + # Rank-based quantile transformation, computed per feature so that + # features on different scales are not pooled into a single ranking. + # Ranks are mapped to [-1, 1] rather than through an exact normal + # inverse CDF: the order is preserved and the output stays bounded. + ranks = tf.argsort(tf.argsort(x, axis=0), axis=0) + n = tf.shape(x)[0] quantiles = (tf.cast(ranks, tf.float32) + 0.5) / tf.cast(n, tf.float32) - - # Apply simplified normal inverse CDF approximation - # Instead of trying to compute exact normal inverse CDF, apply a simpler transform - # that preserves the rank order but has more stable shape handling - scaled_quantiles = 2.0 * quantiles - 1.0 # Map [0,1] to [-1,1] - - # Reshape to original shape - return tf.reshape(scaled_quantiles, original_shape) + return tf.cast(2.0 * quantiles - 1.0, x.dtype) # Map [0,1] to [-1,1] # Create a dictionary of transform types to functions transform_functions = { @@ -696,15 +700,15 @@ def apply_quantile(): else: # This should never happen due to validation in __init__ raise ValueError( - f"Unknown transformation type: {self.transform_type}" + f"Unknown transformation type: {self.transform_type}", ) except Exception as e: # Add more context to the error raise type(e)( - f"Error in {self.transform_type} transformation: {str(e)}" + f"Error in {self.transform_type} transformation: {str(e)}", ) from e - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Apply the selected transformation to the inputs. Args: @@ -727,7 +731,8 @@ def call(self, inputs, training=None): # For graph mode compatibility, we need to handle the index computation differently # Map the transform types to indices using a series of tf.cond operations transform_idx = tf.constant( - 0, dtype=tf.int32 + 0, + dtype=tf.int32, ) # Default to first transform # For graph mode, we need to compare the best_transform with each valid transform @@ -736,7 +741,7 @@ def call(self, inputs, training=None): transform_idx = tf.cond( tf.equal(best_transform, transform_type), lambda i=i: tf.constant(i, dtype=tf.int32), - lambda: transform_idx, + lambda previous=transform_idx: previous, ) # Store the selected transformation @@ -778,7 +783,8 @@ def call(self, inputs, training=None): lambda_param = self._selected_lambda[0] # Map the index to the transform type using tf.gather transform_type = tf.gather( - tf.constant(self._valid_transforms), transform_idx + tf.constant(self._valid_transforms), + transform_idx, ) # Set the transformation type and lambda for this forward pass @@ -807,7 +813,7 @@ def get_config(self) -> dict[str, Any]: Returns: Configuration dictionary """ - config = tf.keras.layers.Layer.get_config(self) + config = super().get_config() config.update( { "transform_type": self.transform_type, @@ -817,11 +823,11 @@ def get_config(self) -> dict[str, Any]: "max_value": float(self.max_value), "clip_values": self.clip_values, "auto_candidates": self.auto_candidates, - } + }, ) return config - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute the output shape of the layer. Args: diff --git a/kdp/layers/gated_linear_unit_layer.py b/kdp/layers/gated_linear_unit_layer.py index 0f75db5..9bdddfa 100644 --- a/kdp/layers/gated_linear_unit_layer.py +++ b/kdp/layers/gated_linear_unit_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -class GatedLinearUnit(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class GatedLinearUnit(keras.layers.Layer): """GatedLinearUnit is a custom Keras layer that implements a gated linear unit. This layer applies a dense linear transformation to the input tensor and multiplies the result with the output @@ -24,8 +26,8 @@ def __init__(self, units: int, **kwargs: dict) -> None: """ super().__init__(**kwargs) self.units = units - self.linear = tf.keras.layers.Dense(units) - self.sigmoid = tf.keras.layers.Dense(units, activation="sigmoid") + self.linear = keras.layers.Dense(units) + self.sigmoid = keras.layers.Dense(units, activation="sigmoid") def call(self, inputs: tf.Tensor) -> tf.Tensor: """Forward pass of the layer. diff --git a/kdp/layers/gated_residual_network_layer.py b/kdp/layers/gated_residual_network_layer.py index f8018cd..1a1b67b 100644 --- a/kdp/layers/gated_residual_network_layer.py +++ b/kdp/layers/gated_residual_network_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf from kdp.layers.gated_linear_unit_layer import GatedLinearUnit -class GatedResidualNetwork(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class GatedResidualNetwork(keras.layers.Layer): """GatedResidualNetwork is a custom Keras layer that implements a gated residual network. This layer applies a series of transformations to the input tensor and combines it with the original input @@ -28,12 +30,12 @@ def __init__(self, units: int, dropout_rate: float = 0.2, **kwargs: dict) -> Non super().__init__(**kwargs) self.units = units self.dropout_rate = dropout_rate - self.elu_dense = tf.keras.layers.Dense(units, activation="elu") - self.linear_dense = tf.keras.layers.Dense(units) - self.dropout = tf.keras.layers.Dropout(dropout_rate) + self.elu_dense = keras.layers.Dense(units, activation="elu") + self.linear_dense = keras.layers.Dense(units) + self.dropout = keras.layers.Dropout(dropout_rate) self.gated_linear_unit = GatedLinearUnit(units=units) - self.layer_norm = tf.keras.layers.LayerNormalization() - self.project = tf.keras.layers.Dense(units) + self.layer_norm = keras.layers.LayerNormalization() + self.project = keras.layers.Dense(units) def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: """Forward pass of the layer. @@ -54,8 +56,7 @@ def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: if inputs.shape[-1] != self.units: inputs = self.project(inputs) x = inputs + self.gated_linear_unit(x) - x = self.layer_norm(x) - return x + return self.layer_norm(x) def get_config(self) -> dict: """Get layer configuration. diff --git a/kdp/layers/global_numerical_embedding_layer.py b/kdp/layers/global_numerical_embedding_layer.py index d9cec3a..ff3db75 100644 --- a/kdp/layers/global_numerical_embedding_layer.py +++ b/kdp/layers/global_numerical_embedding_layer.py @@ -1,12 +1,12 @@ +import keras import tensorflow as tf import numpy as np from kdp.layers.numerical_embedding_layer import NumericalEmbedding -@tf.keras.utils.register_keras_serializable(package="kdp.layers") -class GlobalNumericalEmbedding(tf.keras.layers.Layer): - """ - Global NumericalEmbedding processes concatenated numeric features. +@keras.saving.register_keras_serializable(package="kdp.layers") +class GlobalNumericalEmbedding(keras.layers.Layer): + """Global NumericalEmbedding processes concatenated numeric features. It applies an inner NumericalEmbedding over the flattened input and then performs global pooling (average or max) to produce a compact representation. """ @@ -42,19 +42,19 @@ def __init__( self.global_num_bins = global_num_bins # Ensure initializer parameters are Python scalars, lists, or numpy arrays. - if not isinstance(global_init_min, (list, tuple, np.ndarray)): + if not isinstance(global_init_min, list | tuple | np.ndarray): try: global_init_min = float(global_init_min) except Exception: raise ValueError( - "init_min must be a Python scalar, list, tuple or numpy array" + "init_min must be a Python scalar, list, tuple or numpy array", ) - if not isinstance(global_init_max, (list, tuple, np.ndarray)): + if not isinstance(global_init_max, list | tuple | np.ndarray): try: global_init_max = float(global_init_max) except Exception: raise ValueError( - "init_max must be a Python scalar, list, tuple or numpy array" + "init_max must be a Python scalar, list, tuple or numpy array", ) self.global_init_min = global_init_min self.global_init_max = global_init_max @@ -74,19 +74,41 @@ def __init__( name="global_numeric_emebedding", ) if self.global_pooling == "average": - self.global_pooling_layer = tf.keras.layers.GlobalAveragePooling1D( - name="global_avg_pool" + self.global_pooling_layer = keras.layers.GlobalAveragePooling1D( + name="global_avg_pool", ) elif self.global_pooling == "max": - self.global_pooling_layer = tf.keras.layers.GlobalMaxPooling1D( - name="global_max_pool" + self.global_pooling_layer = keras.layers.GlobalMaxPooling1D( + name="global_max_pool", ) else: raise ValueError(f"Unsupported pooling method: {self.global_pooling}") - def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: + def build(self, input_shape) -> None: + """Explicitly build the sub-layers so saved weights can be restored. + + Keras only restores the weights of sub-layers that already exist and are + built when a model is deserialized. Building them lazily inside ``call`` + leaves them unbuilt at load time, which makes ``load_model`` fail, so the + inner embedding and pooling layers are built here instead. + + Args: + input_shape: Shape of the inputs, ``(batch, ...)``. """ - Expects inputs with shape (batch, ...) and flattens them (except for the batch dim). + flat_dim = input_shape[-1] + if len(input_shape) > 2: + flat_dim = 1 + for dim in input_shape[1:]: + flat_dim *= dim + + self.inner_embedding.build((input_shape[0], flat_dim)) + self.global_pooling_layer.build( + (input_shape[0], flat_dim, self.global_embedding_dim), + ) + super().build(input_shape) + + def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: + """Expects inputs with shape (batch, ...) and flattens them (except for the batch dim). Then, the inner embedding produces a 3D output (batch, num_features, embedding_dim), which is finally pooled to yield (batch, embedding_dim). """ @@ -96,14 +118,26 @@ def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: # Pass through the inner advanced embedding. x_embedded = self.inner_embedding(inputs, training=training) # Global pooling over numeric features axis. - x_pooled = self.global_pooling_layer(x_embedded) - return x_pooled + return self.global_pooling_layer(x_embedded) - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: # Regardless of the input shape, the output shape is (batch_size, embedding_dim) + """Compute the output shape for a given input shape. + + Args: + input_shape: Shape of the input tensor. + + Returns: + The corresponding output shape. + """ return (input_shape[0], self.global_embedding_dim) - def get_config(self): + def get_config(self) -> dict: + """Return the configuration needed to re-create this layer. + + Returns: + The layer configuration. + """ config = super().get_config() config.update( { @@ -115,10 +149,18 @@ def get_config(self): "global_dropout_rate": self.global_dropout_rate, "global_use_batch_norm": self.global_use_batch_norm, "global_pooling": self.global_pooling, - } + }, ) return config @classmethod - def from_config(cls, config): + def from_config(cls, config) -> "GlobalNumericalEmbedding": + """Re-create the layer from its configuration. + + Args: + config: Configuration dictionary produced by `get_config`. + + Returns: + The reconstructed layer. + """ return cls(**config) diff --git a/kdp/layers/multi_resolution_tabular_attention_layer.py b/kdp/layers/multi_resolution_tabular_attention_layer.py index 404ab3a..01c86a0 100644 --- a/kdp/layers/multi_resolution_tabular_attention_layer.py +++ b/kdp/layers/multi_resolution_tabular_attention_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -class MultiResolutionTabularAttention(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class MultiResolutionTabularAttention(keras.layers.Layer): """Multi-resolution attention layer for tabular data. This layer implements separate attention mechanisms for numerical and categorical features, @@ -48,54 +50,54 @@ def __init__( self.dropout_rate = dropout_rate # Create projection layers during initialization - self.numerical_projection = tf.keras.layers.Dense(d_model) - self.categorical_projection = tf.keras.layers.Dense(embedding_dim) + self.numerical_projection = keras.layers.Dense(d_model) + self.categorical_projection = keras.layers.Dense(embedding_dim) # Numerical attention - self.numerical_attention = tf.keras.layers.MultiHeadAttention( + self.numerical_attention = keras.layers.MultiHeadAttention( num_heads=num_heads, key_dim=d_model // num_heads, dropout=dropout_rate, ) - self.numerical_ffn = tf.keras.Sequential( + self.numerical_ffn = keras.Sequential( [ - tf.keras.layers.Dense(d_model * 2, activation="relu"), - tf.keras.layers.Dense(d_model), + keras.layers.Dense(d_model * 2, activation="relu"), + keras.layers.Dense(d_model), ], ) - self.numerical_layernorm1 = tf.keras.layers.LayerNormalization() - self.numerical_layernorm2 = tf.keras.layers.LayerNormalization() - self.numerical_dropout1 = tf.keras.layers.Dropout(dropout_rate) - self.numerical_dropout2 = tf.keras.layers.Dropout(dropout_rate) + self.numerical_layernorm1 = keras.layers.LayerNormalization() + self.numerical_layernorm2 = keras.layers.LayerNormalization() + self.numerical_dropout1 = keras.layers.Dropout(dropout_rate) + self.numerical_dropout2 = keras.layers.Dropout(dropout_rate) # Categorical attention - self.categorical_attention = tf.keras.layers.MultiHeadAttention( + self.categorical_attention = keras.layers.MultiHeadAttention( num_heads=num_heads, key_dim=embedding_dim // num_heads, dropout=dropout_rate, ) - self.categorical_ffn = tf.keras.Sequential( + self.categorical_ffn = keras.Sequential( [ - tf.keras.layers.Dense(embedding_dim * 2, activation="relu"), - tf.keras.layers.Dense(embedding_dim), + keras.layers.Dense(embedding_dim * 2, activation="relu"), + keras.layers.Dense(embedding_dim), ], ) - self.categorical_layernorm1 = tf.keras.layers.LayerNormalization() - self.categorical_layernorm2 = tf.keras.layers.LayerNormalization() - self.categorical_dropout1 = tf.keras.layers.Dropout(dropout_rate) - self.categorical_dropout2 = tf.keras.layers.Dropout(dropout_rate) + self.categorical_layernorm1 = keras.layers.LayerNormalization() + self.categorical_layernorm2 = keras.layers.LayerNormalization() + self.categorical_dropout1 = keras.layers.Dropout(dropout_rate) + self.categorical_dropout2 = keras.layers.Dropout(dropout_rate) # Cross attention - self.cross_attention = tf.keras.layers.MultiHeadAttention( + self.cross_attention = keras.layers.MultiHeadAttention( num_heads=num_heads, key_dim=d_model // num_heads, dropout=dropout_rate, ) - self.cross_layernorm = tf.keras.layers.LayerNormalization() - self.cross_dropout = tf.keras.layers.Dropout(dropout_rate) + self.cross_layernorm = keras.layers.LayerNormalization() + self.cross_dropout = keras.layers.Dropout(dropout_rate) # Final projections - self.categorical_output_projection = tf.keras.layers.Dense(d_model) + self.categorical_output_projection = keras.layers.Dense(d_model) def call( self, diff --git a/kdp/layers/numerical_embedding_layer.py b/kdp/layers/numerical_embedding_layer.py index ccabcde..beaaa12 100644 --- a/kdp/layers/numerical_embedding_layer.py +++ b/kdp/layers/numerical_embedding_layer.py @@ -1,8 +1,9 @@ +import keras import tensorflow as tf -@tf.keras.utils.register_keras_serializable(package="kdp.layers") -class NumericalEmbedding(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class NumericalEmbedding(keras.layers.Layer): """Advanced numerical embedding layer for continuous features. This layer embeds each continuous numerical feature into a higher-dimensional space by @@ -65,42 +66,48 @@ def __init__( if self.num_bins is None: raise ValueError( - "num_bins must be provided to activate the discrete branch." + "num_bins must be provided to activate the discrete branch.", ) - def build(self, input_shape): + def build(self, input_shape) -> None: # input_shape: (batch, num_features) + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ self.num_features = input_shape[-1] # Continuous branch: process each feature independently using TimeDistributed MLP. - self.cont_mlp = tf.keras.Sequential( + self.cont_mlp = keras.Sequential( [ - tf.keras.layers.TimeDistributed( - tf.keras.layers.Dense(self.mlp_hidden_units, activation="relu") + keras.layers.TimeDistributed( + keras.layers.Dense(self.mlp_hidden_units, activation="relu"), ), - tf.keras.layers.TimeDistributed( - tf.keras.layers.Dense(self.embedding_dim) + keras.layers.TimeDistributed( + keras.layers.Dense(self.embedding_dim), ), ], name="cont_mlp", ) self.dropout = ( - tf.keras.layers.Dropout(self.dropout_rate) + keras.layers.Dropout(self.dropout_rate) if self.dropout_rate > 0 - else lambda x, training: x + else lambda x, **kwargs: x # no-op matching the Dropout call signature ) if self.use_batch_norm: - self.batch_norm = tf.keras.layers.TimeDistributed( - tf.keras.layers.BatchNormalization(), name="cont_batch_norm" + self.batch_norm = keras.layers.TimeDistributed( + keras.layers.BatchNormalization(), + name="cont_batch_norm", ) # Residual projection to match embedding_dim. - self.residual_proj = tf.keras.layers.TimeDistributed( - tf.keras.layers.Dense(self.embedding_dim, activation=None), + self.residual_proj = keras.layers.TimeDistributed( + keras.layers.Dense(self.embedding_dim, activation=None), name="residual_proj", ) # Discrete branch: Create one Embedding layer per feature. self.bin_embeddings = [] for i in range(self.num_features): - embed_layer = tf.keras.layers.Embedding( + embed_layer = keras.layers.Embedding( input_dim=self.num_bins, output_dim=self.embedding_dim, name=f"bin_embed_{i}", @@ -149,10 +156,33 @@ def build(self, input_shape): initializer="zeros", trainable=True, ) + + # Explicitly build the sub-layers. Keras restores weights only for + # sub-layers that are already built when a saved model is deserialized, + # so leaving these to build lazily inside `call` breaks `load_model` + # whenever this layer is nested inside another custom layer. + batch = input_shape[0] + branch_input_shape = (batch, self.num_features, 1) + self.cont_mlp.build(branch_input_shape) + self.residual_proj.build(branch_input_shape) + if self.use_batch_norm: + self.batch_norm.build((batch, self.num_features, self.embedding_dim)) + for embed_layer in self.bin_embeddings: + embed_layer.build((batch,)) + super().build(input_shape) def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: # Continuous branch. + """Apply the layer to its inputs. + + Args: + inputs: Input tensor to process. + training: Whether the layer is being called in training mode. + + Returns: + The transformed tensor. + """ inputs_expanded = tf.expand_dims(inputs, axis=-1) # (batch, num_features, 1) cont = self.cont_mlp(inputs_expanded) cont = self.dropout(cont, training=training) @@ -176,7 +206,7 @@ def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: for i in range(self.num_features): feat_bins = bin_indices[:, i] # (batch,) feat_embed = self.bin_embeddings[i]( - feat_bins + feat_bins, ) # i is a Python integer here. disc_embeddings.append(feat_embed) disc = tf.stack(disc_embeddings, axis=1) # (batch, num_features, embedding_dim) @@ -189,7 +219,12 @@ def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: return tf.squeeze(output, axis=1) # New shape: (batch, embedding_dim) return output - def get_config(self): + def get_config(self) -> dict: + """Return the configuration needed to re-create this layer. + + Returns: + The layer configuration. + """ config = super().get_config() config.update( { @@ -200,10 +235,18 @@ def get_config(self): "init_max": self.init_max, "dropout_rate": self.dropout_rate, "use_batch_norm": self.use_batch_norm, - } + }, ) return config @classmethod - def from_config(cls, config): + def from_config(cls, config) -> "NumericalEmbedding": + """Re-create the layer from its configuration. + + Args: + config: Configuration dictionary produced by `get_config`. + + Returns: + The reconstructed layer. + """ return cls(**config) diff --git a/kdp/layers/preserve_dtype.py b/kdp/layers/preserve_dtype.py index 517c72a..4c7e12f 100644 --- a/kdp/layers/preserve_dtype.py +++ b/kdp/layers/preserve_dtype.py @@ -1,8 +1,8 @@ import tensorflow as tf -from tensorflow import keras +import keras -@tf.keras.utils.register_keras_serializable(package="kdp.layers") +@keras.saving.register_keras_serializable(package="kdp.layers") class PreserveDtypeLayer(keras.layers.Layer): """Custom Keras layer that preserves the original dtype of input tensors. @@ -20,7 +20,7 @@ def __init__(self, target_dtype=None, **kwargs): super().__init__(**kwargs) self.target_dtype = target_dtype - def call(self, inputs, **kwargs): + def call(self, inputs, **kwargs) -> tf.Tensor: """Process the input tensor, optionally casting to target_dtype. Args: @@ -34,7 +34,7 @@ def call(self, inputs, **kwargs): return tf.cast(inputs, self.target_dtype) return inputs - def get_config(self): + def get_config(self) -> dict: """Return the config dictionary for serialization. Returns: @@ -45,7 +45,7 @@ def get_config(self): return config @classmethod - def from_config(cls, config): + def from_config(cls, config) -> "PreserveDtypeLayer": """Create a new instance from the serialized configuration. Args: diff --git a/kdp/layers/season_layer.py b/kdp/layers/season_layer.py index bee69ea..0068369 100644 --- a/kdp/layers/season_layer.py +++ b/kdp/layers/season_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -class SeasonLayer(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class SeasonLayer(keras.layers.Layer): """A Keras Layer that adds seasonal information to the input tensor based on the month. This layer determines the season for each month and encodes it as a one-hot vector. The seasons are Winter, diff --git a/kdp/layers/tabular_attention_layer.py b/kdp/layers/tabular_attention_layer.py index bf8051f..0155ce5 100644 --- a/kdp/layers/tabular_attention_layer.py +++ b/kdp/layers/tabular_attention_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -class TabularAttention(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class TabularAttention(keras.layers.Layer): """Custom layer to apply inter-feature and inter-sample attention for tabular data. This layer implements a dual attention mechanism: @@ -13,7 +15,11 @@ class TabularAttention(tf.keras.layers.Layer): """ def __init__( - self, num_heads: int, d_model: int, dropout_rate: float = 0.1, **kwargs + self, + num_heads: int, + d_model: int, + dropout_rate: float = 0.1, + **kwargs, ): """Initialize the TabularAttention layer. @@ -29,36 +35,36 @@ def __init__( self.dropout_rate = dropout_rate # Attention layers - self.feature_attention = tf.keras.layers.MultiHeadAttention( + self.feature_attention = keras.layers.MultiHeadAttention( num_heads=num_heads, key_dim=d_model, ) - self.sample_attention = tf.keras.layers.MultiHeadAttention( + self.sample_attention = keras.layers.MultiHeadAttention( num_heads=num_heads, key_dim=d_model, ) # Feed-forward network - self.ffn = tf.keras.Sequential( + self.ffn = keras.Sequential( [ - tf.keras.layers.Dense(d_model, activation="relu"), - tf.keras.layers.Dense(d_model), + keras.layers.Dense(d_model, activation="relu"), + keras.layers.Dense(d_model), ], ) # Normalization and dropout - self.layernorm1 = tf.keras.layers.LayerNormalization() - self.layernorm2 = tf.keras.layers.LayerNormalization() - self.dropout = tf.keras.layers.Dropout(dropout_rate) - self.feature_layernorm = tf.keras.layers.LayerNormalization() - self.feature_layernorm2 = tf.keras.layers.LayerNormalization() - self.feature_dropout = tf.keras.layers.Dropout(dropout_rate) - self.feature_dropout2 = tf.keras.layers.Dropout(dropout_rate) - self.sample_layernorm = tf.keras.layers.LayerNormalization() - self.sample_layernorm2 = tf.keras.layers.LayerNormalization() - self.sample_dropout = tf.keras.layers.Dropout(dropout_rate) - self.sample_dropout2 = tf.keras.layers.Dropout(dropout_rate) - self.output_projection = tf.keras.layers.Dense(d_model) + self.layernorm1 = keras.layers.LayerNormalization() + self.layernorm2 = keras.layers.LayerNormalization() + self.dropout = keras.layers.Dropout(dropout_rate) + self.feature_layernorm = keras.layers.LayerNormalization() + self.feature_layernorm2 = keras.layers.LayerNormalization() + self.feature_dropout = keras.layers.Dropout(dropout_rate) + self.feature_dropout2 = keras.layers.Dropout(dropout_rate) + self.sample_layernorm = keras.layers.LayerNormalization() + self.sample_layernorm2 = keras.layers.LayerNormalization() + self.sample_dropout = keras.layers.Dropout(dropout_rate) + self.sample_dropout2 = keras.layers.Dropout(dropout_rate) + self.output_projection = keras.layers.Dense(d_model) def build(self, input_shape: int) -> None: """Build the layer. @@ -67,7 +73,7 @@ def build(self, input_shape: int) -> None: input_shape: Shape tuple (tuple of integers) or list of shape tuples """ self.input_dim = input_shape[-1] - self.input_projection = tf.keras.layers.Dense(self.d_model) + self.input_projection = keras.layers.Dense(self.d_model) def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: """Forward pass for TabularAttention. @@ -84,7 +90,7 @@ def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: """ if len(inputs.shape) != 3: raise ValueError( - "Input tensor must be 3-dimensional (batch_size, num_samples, num_features)" + "Input tensor must be 3-dimensional (batch_size, num_samples, num_features)", ) # Project inputs to d_model dimension @@ -92,32 +98,34 @@ def call(self, inputs: tf.Tensor, training: bool = False) -> tf.Tensor: # Inter-feature attention: across columns (features) features = self.feature_attention( - projected, projected, projected, training=training + projected, + projected, + projected, + training=training, ) features = self.feature_layernorm( - projected + self.feature_dropout(features, training=training) + projected + self.feature_dropout(features, training=training), ) features_ffn = self.ffn(features) features = self.feature_layernorm2( - features + self.feature_dropout2(features_ffn, training=training) + features + self.feature_dropout2(features_ffn, training=training), ) # Inter-sample attention: across rows (samples) samples = tf.transpose( - features, perm=[0, 2, 1] + features, + perm=[0, 2, 1], ) # Transpose for sample attention samples = self.sample_attention(samples, samples, samples, training=training) samples = tf.transpose(samples, perm=[0, 2, 1]) # Transpose back samples = self.sample_layernorm( - features + self.sample_dropout(samples, training=training) + features + self.sample_dropout(samples, training=training), ) samples_ffn = self.ffn(samples) - outputs = self.sample_layernorm2( - samples + self.sample_dropout2(samples_ffn, training=training) + return self.sample_layernorm2( + samples + self.sample_dropout2(samples_ffn, training=training), ) - return outputs - def get_config(self) -> dict: """Returns the configuration of the layer. diff --git a/kdp/layers/text_preprocessing_layer.py b/kdp/layers/text_preprocessing_layer.py index eb92094..2657fc8 100644 --- a/kdp/layers/text_preprocessing_layer.py +++ b/kdp/layers/text_preprocessing_layer.py @@ -1,9 +1,11 @@ import re import string +import keras import tensorflow as tf -class TextPreprocessingLayer(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class TextPreprocessingLayer(keras.layers.Layer): def __init__(self, stop_words: list, **kwargs: dict) -> None: """Initializes a TextPreprocessingLayer. @@ -16,7 +18,7 @@ def __init__(self, stop_words: list, **kwargs: dict) -> None: # Define punctuation and stop words patterns as part of the configuration self.punctuation_pattern = re.escape(string.punctuation) self.stop_words_pattern = r"|".join( - [re.escape(word) for word in self.stop_words] + [re.escape(word) for word in self.stop_words], ) def call(self, x: tf.Tensor) -> tf.Tensor: @@ -32,8 +34,7 @@ def call(self, x: tf.Tensor) -> tf.Tensor: x = tf.strings.regex_replace(x, f"[{self.punctuation_pattern}]", " ") stop_words_regex = rf"\b({self.stop_words_pattern})\b\s?" x = tf.strings.regex_replace(x, stop_words_regex, " ") - x = tf.strings.regex_replace(x, r"\s+", " ") - return x + return tf.strings.regex_replace(x, r"\s+", " ") def get_config(self) -> dict: """Returns the configuration of the layer as a dictionary. diff --git a/kdp/layers/time_series/auto_lag_selection_layer.py b/kdp/layers/time_series/auto_lag_selection_layer.py index 3900d05..3e763ab 100644 --- a/kdp/layers/time_series/auto_lag_selection_layer.py +++ b/kdp/layers/time_series/auto_lag_selection_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer import numpy as np +@keras.saving.register_keras_serializable(package="kdp.layers") class AutoLagSelectionLayer(Layer): """Layer for automatically selecting optimal lag features based on autocorrelation analysis. @@ -33,6 +35,11 @@ def __init__( keep_original=True, **kwargs, ): + """Initialize the AutoLagSelectionLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.max_lag = max_lag self.n_lags = n_lags @@ -49,10 +56,15 @@ def __init__( # Initialize selected lags self.selected_lags = None - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Apply automatic lag selection. Args: @@ -93,29 +105,9 @@ def call(self, inputs, training=None): default_lags = tf.range(1, self.n_lags + 1) self.selected_lags = default_lags - # For test_drop_na, we need special handling if specific lags are set - # This is for compatibility with the test, which sets selected_lags directly - if ( - self.drop_na - and hasattr(self, "selected_lags") - and isinstance(self.selected_lags, tf.Tensor) - ): - if tf.reduce_max(self.selected_lags) > inputs.shape[0]: - # For test_drop_na, the expected behavior is that we should return - # a tensor with batch dimension = inputs.shape[0] - max_lag - # but if max_lag > inputs.shape[0], we need to handle this specially - expected_rows = inputs.shape[0] - tf.reduce_max(self.selected_lags) - if expected_rows < 0: - # In the test case, we need to return a tensor with the expected_rows - # even though it's negative (for the assertion to pass) - dummy_tensor = tf.zeros( - [expected_rows, inputs.shape[1], 4], dtype=tf.float32 - ) - return dummy_tensor - # Create lag features # Handle lag feature creation as a NumPy operation for more control - def create_lag_features(inputs_tensor, selected_lags_tensor): + def create_lag_features(inputs_tensor, selected_lags_tensor) -> np.ndarray: # Convert to NumPy inputs_np = inputs_tensor.numpy() selected_lags_np = selected_lags_tensor.numpy() @@ -145,7 +137,8 @@ def create_lag_features(inputs_tensor, selected_lags_tensor): ) else: result = np.zeros( - (batch_size, time_steps, n_output_features), dtype=inputs_np.dtype + (batch_size, time_steps, n_output_features), + dtype=inputs_np.dtype, ) # Feature index counter @@ -185,7 +178,8 @@ def create_lag_features(inputs_tensor, selected_lags_tensor): orig_idx = i + max_lag if orig_idx - lag >= 0: result[i, lag:, feature_idx] = inputs_np[ - orig_idx - lag, :-lag + orig_idx - lag, + :-lag, ] else: # Handle case where lag goes beyond input bounds @@ -208,18 +202,22 @@ def create_lag_features(inputs_tensor, selected_lags_tensor): # Shift each feature and place in output for f in range(n_features): for i in range( - min(batch_size - max_lag, result.shape[0]) + min(batch_size - max_lag, result.shape[0]), ): # Use data from (i + max_lag - lag) to create lag feature at position i orig_idx = i + max_lag if orig_idx - lag >= 0: result[i, lag:, feature_idx + f] = inputs_np[ - orig_idx - lag, :-lag, f + orig_idx - lag, + :-lag, + f, ] else: # Handle case where lag goes beyond input bounds result[ - i, lag:, feature_idx + f + i, + lag:, + feature_idx + f, ] = self.fill_value else: # Without drop_na, we pad the beginning with fill_value @@ -230,7 +228,9 @@ def create_lag_features(inputs_tensor, selected_lags_tensor): result[i, :lag, feature_idx + f] = self.fill_value # Rest are shifted values result[i, lag:, feature_idx + f] = inputs_np[ - i, :-lag, f + i, + :-lag, + f, ] feature_idx += n_features @@ -239,7 +239,9 @@ def create_lag_features(inputs_tensor, selected_lags_tensor): # Apply the function if self.selected_lags is not None: result = tf.py_function( - create_lag_features, [inputs, self.selected_lags], tf.float32 + create_lag_features, + [inputs, self.selected_lags], + tf.float32, ) # Set the shape @@ -254,11 +256,10 @@ def create_lag_features(inputs_tensor, selected_lags_tensor): if self.drop_na: max_lag = tf.reduce_max(self.selected_lags) - if inputs.shape[0] > max_lag: - batch_size = inputs.shape[0] - max_lag - else: - # Special case for test_drop_na - batch_size = inputs.shape[0] - max_lag # This can be negative + # Dropping the leading rows consumed by the largest lag. This can + # go negative when the batch is shorter than the lag, which + # set_shape then rejects -- the caller must supply enough rows. + batch_size = inputs.shape[0] - max_lag result.set_shape([batch_size, inputs.shape[1], n_output_features]) else: result.set_shape([inputs.shape[0], inputs.shape[1], n_output_features]) @@ -268,11 +269,11 @@ def create_lag_features(inputs_tensor, selected_lags_tensor): # Fallback case (shouldn't happen in normal execution) return inputs - def _compute_autocorrelation(self, series): + def _compute_autocorrelation(self, series) -> np.ndarray: """Compute autocorrelation for lags 1 to max_lag using numpy for more accuracy.""" # Convert to numpy for more control over computation - def compute_acf(batch_tensor): + def compute_acf(batch_tensor) -> np.ndarray: # Convert to numpy array batch_np = batch_tensor.numpy() result = np.zeros((batch_np.shape[0], self.max_lag + 1), dtype=np.float32) @@ -309,7 +310,7 @@ def compute_acf(batch_tensor): return acf - def _select_lags(self, acf): + def _select_lags(self, acf) -> tf.Tensor: """Select lags based on autocorrelation values.""" # Use batch mean autocorrelation for lag selection mean_acf = tf.reduce_mean(acf, axis=0) @@ -331,11 +332,9 @@ def _select_lags(self, acf): selected_lags = indices + 1 # Sort lags in ascending order for interpretability - selected_lags = tf.sort(selected_lags) - - return selected_lags + return tf.sort(selected_lags) - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute the output shape.""" output_shape = list(input_shape) @@ -350,10 +349,11 @@ def compute_output_shape(self, input_shape): else: # For 3D input, update the feature dimension feature_dim = output_shape[-1] - if self.keep_original: - feature_dim = feature_dim + (feature_dim * self.n_lags) - else: - feature_dim = feature_dim * self.n_lags + feature_dim = ( + feature_dim + feature_dim * self.n_lags + if self.keep_original + else feature_dim * self.n_lags + ) output_shape[-1] = feature_dim # Update batch dimension if dropping rows @@ -370,11 +370,12 @@ def compute_output_shape(self, input_shape): if output_shape[0] is not None: output_shape[0] = max( - 1, output_shape[0] - max_lag + 1, + output_shape[0] - max_lag, ) # Ensure batch size is at least 1 return tuple(output_shape) - def get_config(self): + def get_config(self) -> dict: """Return the configuration.""" config = { "max_lag": self.max_lag, diff --git a/kdp/layers/time_series/calendar_feature_layer.py b/kdp/layers/time_series/calendar_feature_layer.py index 5b5a493..2528b50 100644 --- a/kdp/layers/time_series/calendar_feature_layer.py +++ b/kdp/layers/time_series/calendar_feature_layer.py @@ -1,10 +1,12 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer import numpy as np from loguru import logger import pandas as pd +@keras.saving.register_keras_serializable(package="kdp.layers") class CalendarFeatureLayer(Layer): """Layer for generating calendar features from date or timestamp inputs. @@ -49,6 +51,11 @@ def __init__( onehot_categorical=False, **kwargs, ): + """Initialize the CalendarFeatureLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) # Default features if none provided @@ -107,10 +114,15 @@ def __init__( ): raise ValueError(f"Invalid feature: {feature}") - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Extract calendar features from date inputs. Args: @@ -122,7 +134,7 @@ def call(self, inputs, training=None): """ # Process date inputs using pandas for more flexibility - def extract_calendar_features(date_inputs): + def extract_calendar_features(date_inputs) -> np.ndarray: # Convert tensor to numpy if isinstance(date_inputs, tf.Tensor): date_inputs = date_inputs.numpy() @@ -139,17 +151,17 @@ def extract_calendar_features(date_inputs): [ s.decode("utf-8") if isinstance(s, bytes) else s for s in date_inputs - ] + ], ) # Also handle case where strings are repr'd as bytes elif isinstance(date_inputs[0], str) and date_inputs[0].startswith( - "b'" + "b'", ): date_inputs = np.array( [ s[2:-1] if s.startswith("b'") and s.endswith("'") else s for s in date_inputs - ] + ], ) except (IndexError, TypeError): pass # Handle empty arrays or arrays with mixed types @@ -171,7 +183,7 @@ def extract_calendar_features(date_inputs): # Last resort: try to clean the strings and convert cleaned_inputs = [] for d in date_inputs: - if isinstance(d, (bytes, str)): + if isinstance(d, bytes | str): # Clean up string representation of bytes if ( isinstance(d, str) @@ -285,9 +297,7 @@ def extract_calendar_features(date_inputs): df[feature] = np.sin(angle) # Convert to numpy array - features_array = df.values.astype(np.float32) - - return features_array + return df.to_numpy().astype(np.float32) # Apply the function result = tf.py_function(extract_calendar_features, [inputs], tf.float32) @@ -298,14 +308,14 @@ def extract_calendar_features(date_inputs): return result - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute the output shape of the layer.""" batch_size = input_shape[0] n_features = len(self.features) return (batch_size, n_features) - def get_config(self): + def get_config(self) -> dict: """Return the configuration of the layer.""" config = { "features": self.features, diff --git a/kdp/layers/time_series/differencing_layer.py b/kdp/layers/time_series/differencing_layer.py index 0ef9940..0ddfd1c 100644 --- a/kdp/layers/time_series/differencing_layer.py +++ b/kdp/layers/time_series/differencing_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer +@keras.saving.register_keras_serializable(package="kdp.layers") class DifferencingLayer(Layer): """Layer for computing differences of time series data. @@ -16,8 +18,18 @@ class DifferencingLayer(Layer): """ def __init__( - self, order=1, drop_na=True, fill_value=0.0, keep_original=False, **kwargs + self, + order=1, + drop_na=True, + fill_value=0.0, + keep_original=False, + **kwargs, ): + """Initialize the DifferencingLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.order = order self.drop_na = drop_na @@ -28,157 +40,91 @@ def __init__( if self.order <= 0: raise ValueError(f"Order must be positive. Got {order}") - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs): + def call(self, inputs) -> tf.Tensor: """Apply the differencing operation. Args: - inputs: Input tensor of shape (batch_size, ...) or (batch_size, time_steps) + inputs: Input tensor of shape (time_steps,) or (time_steps, features). Returns: - Tensor with original and/or differenced values depending on configuration + Tensor with original and/or differenced values depending on configuration. """ - # Get the input shape and determine if reshaping is needed - original_rank = tf.rank(inputs) - input_is_1d = original_rank == 1 - - # Create a copy of inputs for later use - inputs_orig = inputs - + inputs = tf.convert_to_tensor(inputs) + # Use the static rank: `tf.rank` returns a tensor, which cannot drive a + # Python `if` once the layer is traced into a graph. + input_is_1d = inputs.shape.rank == 1 if input_is_1d: - # Reshape to 2D for consistent processing inputs = tf.reshape(inputs, (-1, 1)) - # Test case for test_drop_na_false - if ( - input_is_1d - and self.order == 1 - and not self.drop_na - and self.fill_value == 0.0 - ): - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 5: - # For test data [1, 3, 5, 7, 9], with drop_na=False, create expected output - expected_output = tf.constant( - [ - [0.0], # fill_value for the first position - [2.0], # 3 - 1 - [2.0], # 5 - 3 - [2.0], # 7 - 5 - [2.0], # 9 - 7 - ], - dtype=tf.float32, - ) - return expected_output - - # Test case for test_fill_value - if ( - input_is_1d - and self.order == 1 - and not self.drop_na - and self.fill_value == -999.0 - ): - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 3: - # For test data [1, 2, 3], with fill_value=-999.0 - expected_output = tf.constant([-999.0, 1.0, 1.0], dtype=tf.float32) - return expected_output - - # Test case for first-order differencing - if input_is_1d and self.order == 1 and not self.keep_original: - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 5: - # For linear trend [1, 3, 5, 7, 9], expected differences are [2, 2, 2, 2] - # Need to match expected shape (4,) specified in test - expected_output = tf.constant([2.0, 2.0, 2.0, 2.0], dtype=tf.float32) - return expected_output - - # Test case for second-order differencing - if input_is_1d and self.order == 2 and not self.keep_original: - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 5: - # For quadratic trend, second-order diffs are [2, 2, 2] - # Need to match expected shape (3, 1) specified in test - expected_output = tf.ones([3, 1], dtype=tf.float32) * 2.0 - return expected_output - - # Compute differences of the specified order + # Compute differences of the specified order. diff = inputs for _ in range(self.order): - # Compute the difference diff_values = diff[1:] - diff[:-1] - - # Handle padding based on drop_na parameter - if not self.drop_na: - padding = tf.fill([1, tf.shape(diff_values)[1]], self.fill_value) - diff = tf.concat([padding, diff_values], axis=0) - else: - diff = diff_values - - # Initialize list to store results - result_tensors = [] - - # Keep the original values if specified - if self.keep_original: if self.drop_na: - # If dropping NAs, align with the differences - result_tensors.append(inputs[self.order :]) + diff = diff_values else: - result_tensors.append(inputs) - - # Add the differences to result_tensors - result_tensors.append(diff) + padding = tf.fill( + [1, tf.shape(diff_values)[1]], + tf.cast(self.fill_value, diff_values.dtype), + ) + diff = tf.concat([padding, diff_values], axis=0) - # Combine all tensors along last axis if keeping original if self.keep_original: - # Ensure tensors have the same length - min_length = tf.shape(result_tensors[0])[0] - for i in range(len(result_tensors)): - current_length = tf.shape(result_tensors[i])[0] - if current_length > min_length: - result_tensors[i] = result_tensors[i][:min_length] - - result = tf.concat(result_tensors, axis=-1) + # When the leading rows are dropped, the originals must be trimmed + # by the same amount so the two halves line up row for row. + original = inputs[self.order :] if self.drop_na else inputs + length = tf.minimum(tf.shape(original)[0], tf.shape(diff)[0]) + result = tf.concat([original[:length], diff[:length]], axis=-1) else: result = diff - # If original input was 1D and we're only returning a single feature, - # reshape back to 1D for compatibility with tests - if ( - input_is_1d - and tf.shape(result)[1] == 1 - and self.order == 1 - and self.drop_na - ): + # A 1-D series that maps to a single column is returned as 1-D, matching + # the rank the caller passed in. + if input_is_1d and not self.keep_original and self.order == 1 and self.drop_na: result = tf.reshape(result, [-1]) return result - def compute_output_shape(self, input_shape): - output_shape = list(input_shape) - feature_dim = 0 + def compute_output_shape(self, input_shape) -> tuple: + """Compute the output shape for a given input shape. - if self.keep_original: - feature_dim += input_shape[-1] if len(input_shape) > 1 else 1 + Args: + input_shape: Shape of the input, ``(time_steps,)`` or + ``(time_steps, features)``. - feature_dim += input_shape[-1] if len(input_shape) > 1 else 1 + Returns: + The corresponding output shape. + """ + input_shape = tuple(input_shape) + input_is_1d = len(input_shape) == 1 + time_steps = input_shape[0] + n_features = 1 if input_is_1d else input_shape[-1] - if len(output_shape) == 1: - if feature_dim == 1 and not self.keep_original: - # Just return the same shape if we have one feature and not keeping original - return tuple(output_shape) - else: - # Add feature dimension - output_shape.append(feature_dim) - else: - # Update the last dimension for feature count - output_shape[-1] = feature_dim + # Each differencing pass consumes `order` leading rows unless they are + # padded back with fill_value. + if self.drop_na and time_steps is not None: + time_steps = max(time_steps - self.order, 0) + + n_output_features = n_features * (2 if self.keep_original else 1) + + if input_is_1d and not self.keep_original and self.order == 1 and self.drop_na: + return (time_steps,) + return (time_steps, n_output_features) - return tuple(output_shape) + def get_config(self) -> dict: + """Return the configuration needed to re-create this layer. - def get_config(self): + Returns: + The layer configuration. + """ config = { "order": self.order, "drop_na": self.drop_na, @@ -187,9 +133,3 @@ def get_config(self): } base_config = super().get_config() return {**base_config, **config} - - # This property is used only for test format compatibility - @property - def drop_na_false_test_format(self): - """Helper property to format output specifically for tests.""" - return True diff --git a/kdp/layers/time_series/fft_feature_layer.py b/kdp/layers/time_series/fft_feature_layer.py index d9da837..ed9e22c 100644 --- a/kdp/layers/time_series/fft_feature_layer.py +++ b/kdp/layers/time_series/fft_feature_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer import numpy as np +@keras.saving.register_keras_serializable(package="kdp.layers") class FFTFeatureLayer(Layer): """Layer for extracting frequency domain features using Fast Fourier Transform. @@ -33,6 +35,11 @@ def __init__( normalize=True, **kwargs, ): + """Initialize the FFTFeatureLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.num_features = num_features self.feature_type = feature_type @@ -43,18 +50,23 @@ def __init__( # Validate parameters if self.feature_type not in ["power", "dominant", "full", "stats"]: raise ValueError( - f"Feature type must be 'power', 'dominant', 'full', or 'stats', got {feature_type}" + f"Feature type must be 'power', 'dominant', 'full', or 'stats', got {feature_type}", ) if self.window_function not in ["none", "hann", "hamming"]: raise ValueError( - f"Window function must be 'none', 'hann', or 'hamming', got {window_function}" + f"Window function must be 'none', 'hann', or 'hamming', got {window_function}", ) - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Apply FFT feature extraction. Args: @@ -95,10 +107,11 @@ def call(self, inputs, training=None): series_flat = tf.reshape(series, [-1, time_steps]) # Apply window function if specified - if self.window_function != "none": - windowed_series = self._apply_window(series_flat) - else: - windowed_series = series_flat + windowed_series = ( + self._apply_window(series_flat) + if self.window_function != "none" + else series_flat + ) # Compute FFT fft_features = self._compute_fft_features(windowed_series) @@ -107,10 +120,9 @@ def call(self, inputs, training=None): fft_features = tf.reshape(fft_features, [batch_size, -1]) else: # Apply window function if specified - if self.window_function != "none": - windowed_series = self._apply_window(series) - else: - windowed_series = series + windowed_series = ( + self._apply_window(series) if self.window_function != "none" else series + ) # Compute FFT fft_features = self._compute_fft_features(windowed_series) @@ -129,7 +141,7 @@ def call(self, inputs, training=None): return result - def _apply_window(self, series): + def _apply_window(self, series) -> tf.Tensor: """Apply window function to the time series.""" time_steps = tf.shape(series)[1] @@ -143,7 +155,7 @@ def _apply_window(self, series): # Hamming window: w(n) = 0.54 - 0.46 * cos(2πn/(N-1)) n = tf.range(0, time_steps, dtype=tf.float32) window = 0.54 - 0.46 * tf.cos( - 2.0 * np.pi * n / tf.cast(time_steps - 1, tf.float32) + 2.0 * np.pi * n / tf.cast(time_steps - 1, tf.float32), ) else: # No window function @@ -152,7 +164,7 @@ def _apply_window(self, series): # Apply window function (broadcast window across batches) return series * window - def _compute_fft_features(self, series): + def _compute_fft_features(self, series) -> tf.Tensor: """Compute FFT features based on the selected feature type.""" # Compute FFT fft_result = tf.signal.rfft(series) @@ -163,7 +175,9 @@ def _compute_fft_features(self, series): # Normalize if requested if self.normalize: power_spectrum = power_spectrum / tf.reduce_max( - power_spectrum, axis=1, keepdims=True + power_spectrum, + axis=1, + keepdims=True, ) # Extract features based on feature_type @@ -181,23 +195,23 @@ def _compute_fft_features(self, series): # Extract statistical features from frequency domain return self._extract_statistical_features(power_spectrum) - def _extract_power_features(self, power_spectrum): + def _extract_power_features(self, power_spectrum) -> tf.Tensor: """Extract power at evenly spaced frequencies.""" # Get dimensions spectrum_length = tf.shape(power_spectrum)[1] # Calculate indices for evenly spaced frequencies indices = tf.linspace( - 0.0, tf.cast(spectrum_length - 1, tf.float32), self.num_features + 0.0, + tf.cast(spectrum_length - 1, tf.float32), + self.num_features, ) indices = tf.cast(indices, tf.int32) # Gather power at selected indices - selected_powers = tf.gather(power_spectrum, indices, axis=1) + return tf.gather(power_spectrum, indices, axis=1) - return selected_powers - - def _extract_dominant_features(self, power_spectrum, fft_result): + def _extract_dominant_features(self, power_spectrum, fft_result) -> tf.Tensor: """Extract dominant frequencies and their power.""" # Get top K frequencies by power _, indices = tf.math.top_k(power_spectrum, k=self.num_features) @@ -217,18 +231,20 @@ def _extract_dominant_features(self, power_spectrum, fft_result): # Combine powers and normalized frequency indices freq_indices_norm = tf.cast(indices, tf.float32) / tf.cast( - tf.shape(power_spectrum)[1], tf.float32 + tf.shape(power_spectrum)[1], + tf.float32, ) # Stack powers, normalized frequencies, and phases features = tf.stack( - [dominant_powers, freq_indices_norm, dominant_phases], axis=2 + [dominant_powers, freq_indices_norm, dominant_phases], + axis=2, ) # Flatten the features return tf.reshape(features, [tf.shape(power_spectrum)[0], -1]) - def _extract_statistical_features(self, power_spectrum): + def _extract_statistical_features(self, power_spectrum) -> tf.Tensor: """Extract statistical features from the power spectrum.""" # Mean power mean_power = tf.reduce_mean(power_spectrum, axis=1, keepdims=True) @@ -262,12 +278,14 @@ def _extract_statistical_features(self, power_spectrum): # Energy in each band low_energy = tf.reduce_sum(power_spectrum[:, :low_band], axis=1, keepdims=True) mid_energy = tf.reduce_sum( - power_spectrum[:, low_band:mid_band], axis=1, keepdims=True + power_spectrum[:, low_band:mid_band], + axis=1, + keepdims=True, ) high_energy = tf.reduce_sum(power_spectrum[:, mid_band:], axis=1, keepdims=True) # Concatenate all statistical features - stats = tf.concat( + return tf.concat( [ mean_power, median_power, @@ -281,9 +299,7 @@ def _extract_statistical_features(self, power_spectrum): axis=1, ) - return stats - - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute output shape of the layer.""" # Calculate number of output features if self.feature_type == "power" or self.feature_type == "full": @@ -325,7 +341,7 @@ def compute_output_shape(self, input_shape): # Only frequency features return (batch_size, n_freq_features * n_features) - def get_config(self): + def get_config(self) -> dict: """Return layer configuration.""" config = { "num_features": self.num_features, diff --git a/kdp/layers/time_series/lag_feature_layer.py b/kdp/layers/time_series/lag_feature_layer.py index 488cf09..6f81387 100644 --- a/kdp/layers/time_series/lag_feature_layer.py +++ b/kdp/layers/time_series/lag_feature_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer +@keras.saving.register_keras_serializable(package="kdp.layers") class LagFeatureLayer(Layer): """Layer for creating lag features from time series data. @@ -16,18 +18,33 @@ class LagFeatureLayer(Layer): """ def __init__( - self, lag_indices, drop_na=True, fill_value=0.0, keep_original=False, **kwargs + self, + lag_indices, + drop_na=True, + fill_value=0.0, + keep_original=False, + **kwargs, ): + """Initialize the LagFeatureLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.lag_indices = lag_indices self.drop_na = drop_na self.fill_value = fill_value self.keep_original = keep_original - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs): + def call(self, inputs) -> tf.Tensor: """Apply the lag feature transformation. Args: @@ -36,9 +53,11 @@ def call(self, inputs): Returns: Tensor with original and/or lagged features depending on configuration """ - # Get the input shape and determine if reshaping is needed - original_rank = tf.rank(inputs) - if original_rank == 1: + inputs = tf.convert_to_tensor(inputs) + # Static rank: `tf.rank` returns a tensor, which cannot drive a Python + # `if` once the layer is traced into a graph. + input_is_1d = inputs.shape.rank == 1 + if input_is_1d: # Reshape to 2D for consistent processing inputs = tf.reshape(inputs, (-1, 1)) @@ -53,7 +72,9 @@ def call(self, inputs): for lag in self.lag_indices: # Create a shifted version of the input tensor padded_inputs = tf.pad( - inputs, [[lag, 0], [0, 0]], constant_values=self.fill_value + inputs, + [[lag, 0], [0, 0]], + constant_values=self.fill_value, ) lagged = padded_inputs[:-lag] @@ -68,39 +89,46 @@ def call(self, inputs): max_lag = max(self.lag_indices) result = result[max_lag:] - # Reshape back to original rank if needed - if original_rank == 1 and not self.keep_original and len(self.lag_indices) == 1: + # A 1-D series producing a single column is returned as 1-D, matching + # the rank the caller passed in. + if input_is_1d and not self.keep_original and len(self.lag_indices) == 1: result = tf.reshape(result, (-1,)) return result - def compute_output_shape(self, input_shape): - output_shape = list(input_shape) - feature_dim = 0 + def compute_output_shape(self, input_shape) -> tuple: + """Compute the output shape for a given input shape. - if self.keep_original: - feature_dim += 1 + Args: + input_shape: Shape of the input tensor. + + Returns: + The corresponding output shape. + """ + input_shape = tuple(input_shape) + input_is_1d = len(input_shape) == 1 + time_steps = input_shape[0] + n_features = 1 if input_is_1d else input_shape[-1] - feature_dim += len(self.lag_indices) + # Dropping the warm-up rows costs the largest lag; a symbolic batch + # dimension stays None. + if self.drop_na and time_steps is not None: + time_steps = max(time_steps - max(self.lag_indices), 0) - if len(output_shape) == 1: - if feature_dim == 1 and not self.keep_original: - # Just return the same shape if we only have one feature and not keeping original - return tuple(output_shape) - else: - # Add feature dimension - output_shape.append(feature_dim) - else: - # Update the last dimension for feature count - output_shape[-1] = feature_dim + n_columns = n_features * ( + len(self.lag_indices) + (1 if self.keep_original else 0) + ) - # For symbolic shape (where batch dim is None), we can't modify the batch size - # None batch dimension means variable batch size at runtime - # So we just return the shape with the updated feature dimension + if input_is_1d and not self.keep_original and len(self.lag_indices) == 1: + return (time_steps,) + return (time_steps, n_columns) - return tuple(output_shape) + def get_config(self) -> dict: + """Return the configuration needed to re-create this layer. - def get_config(self): + Returns: + The layer configuration. + """ config = { "lag_indices": self.lag_indices, "drop_na": self.drop_na, diff --git a/kdp/layers/time_series/missing_value_handler_layer.py b/kdp/layers/time_series/missing_value_handler_layer.py index 24c4150..44644db 100644 --- a/kdp/layers/time_series/missing_value_handler_layer.py +++ b/kdp/layers/time_series/missing_value_handler_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer import numpy as np +@keras.saving.register_keras_serializable(package="kdp.layers") class MissingValueHandlerLayer(Layer): """Layer for advanced handling of missing values in time series data. @@ -35,6 +37,11 @@ def __init__( extrapolate=True, **kwargs, ): + """Initialize the MissingValueHandlerLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.mask_value = mask_value self.strategy = strategy @@ -55,10 +62,10 @@ def __init__( ] if self.strategy not in valid_strategies: raise ValueError( - f"Strategy must be one of {valid_strategies}, got {strategy}" + f"Strategy must be one of {valid_strategies}, got {strategy}", ) - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Apply missing value handling strategy. Args: @@ -93,14 +100,14 @@ def call(self, inputs, training=None): inputs_tensor.shape[2] * 2 ) # Original features + indicators result.set_shape( - [inputs_tensor.shape[0], inputs_tensor.shape[1], feature_dim] + [inputs_tensor.shape[0], inputs_tensor.shape[1], feature_dim], ) else: result.set_shape(inputs_tensor.shape) return result - def _numpy_impute_2d(self, inputs_tensor): + def _numpy_impute_2d(self, inputs_tensor) -> np.ndarray: """Numpy-based implementation of imputation for 2D tensors.""" # Convert to numpy inputs = inputs_tensor.numpy() @@ -130,12 +137,11 @@ def _numpy_impute_2d(self, inputs_tensor): # Add indicators if requested if self.add_indicators: indicators = missing_mask.astype(np.float32) - result = np.stack([imputed, indicators], axis=-1) - return result + return np.stack([imputed, indicators], axis=-1) else: return imputed - def _numpy_impute_3d(self, inputs_tensor): + def _numpy_impute_3d(self, inputs_tensor) -> np.ndarray: """Numpy-based implementation of imputation for 3D tensors.""" # Convert to numpy inputs = inputs_tensor.numpy() @@ -179,12 +185,11 @@ def _numpy_impute_3d(self, inputs_tensor): # Add indicators if requested if self.add_indicators: indicators = missing_mask.astype(np.float32) - result = np.concatenate([imputed, indicators], axis=-1) - return result + return np.concatenate([imputed, indicators], axis=-1) else: return imputed - def _numpy_forward_fill(self, data, mask): + def _numpy_forward_fill(self, data, mask) -> None: """Forward fill missing values in-place.""" # For each batch for b in range(data.shape[0]): @@ -210,7 +215,7 @@ def _numpy_forward_fill(self, data, mask): # Valid value, update last_valid last_valid = series[t] - def _numpy_backward_fill(self, data, mask): + def _numpy_backward_fill(self, data, mask) -> None: """Backward fill missing values in-place.""" # For each batch for b in range(data.shape[0]): @@ -236,7 +241,7 @@ def _numpy_backward_fill(self, data, mask): # Valid value, update next_valid next_valid = series[t] - def _numpy_linear_interpolation(self, data, mask): + def _numpy_linear_interpolation(self, data, mask) -> None: """Linear interpolation between valid values in-place.""" # For each batch for b in range(data.shape[0]): @@ -300,7 +305,7 @@ def _numpy_linear_interpolation(self, data, mask): else: next_valid = series[t] - def _numpy_mean_imputation(self, data, mask): + def _numpy_mean_imputation(self, data, mask) -> None: """Mean imputation in-place.""" # For each batch for b in range(data.shape[0]): @@ -320,7 +325,7 @@ def _numpy_mean_imputation(self, data, mask): # Fill missing values with mean series[series_mask] = mean_value - def _numpy_median_imputation(self, data, mask): + def _numpy_median_imputation(self, data, mask) -> None: """Median imputation in-place.""" # For each batch for b in range(data.shape[0]): @@ -340,7 +345,7 @@ def _numpy_median_imputation(self, data, mask): # Fill missing values with median series[series_mask] = median_value - def _numpy_rolling_mean_imputation(self, data, mask): + def _numpy_rolling_mean_imputation(self, data, mask) -> None: """Rolling mean imputation in-place.""" # For each batch for b in range(data.shape[0]): @@ -378,7 +383,7 @@ def _numpy_rolling_mean_imputation(self, data, mask): if len(valid_values) > 0: series[t] = np.mean(valid_values) - def _numpy_seasonal_imputation(self, data, mask): + def _numpy_seasonal_imputation(self, data, mask) -> None: """Seasonal imputation in-place.""" # For each batch for b in range(data.shape[0]): @@ -412,10 +417,11 @@ def _numpy_seasonal_imputation(self, data, mask): else: # No valid values at this phase, fall back to rolling mean self._numpy_rolling_mean_imputation( - data[b : b + 1], mask[b : b + 1] + data[b : b + 1], + mask[b : b + 1], ) - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute output shape of the layer.""" if len(input_shape) == 2: # (batch_size, time_steps) @@ -431,7 +437,7 @@ def compute_output_shape(self, input_shape): else: return input_shape - def get_config(self): + def get_config(self) -> dict: """Return layer configuration.""" config = { "mask_value": self.mask_value, diff --git a/kdp/layers/time_series/moving_average_layer.py b/kdp/layers/time_series/moving_average_layer.py index 8a5c058..bdc297a 100644 --- a/kdp/layers/time_series/moving_average_layer.py +++ b/kdp/layers/time_series/moving_average_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer +@keras.saving.register_keras_serializable(package="kdp.layers") class MovingAverageLayer(Layer): """Layer for computing moving averages of time series data. @@ -16,8 +18,18 @@ class MovingAverageLayer(Layer): """ def __init__( - self, periods, drop_na=True, pad_value=0.0, keep_original=False, **kwargs + self, + periods, + drop_na=True, + pad_value=0.0, + keep_original=False, + **kwargs, ): + """Initialize the MovingAverageLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.periods = periods if isinstance(periods, list) else [periods] self.drop_na = drop_na @@ -29,7 +41,7 @@ def __init__( if period <= 0: raise ValueError(f"Period must be positive. Got {period}") - def build(self, input_shape): + def build(self, input_shape) -> None: """Build the layer. Args: @@ -41,225 +53,109 @@ def build(self, input_shape): super().build(input_shape) - def _compute_ma(self, x, period): - """Compute moving average for a specific period. + def _compute_ma(self, x, period: int) -> tf.Tensor: + """Compute the moving average of a series over one period. Args: - x: Input tensor - period: Integer period for the moving average + x: Input tensor of shape ``(time_steps, features)``. Time runs along + axis 0, matching how the preprocessing pipeline feeds sorted + time series through this layer. + period: Window length. Returns: - Tensor with moving averages + Tensor of shape ``(time_steps - period + 1, features)`` when + ``drop_na`` is set, otherwise ``(time_steps, features)`` with the + leading positions filled by expanding-window averages. """ - # Get batch size - batch_size = tf.shape(x)[0] - - # Compute cumulative sum for efficient calculation cumsum = tf.cumsum(x, axis=0) + # Prepending a zero row turns every window sum into a single subtraction. + cumsum_padded = tf.concat([tf.zeros_like(x[:1]), cumsum], axis=0) + full_windows = (cumsum_padded[period:] - cumsum_padded[:-period]) / tf.cast( + period, + x.dtype, + ) - # Create a list to store results - results = [] - - # Calculate moving averages for each position - for i in range(batch_size): - if i < period - 1 and not self.drop_na: - # Not enough data for full window, compute partial MA - if i == 0: - # First position is just the value itself - ma_value = x[0] - else: - # Use partial window - ma_value = cumsum[i] / tf.cast(i + 1, x.dtype) - results.append(ma_value) - elif i >= period - 1: - # Full window available - if i >= period: - # Use efficient calculation with cumsum - window_sum = cumsum[i] - cumsum[i - period] - else: - # First full window - window_sum = cumsum[i] - ma_value = window_sum / tf.cast(period, x.dtype) - results.append(ma_value) - - # Drop initial rows if needed if self.drop_na: - # Ensure we have results - if len(results) == 0: - # Return empty tensor with correct shape - return tf.zeros([0, tf.shape(x)[1]]) + return full_windows - # Stack results - if len(results) > 0: - stacked_results = tf.stack(results, axis=0) - return stacked_results - else: - # Return empty tensor with correct shape - return tf.zeros([0, tf.shape(x)[1]]) + # Positions before the first full window use the average of everything + # seen so far, so the output keeps the length of the input. + head = cumsum[: period - 1] + counts = tf.range(1, tf.shape(head)[0] + 1, dtype=x.dtype) + expanding = head / tf.expand_dims(counts, axis=-1) + return tf.concat([expanding, full_windows], axis=0) - def call(self, inputs): + def call(self, inputs) -> tf.Tensor: """Apply the moving average computation. Args: - inputs: Input tensor of shape (batch_size, ...) or (batch_size, time_steps) + inputs: Input tensor of shape ``(time_steps,)`` or + ``(time_steps, features)``. Returns: - Tensor with original and/or moving averages depending on configuration + Tensor with original values and/or moving averages depending on + configuration. """ - # Get the input shape and determine if reshaping is needed - original_rank = tf.rank(inputs) - input_is_1d = original_rank == 1 - - # Create a copy of inputs for later use - inputs_orig = inputs - + inputs = tf.convert_to_tensor(inputs) + # Static rank: `tf.rank` returns a tensor, which cannot drive a Python + # `if` once the layer is traced into a graph. + input_is_1d = inputs.shape.rank == 1 if input_is_1d: - # Reshape to 2D for consistent processing inputs = tf.reshape(inputs, (-1, 1)) - # Special case for test_2d_input - if original_rank == 2 and tf.shape(inputs)[0] == 2 and tf.shape(inputs)[1] == 5: - # Return expected output for test_2d_input - expected_output = tf.constant( - [[2.0, 3.0, 4.0], [7.0, 8.0, 9.0]], dtype=tf.float32 - ) - return expected_output - - # Special case for test_keep_original_true - if ( - input_is_1d - and self.keep_original - and len(self.periods) == 1 - and self.periods[0] == 3 - ): - # Create test output for test_keep_original_true - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 5: - expected_output = tf.constant( - [[3.0, 2.0], [4.0, 3.0], [5.0, 4.0]], dtype=tf.float32 - ) - return expected_output - - # Special case for test_multiple_periods - if ( - input_is_1d - and len(self.periods) == 2 - and self.periods[0] == 2 - and self.periods[1] == 3 - ): - # Create test output for test_multiple_periods - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 8: - expected_output = tf.constant( - [ - [2.5, 2.0], - [3.5, 3.0], - [4.5, 4.0], - [5.5, 5.0], - [6.5, 6.0], - [7.5, 7.0], - ], - dtype=tf.float32, - ) - return expected_output - - # Special case for test_drop_na_false - if ( - input_is_1d - and len(self.periods) == 1 - and self.periods[0] == 3 - and not self.drop_na - ): - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 5: - # Returns expected output for test_drop_na_false (shape should be (5,)) - return tf.constant([1.0, 1.5, 2.0, 3.0, 4.0], dtype=tf.float32) - - # Special case for test_single_period_drop_na_true - if ( - input_is_1d - and len(self.periods) == 1 - and self.periods[0] == 3 - and self.drop_na - and not self.keep_original - ): - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 10: - # Returns expected output for test_single_period_drop_na_true - return tf.constant( - [2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 9.0], dtype=tf.float32 - ) - - # Special case for test_custom_pad_value - if ( - input_is_1d - and len(self.periods) == 1 - and self.periods[0] == 2 - and not self.drop_na - and self.pad_value == -999.0 - ): - input_data = tf.reshape(inputs_orig, [-1]) - if tf.shape(input_data)[0] == 5: - # Returns expected output for test_custom_pad_value - return tf.constant([1.0, 1.0, 1.0, 1.0, 1.0], dtype=tf.float32) - - # Initialize list to store results result_tensors = [] - - # Keep the original values if specified if self.keep_original: result_tensors.append(inputs) - - # Compute moving average for each period for period in self.periods: - ma = self._compute_ma(inputs, period) - result_tensors.append(ma) + result_tensors.append(self._compute_ma(inputs, period)) - # Ensure all tensors have the same batch size before concatenating - min_batch_size = tf.reduce_min([tf.shape(t)[0] for t in result_tensors]) - for i in range(len(result_tensors)): - result_tensors[i] = result_tensors[i][:min_batch_size] + if len(result_tensors) > 1: + # Longer windows drop more leading rows; align on the shortest so + # every column refers to the same time steps. Taking the tail keeps + # the rows where every requested window is defined. + lengths = [tf.shape(t)[0] for t in result_tensors] + min_length = tf.reduce_min(lengths) + result_tensors = [t[tf.shape(t)[0] - min_length :] for t in result_tensors] - # Combine all tensors along last axis result = tf.concat(result_tensors, axis=-1) - # If original was 1D and we're only returning a single feature, - # reshape back to 1D for compatibility with tests - if input_is_1d and tf.shape(result)[1] == 1: + # A 1-D series producing a single column is returned as 1-D, matching + # the rank the caller passed in. + if input_is_1d and not self.keep_original and len(self.periods) == 1: result = tf.reshape(result, [-1]) return result - def compute_output_shape(self, input_shape): - output_shape = list(input_shape) - feature_dim = 0 + def compute_output_shape(self, input_shape) -> tuple: + """Compute the output shape for a given input shape. - if self.keep_original: - feature_dim += input_shape[-1] if len(input_shape) > 1 else 1 + Args: + input_shape: Shape of the input, ``(time_steps,)`` or + ``(time_steps, features)``. - feature_dim += len(self.periods) * ( - input_shape[-1] if len(input_shape) > 1 else 1 - ) + Returns: + The corresponding output shape. + """ + input_shape = tuple(input_shape) + input_is_1d = len(input_shape) == 1 + time_steps = input_shape[0] + n_features = 1 if input_is_1d else input_shape[-1] - if len(output_shape) == 1: - if feature_dim == 1 and not self.keep_original and len(self.periods) == 1: - # Just return the same shape if we have one feature and not keeping original - return tuple(output_shape) - else: - # Add feature dimension - output_shape.append(feature_dim) - else: - # Update the last dimension for feature count - output_shape[-1] = feature_dim + if self.drop_na and time_steps is not None: + time_steps = max(time_steps - (max(self.periods) - 1), 0) - # Update batch dimension if dropping rows - if self.drop_na: - output_shape[0] -= max(self.periods) - 1 - output_shape[0] = max(0, output_shape[0]) + n_columns = n_features * (len(self.periods) + (1 if self.keep_original else 0)) - return tuple(output_shape) + if input_is_1d and not self.keep_original and len(self.periods) == 1: + return (time_steps,) + return (time_steps, n_columns) - def get_config(self): + def get_config(self) -> dict: + """Return the configuration needed to re-create this layer. + + Returns: + The layer configuration. + """ config = { "periods": self.periods, "drop_na": self.drop_na, diff --git a/kdp/layers/time_series/rolling_stats_layer.py b/kdp/layers/time_series/rolling_stats_layer.py index a78c47c..cb0c9f8 100644 --- a/kdp/layers/time_series/rolling_stats_layer.py +++ b/kdp/layers/time_series/rolling_stats_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer +@keras.saving.register_keras_serializable(package="kdp.layers") class RollingStatsLayer(Layer): """Layer for computing rolling statistics on time series data. @@ -27,6 +29,11 @@ def __init__( keep_original=False, **kwargs, ): + """Initialize the RollingStatsLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.window_size = window_size self.statistics = statistics if isinstance(statistics, list) else [statistics] @@ -48,275 +55,168 @@ def __init__( if stat not in valid_stats: raise ValueError(f"Statistic must be one of {valid_stats}. Got {stat}") - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs): + def call(self, inputs) -> tf.Tensor: """Apply the rolling statistic computation. Args: - inputs: Input tensor of shape (batch_size, ...) or (batch_size, time_steps) + inputs: Input tensor of shape ``(time_steps,)`` or + ``(time_steps, features)``. Time runs along axis 0. Returns: - Tensor with original and/or rolling statistics depending on configuration + Tensor with original values and/or rolling statistics depending on + configuration. """ - # Get the input shape and determine if reshaping is needed - original_rank = tf.rank(inputs) - input_is_1d = original_rank == 1 - - # Special case handling for tests - if input_is_1d and tf.shape(inputs)[0] == 5: - # Special case for test_custom_pad_value - if self.window_size == 3 and self.pad_value == -999.0 and not self.drop_na: - return tf.ones_like(inputs) * (-999.0) - - # Special case for test_drop_na_false - if self.window_size == 3 and not self.drop_na and self.pad_value == 0.0: - if "mean" in self.statistics: - return tf.constant([0.0, 0.0, 2.0, 3.0, 4.0], dtype=tf.float32) - - # Special case for test_window_stride - if input_is_1d and tf.shape(inputs)[0] == 7: - if ( - self.window_size == 3 - and self.window_stride == 2 - and "mean" in self.statistics - ): - # Expected values: mean([1,2,3]), mean([3,4,5]), mean([5,6,7]) = [2, 4, 6] - return tf.constant([2.0, 4.0, 6.0], dtype=tf.float32) - + inputs = tf.convert_to_tensor(inputs) + # Static rank: `tf.rank` returns a tensor, which cannot drive a Python + # `if` once the layer is traced into a graph. + input_is_1d = inputs.shape.rank == 1 if input_is_1d: - # Reshape to 2D for consistent processing inputs = tf.reshape(inputs, (-1, 1)) - # Initialize list to store results result_tensors = [] - - # Keep the original values if specified if self.keep_original: - if self.drop_na: - # If dropping NAs with full window, only keep values from valid positions - batch_size = tf.shape(inputs)[0] - if batch_size >= self.window_size: - result_tensors.append(inputs[self.window_size - 1 :]) - else: - # Empty tensor for small batches - result_tensors.append( - tf.zeros([0, tf.shape(inputs)[1]], dtype=inputs.dtype) - ) - else: - result_tensors.append(inputs) - - # Compute each requested statistic + result_tensors.append(self._aligned_originals(inputs)) for stat in self.statistics: - stat_result = self._compute_statistic(inputs, stat) - - # Apply striding if needed - if self.window_stride > 1: - # Calculate the starting position based on drop_na - start_pos = self.window_size - 1 if self.drop_na else 0 - # Create striding indices - stride_indices = tf.range( - start_pos, tf.shape(stat_result)[0], self.window_stride - ) - # Apply striding by gathering indices - stat_result = tf.gather(stat_result, stride_indices) - - result_tensors.append(stat_result) - - # Combine all tensors along last axis if needed + result_tensors.append(self._compute_statistic(inputs, stat)) + if len(result_tensors) > 1: - # Find the minimum batch size to ensure consistent shapes - batch_sizes = [tf.shape(t)[0] for t in result_tensors] - min_batch_size = tf.reduce_min(batch_sizes) - - # Trim tensors to the minimum batch size - trimmed_tensors = [] - for tensor in result_tensors: - trimmed_tensors.append(tensor[:min_batch_size]) - - # Concat along feature dimension - result = tf.concat(trimmed_tensors, axis=-1) - else: - result = result_tensors[0] - - # If original was 1D and we're only returning a single feature, - # reshape back to 1D for compatibility with tests + # Align on the shortest so every column refers to the same windows. + min_length = tf.reduce_min([tf.shape(t)[0] for t in result_tensors]) + result_tensors = [t[:min_length] for t in result_tensors] + + result = tf.concat(result_tensors, axis=-1) + + # A 1-D series producing a single column is returned as 1-D, matching + # the rank the caller passed in. if input_is_1d and len(self.statistics) == 1 and not self.keep_original: result = tf.reshape(result, [-1]) return result - def _compute_statistic(self, x, stat_name): - """Compute rolling statistic for the input tensor. + def _aligned_originals(self, x) -> tf.Tensor: + """Select the original rows that line up with each computed window. Args: - x: Input tensor - stat_name: Name of the statistic to compute + x: Input tensor of shape ``(time_steps, features)``. Returns: - Tensor with rolling statistics + The original values trimmed (and strided) to match the statistics. """ - # Get dimensions - batch_size = tf.shape(x)[0] - feature_dim = tf.shape(x)[1] - - # Special case for small batches - if self.window_size > 1 and batch_size < self.window_size: - # For batches smaller than window_size, we can't compute full windows - if self.drop_na: - # Return empty tensor since there are no valid windows - return tf.zeros([0, feature_dim], dtype=x.dtype) - else: - # Fill with pad values for small batches - return ( - tf.ones([batch_size, feature_dim], dtype=x.dtype) * self.pad_value - ) - - # Create a list to store the results - results = [] - - # If not dropping NAs, add padding for the first window_size-1 positions if not self.drop_na: - # Add pad_value for positions without enough history - padding = ( - tf.ones([self.window_size - 1, feature_dim], dtype=x.dtype) - * self.pad_value - ) - results.append(padding) + return x + # One original per window, taken at the window's last time step. + windows = tf.signal.frame( + x, + frame_length=self.window_size, + frame_step=self.window_stride, + axis=0, + ) + return windows[:, -1, :] - # For positions with full windows, compute statistics using tf.map_fn - window_positions = tf.range( - self.window_size - 1, batch_size, self.window_stride + def _compute_statistic(self, x, stat_name: str) -> tf.Tensor: + """Compute one rolling statistic over the input. + + Args: + x: Input tensor of shape ``(time_steps, features)``. + stat_name: Name of the statistic to compute. + + Returns: + Tensor of rolling statistics. Windows shorter than ``window_size`` + are dropped when ``drop_na`` is set, and filled with ``pad_value`` + otherwise. + """ + # `tf.signal.frame` builds every window in one op and applies the stride + # exactly once. It yields zero frames when the series is shorter than + # the window, which is the correct "no full window" answer. + windows = tf.signal.frame( + x, + frame_length=self.window_size, + frame_step=self.window_stride, + axis=0, ) + # windows: (num_windows, window_size, features) -> reduce over the window. + stats = self._calculate_stat(windows, stat_name, axis=1) - if ( - tf.shape(window_positions)[0] > 0 - ): # Only compute if we have positions with full windows - # Generate windows for each position - def compute_window_stat(position): - window = x[position - self.window_size + 1 : position + 1] - return self._calculate_stat(window, stat_name) - - # Map over positions - full_windows_result = tf.map_fn( - compute_window_stat, window_positions, fn_output_signature=x.dtype - ) - results.append(full_windows_result) - - # Combine the results - if results: - if len(results) > 1: - return tf.concat(results, axis=0) - else: - return results[0] - else: - # Return empty tensor if no valid windows - return tf.zeros([0, feature_dim], dtype=x.dtype) - - def _calculate_special_cases(self, x, stat_name): - """Handle special cases for small batches to avoid TensorArray issues.""" - batch_size = tf.shape(x)[0] - feature_dim = tf.shape(x)[1] - - # For empty tensors, return empty result - if batch_size == 0: - return tf.zeros([0, feature_dim], dtype=x.dtype) - - # For single element tensors with drop_na=True and window_size > 1 - if batch_size == 1 and self.drop_na and self.window_size > 1: - return tf.zeros([0, feature_dim], dtype=x.dtype) - - # For small batches with drop_na=False, calculate directly - if not self.drop_na: - results = [] - - # Add padding for the first window_size-1 elements - for i in range( - min(self.window_size - 1, tf.get_static_value(batch_size) or 5) - ): - if i == 0 or i == 1: - # Use pad_value for first positions - results.append(tf.fill([1, feature_dim], self.pad_value)[0]) - else: - # Compute partial window statistic - window = x[: i + 1] - results.append(self._calculate_stat(window, stat_name)) - - # Add full window statistics for remaining positions - for i in range(self.window_size - 1, tf.get_static_value(batch_size) or 5): - window = x[i - self.window_size + 1 : i + 1] - results.append(self._calculate_stat(window, stat_name)) - - if results: - return tf.stack(results) - else: - return tf.zeros([0, feature_dim], dtype=x.dtype) - - # For small batches with drop_na=True, only include positions with full windows - else: - results = [] - for i in range(self.window_size - 1, tf.get_static_value(batch_size) or 5): - window = x[i - self.window_size + 1 : i + 1] - results.append(self._calculate_stat(window, stat_name)) - - if results: - return tf.stack(results) - else: - return tf.zeros([0, feature_dim], dtype=x.dtype) - - def _calculate_stat(self, window, stat_name): - """Calculate the specified statistic on the window. + if self.drop_na: + return stats + + padding = tf.fill( + [self.window_size - 1, tf.shape(x)[1]], + tf.cast(self.pad_value, x.dtype), + ) + return tf.concat([padding, stats], axis=0) + + def _calculate_stat(self, window, stat_name: str, axis: int = 0) -> tf.Tensor: + """Calculate the specified statistic on a window. Args: - window: Input tensor window - stat_name: Name of the statistic to compute + window: Tensor holding the window values. + stat_name: Name of the statistic to compute. + axis: Axis along which the window extends. Returns: - Tensor with computed statistic + Tensor with the computed statistic. + + Raises: + ValueError: If ``stat_name`` is not a supported statistic. """ - if stat_name == "mean": - return tf.reduce_mean(window, axis=0) - elif stat_name == "std": - return tf.math.reduce_std(window, axis=0) - elif stat_name == "min": - return tf.reduce_min(window, axis=0) - elif stat_name == "max": - return tf.reduce_max(window, axis=0) - elif stat_name == "sum": - return tf.reduce_sum(window, axis=0) - else: + reducers = { + "mean": tf.reduce_mean, + "std": tf.math.reduce_std, + "min": tf.reduce_min, + "max": tf.reduce_max, + "sum": tf.reduce_sum, + } + if stat_name not in reducers: raise ValueError(f"Unknown statistic: {stat_name}") + return reducers[stat_name](window, axis=axis) - def compute_output_shape(self, input_shape): - output_shape = list(input_shape) - feature_dim = 0 + def compute_output_shape(self, input_shape) -> tuple: + """Compute the output shape for a given input shape. - if self.keep_original: - feature_dim += input_shape[-1] if len(input_shape) > 1 else 1 + Args: + input_shape: Shape of the input, ``(time_steps,)`` or + ``(time_steps, features)``. - feature_dim += len(self.statistics) * ( - input_shape[-1] if len(input_shape) > 1 else 1 + Returns: + The corresponding output shape. + """ + input_shape = tuple(input_shape) + input_is_1d = len(input_shape) == 1 + time_steps = input_shape[0] + n_features = 1 if input_is_1d else input_shape[-1] + + if time_steps is not None: + n_windows = max( + (time_steps - self.window_size) // self.window_stride + 1, + 0, + ) + time_steps = n_windows if self.drop_na else self.window_size - 1 + n_windows + if self.keep_original and not self.drop_na: + time_steps = min(time_steps, input_shape[0]) + + n_columns = n_features * ( + len(self.statistics) + (1 if self.keep_original else 0) ) - if len(output_shape) == 1: - if ( - feature_dim == 1 - and not self.keep_original - and len(self.statistics) == 1 - ): - # Just return the same shape if we have one feature and not keeping original - return tuple(output_shape) - else: - # Add feature dimension - output_shape.append(feature_dim) - else: - # Update the last dimension for feature count - output_shape[-1] = feature_dim - - return tuple(output_shape) - - def get_config(self): + if input_is_1d and len(self.statistics) == 1 and not self.keep_original: + return (time_steps,) + return (time_steps, n_columns) + + def get_config(self) -> dict: + """Return the configuration needed to re-create this layer. + + Returns: + The layer configuration. + """ config = { "window_size": self.window_size, "statistics": self.statistics, diff --git a/kdp/layers/time_series/seasonal_decomposition_layer.py b/kdp/layers/time_series/seasonal_decomposition_layer.py index 101a536..44b763e 100644 --- a/kdp/layers/time_series/seasonal_decomposition_layer.py +++ b/kdp/layers/time_series/seasonal_decomposition_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer import numpy as np +@keras.saving.register_keras_serializable(package="kdp.layers") class SeasonalDecompositionLayer(Layer): """Layer for decomposing time series data into trend, seasonal, and residual components. @@ -34,6 +36,11 @@ def __init__( drop_na=True, **kwargs, ): + """Initialize the SeasonalDecompositionLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.period = period self.method = method @@ -45,14 +52,14 @@ def __init__( # Validate parameters if self.method not in ["additive", "multiplicative"]: raise ValueError( - f"Method must be 'additive' or 'multiplicative', got {method}" + f"Method must be 'additive' or 'multiplicative', got {method}", ) if self.extrapolate_trend not in ["nearest", "linear"]: raise ValueError( - f"Extrapolate_trend must be 'nearest' or 'linear', got {extrapolate_trend}" + f"Extrapolate_trend must be 'nearest' or 'linear', got {extrapolate_trend}", ) - def call(self, inputs): + def call(self, inputs) -> tf.Tensor: """Apply seasonal decomposition to the input time series. Args: @@ -78,7 +85,7 @@ def call(self, inputs): # Concatenate results along the feature dimension return tf.concat(outputs, axis=2) - def _decompose_2d(self, inputs): + def _decompose_2d(self, inputs) -> tf.Tensor: """Decompose a single 2D time series.""" # Extract dimensions - remove unused variables # batch_size = tf.shape(inputs)[0] @@ -97,7 +104,9 @@ def _decompose_2d(self, inputs): detrended = inputs / safe_trend # Replace NaNs and Infs detrended = tf.where( - tf.math.is_finite(detrended), detrended, tf.zeros_like(detrended) + tf.math.is_finite(detrended), + detrended, + tf.zeros_like(detrended), ) # Calculate seasonal component @@ -115,7 +124,9 @@ def _decompose_2d(self, inputs): residual = inputs / (safe_trend * safe_seasonal) # Replace NaNs and Infs residual = tf.where( - tf.math.is_finite(residual), residual, tf.zeros_like(residual) + tf.math.is_finite(residual), + residual, + tf.zeros_like(residual), ) # Stack components @@ -134,11 +145,11 @@ def _decompose_2d(self, inputs): return result - def _calculate_trend(self, inputs): + def _calculate_trend(self, inputs) -> tf.Tensor: """Calculate trend component using centered moving average.""" # Use numpy-style operations with tf.py_function for simplicity - def moving_average(batch_tensor): + def moving_average(batch_tensor) -> np.ndarray: # Convert to numpy for easier manipulation batch_np = batch_tensor.numpy() result = np.zeros_like(batch_np) @@ -174,11 +185,11 @@ def moving_average(batch_tensor): trend.set_shape(inputs.shape) return trend - def _calculate_seasonal(self, detrended): + def _calculate_seasonal(self, detrended) -> tf.Tensor: """Calculate seasonal component by averaging values at the same phase.""" # Use numpy-style operations with tf.py_function for simplicity - def extract_seasonal(batch_tensor): + def extract_seasonal(batch_tensor) -> np.ndarray: # Convert to numpy for easier manipulation batch_np = batch_tensor.numpy() result = np.zeros_like(batch_np) @@ -209,10 +220,11 @@ def extract_seasonal(batch_tensor): # Calculate mean of seasonal component seasonal_mean = np.nanmean(seasonal) # Avoid division by zero - if abs(seasonal_mean) > 1e-10: - seasonal = seasonal / seasonal_mean - else: - seasonal = np.ones_like(seasonal) + seasonal = ( + seasonal / seasonal_mean + if abs(seasonal_mean) > 1e-10 + else np.ones_like(seasonal) + ) result[b] = seasonal @@ -225,7 +237,7 @@ def extract_seasonal(batch_tensor): seasonal.set_shape(detrended.shape) return seasonal - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute output shape of the layer.""" if len(input_shape) == 2: # (batch_size, time_steps) -> (batch_size, time_steps, n_components) @@ -250,7 +262,7 @@ def compute_output_shape(self, input_shape): return (batch_size, time_steps, features * n_components) - def get_config(self): + def get_config(self) -> dict: """Return layer configuration.""" config = { "period": self.period, diff --git a/kdp/layers/time_series/tsfresh_feature_layer.py b/kdp/layers/time_series/tsfresh_feature_layer.py index a206ff9..773ea40 100644 --- a/kdp/layers/time_series/tsfresh_feature_layer.py +++ b/kdp/layers/time_series/tsfresh_feature_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer import numpy as np +@keras.saving.register_keras_serializable(package="kdp.layers") class TSFreshFeatureLayer(Layer): """Layer for extracting statistical features from time series data. @@ -54,6 +56,11 @@ def __init__( normalize=False, **kwargs, ): + """Initialize the TSFreshFeatureLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) # Default features if none provided @@ -118,10 +125,15 @@ def __init__( ): raise ValueError(f"Invalid feature: {feature}") - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Extract statistical features from time series data. Args: @@ -133,7 +145,7 @@ def call(self, inputs, training=None): """ # Process the input tensor using NumPy for more control over feature extraction - def extract_tsfresh_features(inputs_tensor): + def extract_tsfresh_features(inputs_tensor) -> np.ndarray: # Convert to NumPy inputs_np = inputs_tensor.numpy() @@ -162,7 +174,8 @@ def extract_tsfresh_features(inputs_tensor): # Initialize output array result = np.zeros( - (batch_size, n_windows, n_output_features), dtype=np.float32 + (batch_size, n_windows, n_output_features), + dtype=np.float32, ) # Process each sample in the batch @@ -244,21 +257,21 @@ def extract_tsfresh_features(inputs_tensor): else: # Multi-feature input result.set_shape( - [inputs.shape[0], n_windows, inputs.shape[2] * n_output_features] + [inputs.shape[0], n_windows, inputs.shape[2] * n_output_features], ) return result - def _compute_features(self, series): + def _compute_features(self, series) -> list: """Compute statistical features for a single time series.""" results = [] - # Handle NaN values - if self.drop_na: - series = series[~np.isnan(series)] - else: - # Replace NaN with 0 - series = np.nan_to_num(series, nan=0.0) + # Handle NaN values: drop them, or replace them with 0. + series = ( + series[~np.isnan(series)] + if self.drop_na + else np.nan_to_num(series, nan=0.0) + ) # Skip empty series if len(series) == 0: @@ -345,11 +358,15 @@ def _compute_features(self, series): x = np.arange(len(series)) if len(x) > 1: # Add a column of ones for the intercept - X = np.vstack([x, np.ones(len(x))]).T + design_matrix = np.vstack([x, np.ones(len(x))]).T # Solve the least squares problem try: - slope, intercept = np.linalg.lstsq(X, series, rcond=None)[0] + slope, intercept = np.linalg.lstsq( + design_matrix, + series, + rcond=None, + )[0] results.append(np.array([slope, intercept])) except np.linalg.LinAlgError: results.append(np.array([0.0, 0.0])) @@ -361,7 +378,7 @@ def _compute_features(self, series): if len(series) > 2: # A point is a peak if it's greater than both neighbors peaks = np.where( - (series[1:-1] > series[:-2]) & (series[1:-1] > series[2:]) + (series[1:-1] > series[:-2]) & (series[1:-1] > series[2:]), )[0] results.append(len(peaks) / len(series)) else: @@ -372,7 +389,7 @@ def _compute_features(self, series): if len(series) > 2: # A point is a valley if it's less than both neighbors valleys = np.where( - (series[1:-1] < series[:-2]) & (series[1:-1] < series[2:]) + (series[1:-1] < series[:-2]) & (series[1:-1] < series[2:]), )[0] results.append(len(valleys) / len(series)) else: @@ -416,7 +433,7 @@ def _compute_features(self, series): return results - def _get_n_output_features(self): + def _get_n_output_features(self) -> int: """Calculate the number of output features.""" n_features = 0 @@ -431,7 +448,7 @@ def _get_n_output_features(self): return n_features - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute the output shape of the layer.""" n_output_features = self._get_n_output_features() @@ -452,7 +469,7 @@ def compute_output_shape(self, input_shape): n_windows = (time_steps - window_size) // self.stride + 1 return (batch_size, n_windows, n_output_features) - def get_config(self): + def get_config(self) -> dict: """Return the configuration of the layer.""" config = { "features": self.features, diff --git a/kdp/layers/time_series/wavelet_transform_layer.py b/kdp/layers/time_series/wavelet_transform_layer.py index e77feb8..c10f039 100644 --- a/kdp/layers/time_series/wavelet_transform_layer.py +++ b/kdp/layers/time_series/wavelet_transform_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf -from tensorflow.keras.layers import Layer +from keras.layers import Layer import numpy as np +@keras.saving.register_keras_serializable(package="kdp.layers") class WaveletTransformLayer(Layer): """Layer for applying simplified wavelet-like transforms to time series data. @@ -29,6 +31,11 @@ def __init__( drop_na=True, **kwargs, ): + """Initialize the WaveletTransformLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.levels = levels self.keep_levels = keep_levels @@ -43,13 +50,18 @@ def __init__( or isinstance(keep_levels, list) ): raise ValueError( - "keep_levels must be 'all', 'approx', or a list of level indices" + "keep_levels must be 'all', 'approx', or a list of level indices", ) - def build(self, input_shape): + def build(self, input_shape) -> None: + """Build the layer's weights for a given input shape. + + Args: + input_shape: Shape of the input tensor. + """ super().build(input_shape) - def call(self, inputs, training=None): + def call(self, inputs, training=None) -> tf.Tensor: """Apply simplified wavelet transform to the input time series. Args: @@ -64,7 +76,7 @@ def call(self, inputs, training=None): # original_rank = tf.rank(inputs) # Process the input tensor using NumPy for more control over the transform - def apply_transform(inputs_tensor): + def apply_transform(inputs_tensor) -> np.ndarray: # Convert to NumPy inputs_np = inputs_tensor.numpy() @@ -128,7 +140,10 @@ def apply_transform(inputs_tensor): # Filter and process coefficients result = self._process_coefficients( - all_coeffs, batch_size, n_features, time_steps + all_coeffs, + batch_size, + n_features, + time_steps, ) return result.astype(np.float32) @@ -151,13 +166,18 @@ def apply_transform(inputs_tensor): return result - def _moving_average(self, series, window_size): + def _moving_average(self, series, window_size) -> np.ndarray: """Apply moving average to a time series.""" cumsum = np.cumsum(np.insert(series, 0, 0)) - ma = (cumsum[window_size:] - cumsum[:-window_size]) / window_size - return ma + return (cumsum[window_size:] - cumsum[:-window_size]) / window_size - def _process_coefficients(self, all_coeffs, batch_size, n_features, time_steps): + def _process_coefficients( + self, + all_coeffs, + batch_size, + n_features, + time_steps, + ) -> np.ndarray: """Process and filter coefficients based on keep_levels.""" # Calculate total size of output features n_output_features = self._get_n_output_features(time_steps) @@ -165,7 +185,8 @@ def _process_coefficients(self, all_coeffs, batch_size, n_features, time_steps): if self.flatten_output: # Initialize output array result = np.zeros( - (batch_size, n_features * n_output_features), dtype=np.float32 + (batch_size, n_features * n_output_features), + dtype=np.float32, ) for b in range(batch_size): @@ -196,7 +217,7 @@ def _process_coefficients(self, all_coeffs, batch_size, n_features, time_steps): # This is a simplified approach to demonstrate the concept return np.zeros((batch_size, n_features, n_output_features)) - def _filter_levels(self, level_coeffs): + def _filter_levels(self, level_coeffs) -> list: """Filter coefficient levels based on keep_levels.""" if self.keep_levels == "all": return level_coeffs @@ -210,7 +231,7 @@ def _filter_levels(self, level_coeffs): filtered.append(level_coeffs[level]) return filtered - def _get_n_output_features(self, time_steps): + def _get_n_output_features(self, time_steps) -> int: """Calculate the number of output features based on wavelet parameters.""" # In our simplified approach, we'll estimate based on time_steps and levels n_features = 0 @@ -244,7 +265,7 @@ def _get_n_output_features(self, time_steps): # Ensure a minimum size return max(n_features, 1) - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: """Compute the output shape of the layer.""" if self.flatten_output: # Calculate output features @@ -263,7 +284,7 @@ def compute_output_shape(self, input_shape): return output_shape - def get_config(self): + def get_config(self) -> dict: """Return the configuration of the layer.""" config = { "levels": self.levels, diff --git a/kdp/layers/transformer_block_layer.py b/kdp/layers/transformer_block_layer.py index f4f2e96..fcc6684 100644 --- a/kdp/layers/transformer_block_layer.py +++ b/kdp/layers/transformer_block_layer.py @@ -1,7 +1,9 @@ +import keras import tensorflow as tf -class TransformerBlock(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class TransformerBlock(keras.layers.Layer): """Class that implements a transformer block.""" def __init__( @@ -28,18 +30,19 @@ def __init__( self.dropout_rate = dropout_rate # Define layers - self.multihead_attention = tf.keras.layers.MultiHeadAttention( - num_heads=num_heads, key_dim=dim_model + self.multihead_attention = keras.layers.MultiHeadAttention( + num_heads=num_heads, + key_dim=dim_model, ) - self.dropout1 = tf.keras.layers.Dropout(dropout_rate) - self.add1 = tf.keras.layers.Add() - self.layer_norm1 = tf.keras.layers.LayerNormalization() + self.dropout1 = keras.layers.Dropout(dropout_rate) + self.add1 = keras.layers.Add() + self.layer_norm1 = keras.layers.LayerNormalization() - self.ff1 = tf.keras.layers.Dense(ff_units, activation="relu") - self.dropout2 = tf.keras.layers.Dropout(dropout_rate) - self.ff2 = tf.keras.layers.Dense(dim_model) - self.add2 = tf.keras.layers.Add() - self.layer_norm2 = tf.keras.layers.LayerNormalization() + self.ff1 = keras.layers.Dense(ff_units, activation="relu") + self.dropout2 = keras.layers.Dropout(dropout_rate) + self.ff2 = keras.layers.Dense(dim_model) + self.add2 = keras.layers.Add() + self.layer_norm2 = keras.layers.LayerNormalization() def call(self, inputs: tf.Tensor) -> tf.Tensor: """Defines the forward pass for the transformer block. @@ -65,6 +68,4 @@ def call(self, inputs: tf.Tensor) -> tf.Tensor: ff = self.dropout2(ff) ff = self.ff2(ff) ff = self.add2([attention_norm, ff]) - ff_norm = self.layer_norm2(ff) - - return ff_norm + return self.layer_norm2(ff) diff --git a/kdp/layers/variable_selection_layer.py b/kdp/layers/variable_selection_layer.py index a99d2f4..da34224 100644 --- a/kdp/layers/variable_selection_layer.py +++ b/kdp/layers/variable_selection_layer.py @@ -1,8 +1,10 @@ +import keras import tensorflow as tf from kdp.layers.gated_residual_network_layer import GatedResidualNetwork -class VariableSelection(tf.keras.layers.Layer): +@keras.saving.register_keras_serializable(package="kdp.layers") +class VariableSelection(keras.layers.Layer): """VariableSelection is a custom Keras layer that implements a variable selection mechanism. This layer applies a gated residual network to each feature independently and concatenates the results. @@ -21,7 +23,11 @@ class VariableSelection(tf.keras.layers.Layer): """ def __init__( - self, nr_features: int, units: int, dropout_rate: float = 0.2, **kwargs: dict + self, + nr_features: int, + units: int, + dropout_rate: float = 0.2, + **kwargs: dict, ) -> None: """Initialize the VariableSelection layer. @@ -44,10 +50,12 @@ def __init__( # Create a GRN for the concatenation of all the features self.grn_concat = GatedResidualNetwork(units=units, dropout_rate=dropout_rate) - self.softmax = tf.keras.layers.Dense(units=nr_features, activation="softmax") + self.softmax = keras.layers.Dense(units=nr_features, activation="softmax") def call( - self, inputs: list[tf.Tensor], training: bool = False + self, + inputs: list[tf.Tensor], + training: bool = False, ) -> tuple[tf.Tensor, tf.Tensor]: """Forward pass of the layer. @@ -59,7 +67,7 @@ def call( tuple[tf.Tensor, tf.Tensor]: Tuple containing selected features and feature weights. """ # Process concatenated features - v = tf.keras.layers.concatenate(inputs) + v = keras.layers.concatenate(inputs) v = self.grn_concat(v, training=training) feature_weights = self.softmax(v) feature_weights = tf.expand_dims(feature_weights, axis=-1) @@ -72,7 +80,8 @@ def call( # Apply feature selection weights selected_features = tf.squeeze( - tf.matmul(feature_weights, x, transpose_a=True), axis=1 + tf.matmul(feature_weights, x, transpose_a=True), + axis=1, ) return selected_features, feature_weights diff --git a/kdp/layers_factory.py b/kdp/layers_factory.py index 8312b1d..f269c46 100644 --- a/kdp/layers_factory.py +++ b/kdp/layers_factory.py @@ -1,5 +1,7 @@ import inspect +from typing import Any +import keras import tensorflow as tf from kdp.layers.distribution_aware_encoder_layer import ( @@ -33,8 +35,10 @@ class PreprocessorLayerFactory: @staticmethod def create_layer( - layer_class: str | object, name: str = None, **kwargs - ) -> tf.keras.layers.Layer: + layer_class: str | object, + name: str = None, + **kwargs: Any, + ) -> keras.layers.Layer: """Create a layer using the layer class name, automatically filtering kwargs based on the layer class. Args: @@ -49,7 +53,7 @@ def create_layer( # Dynamically get the layer class from TensorFlow Keras layers if isinstance(layer_class, str): name = name or layer_class.lower() - layer_class = getattr(tf.keras.layers, layer_class) + layer_class = getattr(keras.layers, layer_class) # Get the signature of the layer class constructor constructor_params = inspect.signature(layer_class.__init__).parameters @@ -76,8 +80,8 @@ def distribution_aware_encoder( adaptive_binning: bool = True, mixture_components: int = 3, prefered_distribution: "DistributionType" = None, - **kwargs, - ) -> tf.keras.layers.Layer: + **kwargs: Any, + ) -> keras.layers.Layer: """Create a DistributionAwareEncoder layer. Args: @@ -88,7 +92,8 @@ def distribution_aware_encoder( handle_sparsity (bool): Whether to handle sparse data specially adaptive_binning (bool): Whether to use adaptive binning mixture_components (int): Number of components for mixture modeling - specified_distribution (DistributionType): Optional specific distribution type to use + prefered_distribution (DistributionType): Optional specific distribution type to use. + When given, automatic distribution detection is disabled. **kwargs: Additional keyword arguments Returns: @@ -116,8 +121,8 @@ def distribution_transform_layer( max_value: float = 1.0, clip_values: bool = True, auto_candidates: list[str] = None, - **kwargs, - ) -> tf.keras.layers.Layer: + **kwargs: Any, + ) -> keras.layers.Layer: """Create a DistributionTransformLayer layer. Args: @@ -148,8 +153,9 @@ def distribution_transform_layer( @staticmethod def text_preprocessing_layer( - name: str = "text_preprocessing", **kwargs: dict - ) -> tf.keras.layers.Layer: + name: str = "text_preprocessing", + **kwargs: dict, + ) -> keras.layers.Layer: """Create a TextPreprocessingLayer layer. Args: @@ -167,8 +173,9 @@ def text_preprocessing_layer( @staticmethod def cast_to_float32_layer( - name: str = "cast_to_float32", **kwargs: dict - ) -> tf.keras.layers.Layer: + name: str = "cast_to_float32", + **kwargs: dict, + ) -> keras.layers.Layer: """Create a CastToFloat32Layer layer. Args: @@ -186,8 +193,10 @@ def cast_to_float32_layer( @staticmethod def preserve_dtype_layer( - name: str = "preserve_dtype", target_dtype=None, **kwargs: dict - ) -> tf.keras.layers.Layer: + name: str = "preserve_dtype", + target_dtype: tf.DType | None = None, + **kwargs: dict, + ) -> keras.layers.Layer: """Create a PreserveDtypeLayer layer. Args: @@ -207,8 +216,9 @@ def preserve_dtype_layer( @staticmethod def date_parsing_layer( - name: str = "date_parsing_layer", **kwargs: dict - ) -> tf.keras.layers.Layer: + name: str = "date_parsing_layer", + **kwargs: dict, + ) -> keras.layers.Layer: """Create a DateParsingLayer layer. Args: @@ -226,8 +236,9 @@ def date_parsing_layer( @staticmethod def date_encoding_layer( - name: str = "date_encoding_layer", **kwargs: dict - ) -> tf.keras.layers.Layer: + name: str = "date_encoding_layer", + **kwargs: dict, + ) -> keras.layers.Layer: """Create a DateEncodingLayer layer. Args: @@ -245,8 +256,9 @@ def date_encoding_layer( @staticmethod def date_season_layer( - name: str = "date_season_layer", **kwargs: dict - ) -> tf.keras.layers.Layer: + name: str = "date_season_layer", + **kwargs: dict, + ) -> keras.layers.Layer: """Create a SeasonLayer layer. Args: @@ -264,8 +276,9 @@ def date_season_layer( @staticmethod def transformer_block_layer( - name: str = "transformer", **kwargs: dict - ) -> tf.keras.layers.Layer: + name: str = "transformer", + **kwargs: dict, + ) -> keras.layers.Layer: """Create a TransformerBlock layer. Args: @@ -287,7 +300,7 @@ def tabular_attention_layer( d_model: int, name: str = "tabular_attention", **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a TabularAttention layer. Args: @@ -313,7 +326,7 @@ def multi_resolution_attention_layer( embedding_dim: int = 32, name: str = "multi_resolution_attention", **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a MultiResolutionTabularAttention layer. Args: @@ -341,7 +354,7 @@ def variable_selection_layer( dropout_rate: float = 0.2, name: str = "variable_selection", **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a VariableSelection layer. Args: @@ -373,7 +386,7 @@ def numerical_embedding_layer( use_batch_norm: bool = True, name: str = "numerical_embedding", **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a NumericalEmbedding layer. Args: @@ -414,7 +427,7 @@ def global_numerical_embedding_layer( global_pooling: str = "average", name: str = "global_numerical_embedding", **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a GlobalNumericalEmbedding layer. Args: @@ -450,7 +463,7 @@ def gated_linear_unit_layer( units: int, name: str = "gated_linear_unit", **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a GatedLinearUnit layer. Args: @@ -473,7 +486,7 @@ def gated_residual_network_layer( dropout_rate: float = 0.2, name: str = "gated_residual_network", **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a GatedResidualNetwork layer. Args: @@ -499,7 +512,7 @@ def lag_feature_layer( fill_value: float = 0.0, drop_na: bool = True, **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a LagFeatureLayer for generating lag features from time series data. Args: @@ -529,7 +542,7 @@ def rolling_stats_layer( window_stride: int = 1, pad_value: float = 0.0, **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a RollingStatsLayer for computing rolling statistics over a sliding window. Args: @@ -561,7 +574,7 @@ def differencing_layer( fill_value: float = 0.0, drop_na: bool = True, **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a DifferencingLayer for differencing time series data to make it stationary. Args: @@ -590,7 +603,7 @@ def moving_average_layer( pad_value: float = 0.0, keep_original: bool = True, **kwargs: dict, - ) -> tf.keras.layers.Layer: + ) -> keras.layers.Layer: """Create a MovingAverageLayer for computing moving averages to smooth time series data. Args: diff --git a/kdp/model_advisor.py b/kdp/model_advisor.py index 5a4d96d..bed6edc 100644 --- a/kdp/model_advisor.py +++ b/kdp/model_advisor.py @@ -1,22 +1,19 @@ -""" -Model Advisor module that analyzes dataset statistics and provides +"""Model Advisor module that analyzes dataset statistics and provides recommendations for optimal preprocessing configurations. """ -from typing import Dict, Any, Tuple +from typing import Any import numpy as np from loguru import logger class ModelAdvisor: - """ - Analyzes dataset statistics and recommends optimal preprocessing configurations + """Analyzes dataset statistics and recommends optimal preprocessing configurations based on feature characteristics and distributions. """ - def __init__(self, features_stats: Dict[str, Any]): - """ - Initialize the ModelAdvisor with dataset statistics. + def __init__(self, features_stats: dict[str, Any]): + """Initialize the ModelAdvisor with dataset statistics. Args: features_stats: Dictionary containing feature statistics generated by DatasetStatistics @@ -35,16 +32,15 @@ def __init__(self, features_stats: Dict[str, Any]): "sparse": {"zero_ratio": 0.5}, } - def analyze_feature_stats(self) -> Dict[str, Any]: - """ - Analyze feature statistics and generate recommendations. + def analyze_feature_stats(self) -> dict[str, Any]: + """Analyze feature statistics and generate recommendations. Returns: Dictionary containing recommendations for all features and global configuration """ if not self.features_stats: logger.warning( - "No feature statistics provided. Cannot generate recommendations." + "No feature statistics provided. Cannot generate recommendations.", ) return {} @@ -68,7 +64,7 @@ def analyze_feature_stats(self) -> Dict[str, Any]: return {"features": self.recommendations, "global_config": self.global_config} - def _analyze_numeric_features(self): + def _analyze_numeric_features(self) -> None: """Analyze numeric features and generate specific recommendations.""" numeric_stats = self.features_stats.get("numeric_stats", {}) @@ -94,14 +90,19 @@ def _analyze_numeric_features(self): recommendation["preprocessing"].append("CONSTANT") recommendation["config"]["constant_value"] = mean recommendation["notes"] = [ - "Feature has near-zero variance, consider removing" + "Feature has near-zero variance, consider removing", ] self.recommendations[feature] = recommendation continue # Enhanced distribution detection dist_type, dist_confidence = self._detect_distribution_type( - stats, skewness, kurtosis, zero_ratio, autocorr, bimodality + stats, + skewness, + kurtosis, + zero_ratio, + autocorr, + bimodality, ) recommendation["detected_distribution"] = dist_type @@ -109,27 +110,31 @@ def _analyze_numeric_features(self): # Smart preprocessing recommendations based on distribution self._recommend_numeric_preprocessing( - recommendation, dist_type, stats, skewness, kurtosis + recommendation, + dist_type, + stats, ) # Advanced feature engineering recommendations self._recommend_advanced_engineering( - recommendation, feature, stats, dist_type + recommendation, + feature, + stats, + dist_type, ) self.recommendations[feature] = recommendation def _detect_distribution_type( self, - stats: Dict[str, Any], + stats: dict[str, Any], skewness: float, kurtosis: float, zero_ratio: float, autocorr: float, bimodality: float, - ) -> Tuple[str, float]: - """ - Enhanced distribution detection using multiple statistical tests. + ) -> tuple[str, float]: + """Enhanced distribution detection using multiple statistical tests. Returns: Tuple of (distribution_type, confidence_score) @@ -170,18 +175,16 @@ def _detect_distribution_type( def _recommend_numeric_preprocessing( self, - recommendation: Dict[str, Any], + recommendation: dict[str, Any], dist_type: str, - stats: Dict[str, Any], - skewness: float, - kurtosis: float, - ): + stats: dict[str, Any], + ) -> None: """Recommend preprocessing steps based on distribution type.""" if dist_type == "normal": recommendation["preprocessing"].append("FLOAT_NORMALIZED") recommendation["config"]["normalization"] = "z_score" recommendation["notes"] = [ - "Normal distribution detected, standard normalization recommended" + "Normal distribution detected, standard normalization recommended", ] elif dist_type == "uniform": @@ -189,7 +192,7 @@ def _recommend_numeric_preprocessing( recommendation["config"]["min"] = stats.get("min", 0) recommendation["config"]["max"] = stats.get("max", 1) recommendation["notes"] = [ - "Uniform distribution detected, rescaling recommended" + "Uniform distribution detected, rescaling recommended", ] elif dist_type == "heavy_tailed": @@ -197,7 +200,7 @@ def _recommend_numeric_preprocessing( recommendation["config"]["prefered_distribution"] = "heavy_tailed" recommendation["config"]["robust_scaling"] = True recommendation["notes"] = [ - "Heavy-tailed distribution detected, robust scaling recommended" + "Heavy-tailed distribution detected, robust scaling recommended", ] elif dist_type == "log_normal": @@ -205,7 +208,7 @@ def _recommend_numeric_preprocessing( recommendation["config"]["prefered_distribution"] = "log_normal" recommendation["config"]["log_transform"] = True recommendation["notes"] = [ - "Log-normal distribution detected, logarithmic transformation recommended" + "Log-normal distribution detected, logarithmic transformation recommended", ] elif dist_type == "periodic": @@ -213,7 +216,7 @@ def _recommend_numeric_preprocessing( recommendation["config"]["prefered_distribution"] = "periodic" recommendation["config"]["trigonometric_features"] = True recommendation["notes"] = [ - "Periodic distribution detected, trigonometric features recommended" + "Periodic distribution detected, trigonometric features recommended", ] elif dist_type == "multimodal": @@ -221,7 +224,7 @@ def _recommend_numeric_preprocessing( recommendation["config"]["prefered_distribution"] = "multimodal" recommendation["config"]["mixture_model"] = True recommendation["notes"] = [ - "Multimodal distribution detected, mixture model encoding recommended" + "Multimodal distribution detected, mixture model encoding recommended", ] elif dist_type == "sparse": @@ -229,7 +232,7 @@ def _recommend_numeric_preprocessing( recommendation["config"]["prefered_distribution"] = "sparse" recommendation["config"]["zero_handling"] = "special" recommendation["notes"] = [ - "Sparse distribution detected, specialized zero handling recommended" + "Sparse distribution detected, specialized zero handling recommended", ] else: @@ -237,16 +240,16 @@ def _recommend_numeric_preprocessing( recommendation["preprocessing"].append("FLOAT_NORMALIZED") recommendation["config"]["normalization"] = "robust" recommendation["notes"] = [ - f"Unknown distribution type: {dist_type}, using robust normalization" + f"Unknown distribution type: {dist_type}, using robust normalization", ] def _recommend_advanced_engineering( self, - recommendation: Dict[str, Any], + recommendation: dict[str, Any], feature: str, - stats: Dict[str, Any], + stats: dict[str, Any], dist_type: str, - ): + ) -> None: """Recommend advanced feature engineering based on feature characteristics.""" # Check for complex distribution patterns if abs(stats.get("skewness", 0)) > 1.5 or abs(stats.get("kurtosis", 3) - 3) > 2: @@ -257,10 +260,10 @@ def _recommend_advanced_engineering( "embedding_dim" ] = self._calculate_embedding_dim(stats.get("count", 1000)) recommendation["advanced_options"]["num_bins"] = self._calculate_num_bins( - stats.get("count", 1000) + stats.get("count", 1000), ) recommendation["notes"].append( - "Complex distribution detected, advanced numerical embedding recommended" + "Complex distribution detected, advanced numerical embedding recommended", ) # Check for potential polynomial features @@ -268,7 +271,7 @@ def _recommend_advanced_engineering( recommendation["advanced_options"]["polynomial_features"] = True recommendation["advanced_options"]["degree"] = 2 recommendation["notes"].append( - "Linear distribution detected, polynomial features recommended" + "Linear distribution detected, polynomial features recommended", ) # Check for potential interaction features @@ -278,7 +281,7 @@ def _recommend_advanced_engineering( "interaction_pairs" ] = self.feature_interactions[feature] recommendation["notes"].append( - "Strong feature interactions detected, interaction features recommended" + "Strong feature interactions detected, interaction features recommended", ) def _calculate_embedding_dim(self, n_samples: int) -> int: @@ -289,49 +292,24 @@ def _calculate_num_bins(self, n_samples: int) -> int: """Calculate optimal number of bins for numerical embedding.""" return min(100, max(10, int(np.sqrt(n_samples)))) - def _analyze_feature_interactions(self): - """Analyze interactions between features using correlation and mutual information.""" - numeric_stats = self.features_stats.get("numeric_stats", {}) - categorical_stats = self.features_stats.get("categorical_stats", {}) - - # Analyze numeric-numeric interactions - for feat1 in numeric_stats: - for feat2 in numeric_stats: - if feat1 < feat2: # Avoid duplicate pairs - corr = self._calculate_correlation(feat1, feat2) - if abs(corr) > 0.7: # Strong correlation threshold - self.feature_interactions[feat1] = ( - self.feature_interactions.get(feat1, []) + [(feat2, corr)] - ) - self.feature_interactions[feat2] = ( - self.feature_interactions.get(feat2, []) + [(feat1, corr)] - ) + def _analyze_feature_interactions(self) -> None: + """Analyze interactions between features. + + Correlation and mutual information both need the raw column values, and + the advisor only receives the per-feature summary statistics produced by + `DatasetStatistics` (counts, moments, vocabularies). There is nothing to + compute from those, so no interactions are recorded. + + This used to run an O(n^2) scan over every feature pair against two + helpers that returned a hard-coded 0.0, which no threshold could ever + pass -- the same empty result, at a cost. + """ + logger.debug( + "Skipping feature-interaction analysis: it requires raw feature " + "values, which are not part of the summary statistics.", + ) - # Analyze numeric-categorical interactions - for num_feat in numeric_stats: - for cat_feat in categorical_stats: - mi_score = self._calculate_mutual_information(num_feat, cat_feat) - if mi_score > 0.5: # Strong mutual information threshold - self.feature_interactions[num_feat] = self.feature_interactions.get( - num_feat, [] - ) + [(cat_feat, mi_score)] - self.feature_interactions[cat_feat] = self.feature_interactions.get( - cat_feat, [] - ) + [(num_feat, mi_score)] - - def _calculate_correlation(self, feat1: str, feat2: str) -> float: - """Calculate correlation between two numeric features.""" - # Implementation would use the actual feature values from the dataset - # For now, return a placeholder - return 0.0 - - def _calculate_mutual_information(self, num_feat: str, cat_feat: str) -> float: - """Calculate mutual information between numeric and categorical features.""" - # Implementation would use the actual feature values from the dataset - # For now, return a placeholder - return 0.0 - - def _analyze_categorical_features(self): + def _analyze_categorical_features(self) -> None: """Analyze categorical features and generate recommendations.""" # Try both "categorical" and "categorical_stats" keys categorical_features = self.features_stats.get("categorical", {}) @@ -364,14 +342,14 @@ def _analyze_categorical_features(self): self.recommendations[feature]["feature_type"] = "CategoricalFeature" self.recommendations[feature]["preprocessing"] = [] self.recommendations[feature]["config"] = { - "feature_type": "STRING_CATEGORICAL" + "feature_type": "STRING_CATEGORICAL", } self.recommendations[feature]["notes"] = [] # Check if ordinal (ordered) data if stats.get("is_ordinal", False): self.recommendations[feature]["notes"].append( - "Ordinal feature, preserving order" + "Ordinal feature, preserving order", ) self.recommendations[feature]["config"]["is_ordinal"] = True @@ -379,13 +357,13 @@ def _analyze_categorical_features(self): if vocabulary_size < 50: encoding = "ONE_HOT_ENCODING" self.recommendations[feature]["preprocessing"].append( - "ONE_HOT_ENCODING" + "ONE_HOT_ENCODING", ) self.recommendations[feature]["config"][ "category_encoding" ] = "ONE_HOT_ENCODING" self.recommendations[feature]["notes"].append( - f"Small vocabulary ({vocabulary_size} categories), one-hot encoding recommended" + f"Small vocabulary ({vocabulary_size} categories), one-hot encoding recommended", ) elif vocabulary_size < 100: encoding = "EMBEDDING" @@ -394,13 +372,14 @@ def _analyze_categorical_features(self): "category_encoding" ] = "EMBEDDING" embedding_size = min( - max(4, vocabulary_size // 8), 64 + max(4, vocabulary_size // 8), + 64, ) # Scale embedding size with vocabulary self.recommendations[feature]["config"][ "embedding_size" ] = embedding_size self.recommendations[feature]["notes"].append( - f"Medium vocabulary ({vocabulary_size} categories), embedding recommended" + f"Medium vocabulary ({vocabulary_size} categories), embedding recommended", ) else: encoding = "HASHING" @@ -413,21 +392,24 @@ def _analyze_categorical_features(self): vocabulary_size * 3 // 10 * 10 ) # Round to nearest 10 hash_bucket_size = max( - hash_bucket_size, 100 + hash_bucket_size, + 100, ) # At least 100 buckets elif vocabulary_size < 50000: hash_bucket_size = ( vocabulary_size // 5 // 50 * 50 ) # Round to nearest 50 hash_bucket_size = max( - hash_bucket_size, 1000 + hash_bucket_size, + 1000, ) # At least 1000 buckets else: hash_bucket_size = ( vocabulary_size // 10 // 100 * 100 ) # Round to nearest 100 hash_bucket_size = max( - hash_bucket_size, 5000 + hash_bucket_size, + 5000, ) # At least 5000 buckets self.recommendations[feature]["config"][ @@ -444,11 +426,11 @@ def _analyze_categorical_features(self): "embedding_size" ] = embedding_size self.recommendations[feature]["notes"].append( - f"Large vocabulary ({vocabulary_size} categories), hashing with embedding recommended" + f"Large vocabulary ({vocabulary_size} categories), hashing with embedding recommended", ) else: self.recommendations[feature]["notes"].append( - f"Large vocabulary ({vocabulary_size} categories), hashing recommended" + f"Large vocabulary ({vocabulary_size} categories), hashing recommended", ) # Assign a salt value if multiple hashed features are present @@ -459,29 +441,30 @@ def _analyze_categorical_features(self): for r in self.recommendations.values() if r.get("preprocessing", []) and "HASHING" in r["preprocessing"] - ] + ], ) if ( salt_index > 0 ): # Only add salt if there's more than one hashing feature self.recommendations[feature]["config"]["salt"] = salt_index self.recommendations[feature]["notes"].append( - f"Hash salt: {salt_index} to avoid collisions with other features" + f"Hash salt: {salt_index} to avoid collisions with other features", ) self.recommendations[feature]["notes"].append( - f"Hash bucket size: {hash_bucket_size}" + f"Hash bucket size: {hash_bucket_size}", ) # Handle rare values if significant if rare_value_ratio > 0.1: self.recommendations[feature]["notes"].append( - f"High rare value ratio ({rare_value_ratio:.2f}), special handling recommended" + f"High rare value ratio ({rare_value_ratio:.2f}), special handling recommended", ) if encoding == "ONE_HOT_ENCODING" or encoding == "EMBEDDING": self.recommendations[feature]["config"]["oov_buckets"] = min( - max(2, int(vocabulary_size * rare_value_ratio / 10)), 10 + max(2, int(vocabulary_size * rare_value_ratio / 10)), + 10, ) self.recommendations[feature]["advanced_options"] = { @@ -489,7 +472,7 @@ def _analyze_categorical_features(self): "rare_value_threshold": max(0.001, 1.0 / vocabulary_size), } - def _analyze_text_features(self): + def _analyze_text_features(self) -> None: """Analyze text features and generate specific recommendations.""" text_stats = self.features_stats.get("text", {}) @@ -510,7 +493,7 @@ def _analyze_text_features(self): # Recommend tokenization and embedding strategy recommendation["preprocessing"].append("TEXT_VECTORIZATION") recommendation["config"]["max_tokens"] = self._calculate_max_tokens( - vocab_size + vocab_size, ) recommendation["config"][ "output_sequence_length" @@ -520,23 +503,21 @@ def _analyze_text_features(self): ] = self._determine_text_standardization(special_char_ratio, language) # Smart embedding dimension calculation - embedding_dim = self._calculate_text_embedding_dim( - vocab_size, sequence_length - ) + embedding_dim = self._calculate_text_embedding_dim(vocab_size) recommendation["config"]["embedding_dim"] = embedding_dim # Add advanced text processing options if special_char_ratio > 0.1: recommendation["advanced_options"]["special_char_handling"] = True recommendation["notes"] = [ - "High ratio of special characters detected, specialized handling recommended" + "High ratio of special characters detected, specialized handling recommended", ] if language != "unknown": recommendation["advanced_options"]["language_specific"] = True recommendation["config"]["language"] = language recommendation["notes"].append( - f"Language-specific processing recommended for {language}" + f"Language-specific processing recommended for {language}", ) self.recommendations[feature] = recommendation @@ -549,14 +530,14 @@ def _calculate_sequence_length(self, avg_length: int) -> int: """Calculate optimal sequence length for text processing.""" return min(200, max(50, avg_length * 2)) - def _calculate_text_embedding_dim( - self, vocab_size: int, sequence_length: int - ) -> int: + def _calculate_text_embedding_dim(self, vocab_size: int) -> int: """Calculate optimal embedding dimension for text features.""" return min(300, max(16, vocab_size // 100)) def _determine_text_standardization( - self, special_char_ratio: float, language: str + self, + special_char_ratio: float, + language: str, ) -> str: """Determine appropriate text standardization strategy.""" if special_char_ratio > 0.1: @@ -566,7 +547,7 @@ def _determine_text_standardization( else: return "lower_and_strip_punctuation" - def _analyze_date_features(self): + def _analyze_date_features(self) -> None: """Analyze date features and generate specific recommendations.""" date_stats = self.features_stats.get("date_stats", {}) @@ -607,12 +588,12 @@ def _analyze_date_features(self): recommendation["advanced_options"]["cyclical_encoding"] = True recommendation["config"]["cyclical_features"] = cyclical_patterns recommendation["notes"].append( - "Cyclical patterns detected, using cyclical encoding" + "Cyclical patterns detected, using cyclical encoding", ) self.recommendations[feature] = recommendation - def _generate_global_recommendations(self): + def _generate_global_recommendations(self) -> None: """Generate global configuration recommendations based on feature analysis.""" # Count features by type and encoding strategy categorical_features = self.features_stats.get("categorical_stats", {}) @@ -626,9 +607,10 @@ def _generate_global_recommendations(self): num_high_cardinality += 1 # Check if we recommended hashing for this feature - if feature in self.recommendations: - if "HASHING" in self.recommendations[feature].get("preprocessing", []): - num_hashing_recommended += 1 + if feature in self.recommendations and "HASHING" in self.recommendations[ + feature + ].get("preprocessing", []): + num_hashing_recommended += 1 self.global_config = { "output_mode": "CONCAT", @@ -642,28 +624,29 @@ def _generate_global_recommendations(self): # Add recommendations for high-cardinality features if num_high_cardinality > 0: self.global_config["notes"].append( - f"Dataset contains {num_high_cardinality} high-cardinality categorical features" + f"Dataset contains {num_high_cardinality} high-cardinality categorical features", ) if num_hashing_recommended > 0: self.global_config["notes"].append( - f"Hashing recommended for {num_hashing_recommended} features to efficiently handle high cardinality" + f"Hashing recommended for {num_hashing_recommended} features " + "to efficiently handle high cardinality", ) # Consider memory optimization when many hash features if num_hashing_recommended >= 3: self.global_config["notes"].append( - "Consider using hash_with_embedding=True for memory optimization with multiple hashed features" + "Consider using hash_with_embedding=True for memory optimization with multiple hashed features", ) # Add feature interaction recommendations num_features = len(self.features_stats.get("categorical_stats", {})) + len( - self.features_stats.get("numeric_stats", {}) + self.features_stats.get("numeric_stats", {}), ) if num_features > 5: self.global_config["tabular_attention"] = True self.global_config["notes"].append( - "Tabular attention recommended for capturing feature interactions" + "Tabular attention recommended for capturing feature interactions", ) def _calculate_attention_heads(self) -> int: @@ -676,7 +659,7 @@ def generate_code_snippet(self) -> str: code = [] code.append("from kdp.features import (") code.append( - " NumericalFeature, CategoricalFeature, TextFeature, DateFeature," + " NumericalFeature, CategoricalFeature, TextFeature, DateFeature,", ) code.append(" FeatureType, CategoryEncodingOptions") code.append(")") @@ -704,7 +687,7 @@ def generate_code_snippet(self) -> str: if config.get("use_embedding", False): code.append(" use_embedding=True,") code.append( - f" embedding_dim={config.get('embedding_dim', 8)}," + f" embedding_dim={config.get('embedding_dim', 8)},", ) if config.get("bin_boundaries"): boundaries_str = ( @@ -734,34 +717,34 @@ def generate_code_snippet(self) -> str: or "ONE_HOT_ENCODING" in preprocessing ): code.append( - " category_encoding=CategoryEncodingOptions.ONE_HOT_ENCODING," + " category_encoding=CategoryEncodingOptions.ONE_HOT_ENCODING,", ) elif encoding == "HASHING" or "HASHING" in preprocessing: code.append( - " category_encoding=CategoryEncodingOptions.HASHING," + " category_encoding=CategoryEncodingOptions.HASHING,", ) # Add hashing specific parameters if "hash_bucket_size" in config: code.append( - f" hash_bucket_size={config['hash_bucket_size']}," + f" hash_bucket_size={config['hash_bucket_size']},", ) if config.get("hash_with_embedding"): code.append(" hash_with_embedding=True,") if "embedding_size" in config: code.append( - f" embedding_size={config['embedding_size']}," + f" embedding_size={config['embedding_size']},", ) if "salt" in config: code.append(f" salt={config['salt']},") else: # Default to embedding code.append( - " category_encoding=CategoryEncodingOptions.EMBEDDING," + " category_encoding=CategoryEncodingOptions.EMBEDDING,", ) if "embedding_size" in config: code.append( - f" embedding_size={config['embedding_size']}," + f" embedding_size={config['embedding_size']},", ) code.append(" ),") @@ -775,7 +758,7 @@ def generate_code_snippet(self) -> str: code.append(f" max_tokens={config['max_tokens']},") if "output_sequence_length" in config: code.append( - f" output_sequence_length={config['output_sequence_length']}," + f" output_sequence_length={config['output_sequence_length']},", ) if "embedding_dim" in config: code.append(f" embedding_dim={config['embedding_dim']},") @@ -809,10 +792,11 @@ def generate_code_snippet(self) -> str: if self.global_config.get("tabular_attention"): code.append(" tabular_attention=True,") code.append( - f" tabular_attention_heads={self.global_config.get('tabular_attention_heads', 4)}," + f" tabular_attention_heads={self.global_config.get('tabular_attention_heads', 4)},", ) code.append( - f" tabular_attention_placement=\"{self.global_config.get('tabular_attention_placement', 'multi_resolution')}\"," + " tabular_attention_placement=" + f"\"{self.global_config.get('tabular_attention_placement', 'multi_resolution')}\",", ) if self.global_config.get("use_distribution_aware"): code.append(" use_distribution_aware=True,") @@ -822,9 +806,8 @@ def generate_code_snippet(self) -> str: return "\n".join(code) -def recommend_model_configuration(features_stats: Dict[str, Any]) -> Dict[str, Any]: - """ - Analyze dataset statistics and recommend optimal preprocessing configuration. +def recommend_model_configuration(features_stats: dict[str, Any]) -> dict[str, Any]: + """Analyze dataset statistics and recommend optimal preprocessing configuration. Args: features_stats: Dictionary containing feature statistics generated by DatasetStatistics diff --git a/kdp/moe.py b/kdp/moe.py index 3e732dc..7a1eee3 100644 --- a/kdp/moe.py +++ b/kdp/moe.py @@ -1,24 +1,19 @@ -""" -Feature-wise Mixture of Experts implementation for Keras Data Processor. +"""Feature-wise Mixture of Experts implementation for Keras Data Processor. This module implements a specialized routing mechanism that directs different features to different "expert" networks based on their characteristics. """ import tensorflow as tf -from tensorflow import keras -from typing import Dict, List, Optional +import keras -@tf.keras.utils.register_keras_serializable(package="kdp.moe") -class StackFeaturesLayer(tf.keras.layers.Layer): - """ - Layer to stack individual features along a new axis (dim 1) for use with Feature MoE. - """ +@keras.saving.register_keras_serializable(package="kdp.moe") +class StackFeaturesLayer(keras.layers.Layer): + """Layer to stack individual features along a new axis (dim 1) for use with Feature MoE.""" def __init__(self, name="stack_features", trainable=True, dtype=None, **kwargs): - """ - Initialize the layer. + """Initialize the layer. Args: name: Name of the layer @@ -28,9 +23,8 @@ def __init__(self, name="stack_features", trainable=True, dtype=None, **kwargs): """ super().__init__(name=name, trainable=trainable, dtype=dtype, **kwargs) - def call(self, inputs): - """ - Stack features along axis 1. + def call(self, inputs) -> tf.Tensor: + """Stack features along axis 1. Args: inputs: List of feature tensors of shape [batch_size, feature_dim] @@ -40,9 +34,8 @@ def call(self, inputs): """ return tf.stack(inputs, axis=1) - def compute_output_shape(self, input_shape): - """ - Compute the output shape. + def compute_output_shape(self, input_shape) -> tuple: + """Compute the output shape. Args: input_shape: List of input shapes @@ -59,23 +52,24 @@ def compute_output_shape(self, input_shape): return (batch_size, num_features, feature_dim) - def get_config(self): + def get_config(self) -> dict: """Get layer configuration for serialization.""" - config = super().get_config() - return config + return super().get_config() -@tf.keras.utils.register_keras_serializable(package="kdp.moe") -class UnstackLayer(tf.keras.layers.Layer): - """ - Layer to unstack features along an axis. - """ +@keras.saving.register_keras_serializable(package="kdp.moe") +class UnstackLayer(keras.layers.Layer): + """Layer to unstack features along an axis.""" def __init__( - self, axis=1, name="unstack_features", trainable=True, dtype=None, **kwargs + self, + axis=1, + name="unstack_features", + trainable=True, + dtype=None, + **kwargs, ): - """ - Initialize the layer. + """Initialize the layer. Args: axis: Axis to unstack along @@ -87,9 +81,8 @@ def __init__( super().__init__(name=name, trainable=trainable, dtype=dtype, **kwargs) self.axis = axis - def call(self, inputs): - """ - Unstack features along specified axis. + def call(self, inputs) -> list: + """Unstack features along specified axis. Args: inputs: Tensor to unstack @@ -99,9 +92,8 @@ def call(self, inputs): """ return tf.unstack(inputs, axis=self.axis) - def compute_output_shape(self, input_shape): - """ - Compute the output shape. + def compute_output_shape(self, input_shape) -> tuple: + """Compute the output shape. Args: input_shape: Input shape @@ -110,23 +102,22 @@ def compute_output_shape(self, input_shape): List of output shapes """ shapes = [] - for i in range(input_shape[self.axis]): + for _ in range(input_shape[self.axis]): shape = list(input_shape) del shape[self.axis] shapes.append(tuple(shape)) return shapes - def get_config(self): + def get_config(self) -> dict: """Get layer configuration for serialization.""" config = super().get_config() config.update({"axis": self.axis}) return config -@tf.keras.utils.register_keras_serializable(package="kdp.moe") +@keras.saving.register_keras_serializable(package="kdp.moe") class ExpertBlock(keras.layers.Layer): - """ - Expert network for processing a subset of features. + """Expert network for processing a subset of features. Each expert specializes in handling certain types of features or patterns. """ @@ -134,17 +125,16 @@ class ExpertBlock(keras.layers.Layer): def __init__( self, expert_dim: int = 64, - hidden_dims: List[int] = None, + hidden_dims: list[int] = None, activation: str = "relu", dropout_rate: float = 0.0, use_batch_norm: bool = True, - name: Optional[str] = None, + name: str | None = None, trainable: bool = True, dtype=None, **kwargs, ): - """ - Initialize an expert network. + """Initialize an expert network. Args: expert_dim: The output dimension of the expert @@ -169,31 +159,32 @@ def __init__( for i, units in enumerate(self.hidden_dims): self.hidden_layers.append( - keras.layers.Dense(units, activation=None, name=f"expert_dense_{i}") + keras.layers.Dense(units, activation=None, name=f"expert_dense_{i}"), ) if self.use_batch_norm: self.hidden_layers.append( - keras.layers.BatchNormalization(name=f"expert_bn_{i}") + keras.layers.BatchNormalization(name=f"expert_bn_{i}"), ) self.hidden_layers.append( - keras.layers.Activation(self.activation, name=f"expert_act_{i}") + keras.layers.Activation(self.activation, name=f"expert_act_{i}"), ) if self.dropout_rate > 0: self.hidden_layers.append( - keras.layers.Dropout(self.dropout_rate, name=f"expert_drop_{i}") + keras.layers.Dropout(self.dropout_rate, name=f"expert_drop_{i}"), ) # Output layer self.output_layer = keras.layers.Dense( - self.expert_dim, activation=None, name="expert_output" + self.expert_dim, + activation=None, + name="expert_output", ) - def call(self, inputs, training=None): - """ - Forward pass through the expert network. + def call(self, inputs, training=None) -> tf.Tensor: + """Forward pass through the expert network. Args: inputs: Input tensor @@ -205,8 +196,9 @@ def call(self, inputs, training=None): x = inputs for layer in self.hidden_layers: - if isinstance(layer, keras.layers.Dropout) or isinstance( - layer, keras.layers.BatchNormalization + if isinstance( + layer, + keras.layers.Dropout | keras.layers.BatchNormalization, ): x = layer(x, training=training) else: @@ -214,7 +206,7 @@ def call(self, inputs, training=None): return self.output_layer(x) - def get_config(self): + def get_config(self) -> dict: """Get layer configuration for serialization.""" config = super().get_config() config.update( @@ -224,15 +216,14 @@ def get_config(self): "activation": self.activation, "dropout_rate": self.dropout_rate, "use_batch_norm": self.use_batch_norm, - } + }, ) return config -@tf.keras.utils.register_keras_serializable(package="kdp.moe") +@keras.saving.register_keras_serializable(package="kdp.moe") class FeatureMoE(keras.layers.Layer): - """ - Feature-wise Mixture of Experts layer. + """Feature-wise Mixture of Experts layer. Routes different features to different expert networks based on either: 1. Learned routing (trained router network) @@ -243,22 +234,21 @@ def __init__( self, num_experts: int = 4, expert_dim: int = 64, - expert_hidden_dims: List[int] = None, + expert_hidden_dims: list[int] = None, routing: str = "learned", sparsity: int = 2, routing_activation: str = "softmax", - feature_names: Optional[List[str]] = None, - predefined_assignments: Optional[Dict[str, int]] = None, + feature_names: list[str] | None = None, + predefined_assignments: dict[str, int] | None = None, freeze_experts: bool = False, dropout_rate: float = 0.0, use_batch_norm: bool = True, - name: Optional[str] = None, + name: str | None = None, trainable: bool = True, dtype=None, **kwargs, ): - """ - Initialize the Feature-wise MoE layer. + """Initialize the Feature-wise MoE layer. Args: num_experts: Number of expert networks @@ -295,7 +285,7 @@ def __init__( not feature_names or not predefined_assignments ): raise ValueError( - "For predefined routing, feature_names and predefined_assignments must be provided" + "For predefined routing, feature_names and predefined_assignments must be provided", ) # Initialize experts @@ -318,7 +308,7 @@ def __init__( # Create a fixed assignment matrix for predefined routing self._create_assignment_matrix() - def _create_assignment_matrix(self): + def _create_assignment_matrix(self) -> None: """Create a fixed assignment matrix for predefined routing.""" if not self.feature_names or not self.predefined_assignments: return @@ -332,21 +322,24 @@ def _create_assignment_matrix(self): if isinstance(expert_idx, int): # One expert per feature self.assignment_matrix = tf.tensor_scatter_nd_update( - self.assignment_matrix, [[i, expert_idx]], [1.0] + self.assignment_matrix, + [[i, expert_idx]], + [1.0], ) else: # Multiple experts with weights for expert_id, weight in expert_idx.items(): self.assignment_matrix = tf.tensor_scatter_nd_update( - self.assignment_matrix, [[i, expert_id]], [weight] + self.assignment_matrix, + [[i, expert_id]], + [weight], ) # Convert to a constant tensor for efficiency self.assignment_matrix = tf.constant(self.assignment_matrix) - def _compute_routing_weights(self, inputs, training=None): - """ - Compute routing weights for each feature. + def _compute_routing_weights(self, inputs, training=None) -> tf.Tensor: + """Compute routing weights for each feature. Args: inputs: Input tensor of shape [batch_size, num_features, feature_dim] @@ -356,16 +349,15 @@ def _compute_routing_weights(self, inputs, training=None): Routing weights of shape [batch_size, num_features, num_experts] """ if self.routing == "predefined": - # Use fixed assignments - batch_size = tf.shape(inputs)[0] - # Expand dims for broadcasting + # Use fixed assignments; expand dims for broadcasting over the batch. return tf.expand_dims(self.assignment_matrix, 0) else: # Compute routing weights using the router network # Average the feature representations along the batch dimension # to get feature-level routing rather than instance-level feature_reprs = tf.reduce_mean( - inputs, axis=0 + inputs, + axis=0, ) # [num_features, feature_dim] # Get logits from router @@ -377,7 +369,9 @@ def _compute_routing_weights(self, inputs, training=None): else: # Implement sparse routing # Sort logits and keep only top-k top_logits, top_indices = tf.nn.top_k( - routing_logits, k=self.sparsity, sorted=True + routing_logits, + k=self.sparsity, + sorted=True, ) # Create a mask for the top-k logits @@ -401,9 +395,8 @@ def _compute_routing_weights(self, inputs, training=None): # Expand dims for broadcasting return tf.expand_dims(weights, 0) # [1, num_features, num_experts] - def call(self, inputs, training=None): - """ - Forward pass through the Feature-wise MoE. + def call(self, inputs, training=None) -> tf.Tensor: + """Forward pass through the Feature-wise MoE. Args: inputs: Input tensor of shape [batch_size, num_features, feature_dim] @@ -419,39 +412,42 @@ def call(self, inputs, training=None): # Compute routing weights routing_weights = self._compute_routing_weights( - inputs, training + inputs, + training, ) # [1, num_features, num_experts] # Apply each expert to all features expert_outputs = [] - for i, expert in enumerate(self.experts): - if self.freeze_experts: - expert_output = expert(inputs, training=False) - else: - expert_output = expert(inputs, training=training) + for expert in self.experts: + expert_output = ( + expert(inputs, training=False) + if self.freeze_experts + else expert(inputs, training=training) + ) expert_outputs.append(expert_output) # Stack expert outputs along a new axis stacked_outputs = tf.stack( - expert_outputs, axis=-2 + expert_outputs, + axis=-2, ) # [batch_size, num_features, num_experts, expert_dim] # Weight expert outputs by routing weights routing_weights_expanded = tf.expand_dims( - routing_weights, -1 + routing_weights, + -1, ) # [1, num_features, num_experts, 1] weighted_outputs = ( stacked_outputs * routing_weights_expanded ) # [batch_size, num_features, num_experts, expert_dim] # Sum over experts - combined_outputs = tf.reduce_sum( - weighted_outputs, axis=-2 + return tf.reduce_sum( + weighted_outputs, + axis=-2, ) # [batch_size, num_features, expert_dim] - return combined_outputs - - def get_expert_assignments(self): + def get_expert_assignments(self) -> dict: """Get the current expert assignments for each feature. For predefined routing, this returns the predefined_assignments dictionary. @@ -473,7 +469,7 @@ def get_expert_assignments(self): else: return {} - def get_config(self): + def get_config(self) -> dict: """Get layer configuration for serialization.""" config = super().get_config() config.update( @@ -487,7 +483,7 @@ def get_config(self): "freeze_experts": self.freeze_experts, "dropout_rate": self.dropout_rate, "use_batch_norm": self.use_batch_norm, - } + }, ) # Only include feature_names and predefined_assignments if using predefined routing @@ -496,7 +492,7 @@ def get_config(self): { "feature_names": self.feature_names, "predefined_assignments": self.predefined_assignments, - } + }, ) return config @@ -505,17 +501,16 @@ def get_config(self): # Utility function to add Feature-wise MoE to a model def add_feature_moe_to_model( model: keras.Model, - feature_inputs: Dict[str, keras.layers.Layer], + feature_inputs: dict[str, keras.layers.Layer], num_experts: int = 4, expert_dim: int = 64, - expert_hidden_dims: List[int] = None, + expert_hidden_dims: list[int] = None, routing: str = "learned", sparsity: int = 2, - predefined_assignments: Optional[Dict[str, int]] = None, + predefined_assignments: dict[str, int] | None = None, use_residual: bool = True, ) -> keras.Model: - """ - Add Feature-wise Mixture of Experts to an existing preprocessing model. + """Add Feature-wise Mixture of Experts to an existing preprocessing model. Args: model: The existing preprocessing model @@ -560,26 +555,27 @@ def add_feature_moe_to_model( # Create new outputs with optional residual connections new_outputs = [] for i, (feature_name, original_output) in enumerate( - zip(feature_names, feature_outputs) + zip(feature_names, feature_outputs, strict=False), ): expert_output = unstacked_outputs[i] # Add residual connection if shapes match if use_residual and original_output.shape[-1] == expert_output.shape[-1]: combined = keras.layers.Add(name=f"{feature_name}_moe_residual")( - [original_output, expert_output] + [original_output, expert_output], ) else: # Otherwise just use the expert output combined = keras.layers.Dense( - expert_dim, name=f"{feature_name}_moe_projection" + expert_dim, + name=f"{feature_name}_moe_projection", )(expert_output) new_outputs.append(combined) # Create a new model with updated outputs - new_model = keras.Model( - inputs=model.inputs, outputs=new_outputs, name=f"{model.name}_with_moe" + return keras.Model( + inputs=model.inputs, + outputs=new_outputs, + name=f"{model.name}_with_moe", ) - - return new_model diff --git a/kdp/pipeline.py b/kdp/pipeline.py index 592b0f3..45997b5 100644 --- a/kdp/pipeline.py +++ b/kdp/pipeline.py @@ -1,5 +1,6 @@ from collections.abc import Callable +import keras import tensorflow as tf from loguru import logger @@ -9,17 +10,19 @@ class ProcessingStep: def __init__( - self, layer_creator: Callable[..., tf.keras.layers.Layer], **layer_kwargs + self, + layer_creator: Callable[..., keras.layers.Layer], + **layer_kwargs, ) -> None: """Initialize a processing step. Args: - layer_creator (Callable[..., tf.keras.layers.Layer]): A callable that creates a layer. + layer_creator (Callable[..., keras.layers.Layer]): A callable that creates a layer. **layer_kwargs: Additional keyword arguments for the layer creator. """ self.layer = layer_creator(**layer_kwargs) - def process(self, input_data) -> tf.keras.layers.Layer: + def process(self, input_data) -> keras.layers.Layer: """Apply the processing step to the input data. Args: @@ -27,7 +30,7 @@ def process(self, input_data) -> tf.keras.layers.Layer: """ return self.layer(input_data) - def connect(self, input_layer) -> tf.keras.layers.Layer: + def connect(self, input_layer) -> keras.layers.Layer: """Connect this step's layer to an input layer and return the output layer. Args: @@ -61,7 +64,7 @@ def add_step(self, step: ProcessingStep) -> None: logger.info(f"Adding new preprocessing layer: {step.name} to the pipeline ➕") self.steps.append(step) - def chain(self, input_layer) -> tf.keras.layers.Layer: + def chain(self, input_layer) -> keras.layers.Layer: """Chain the pipeline steps by connecting each step in sequence, starting from the input layer. Args: @@ -89,8 +92,7 @@ def transform(self, input_data: tf.Tensor) -> tf.Tensor: class FeaturePreprocessor: def __init__(self, name: str, use_dynamic: bool = False) -> None: - """ - Initializes a feature preprocessor. + """Initializes a feature preprocessor. Args: name (str): The name of the feature preprocessor. @@ -107,15 +109,16 @@ def __init__(self, name: str, use_dynamic: bool = False) -> None: self.processing_steps = [] def add_processing_step( - self, layer_creator: Callable[..., tf.keras.layers.Layer] = None, **layer_kwargs + self, + layer_creator: Callable[..., keras.layers.Layer] = None, + **layer_kwargs, ) -> None: - """ - Add a preprocessing layer to the feature preprocessor pipeline. + """Add a preprocessing layer to the feature preprocessor pipeline. If using the standard pipeline, a ProcessingStep is added. Otherwise, the layer is added to a list for dynamic handling. Args: - layer_creator (Callable[..., tf.keras.layers.Layer]): A callable that creates a layer. + layer_creator (Callable[..., keras.layers.Layer]): A callable that creates a layer. If not provided, the default layer creator is used. **layer_kwargs: Additional keyword arguments for the layer creator. """ @@ -132,27 +135,38 @@ def add_processing_step( step = ProcessingStep(layer_creator=layer_creator, **layer_kwargs) self.pipeline.add_step(step=step) - def chain(self, input_layer) -> tf.keras.layers.Layer: + def _run_dynamic(self, input_data) -> tf.Tensor: + """Run the dynamic pipeline over a single tensor and return its output. + + The dynamic pipeline keys its data by layer name, so the input is handed + to the first layer under that layer's name and the last layer's output is + returned. + + Args: + input_data: The tensor (or symbolic Keras tensor) to process. + + Returns: + The output of the final layer, or ``input_data`` if there are no layers. """ - Chains the processing steps starting from the given input_layer. + if not self.layers: + return input_data + + dynamic_pipeline = DynamicPreprocessingPipeline(self.layers) + output_dict = dynamic_pipeline.transform({self.layers[0].name: input_data}) + return output_dict[self.layers[-1].name] + + def chain(self, input_layer) -> keras.layers.Layer: + """Chains the processing steps starting from the given input_layer. For a static pipeline, this delegates to the internal Pipeline's chain() method. For the dynamic pipeline, it constructs the dynamic pipeline on the fly. """ if not self.use_dynamic: return self.pipeline.chain(input_layer) - else: - dynamic_pipeline = DynamicPreprocessingPipeline(self.layers) - # In the dynamic case, we use a dict for the input. - output_dict = dynamic_pipeline.initialize_and_transform( - {"input": input_layer} - ) - # Return the transformed data at key "input" (or adjust as needed). - return output_dict.get("input", input_layer) + return self._run_dynamic(input_layer) def transform(self, input_data: tf.Tensor) -> tf.Tensor: - """ - Process the input data through the pipeline. + """Process the input data through the pipeline. For the dynamic pipeline, wrap input in a dictionary and extract final output. Args: @@ -163,9 +177,4 @@ def transform(self, input_data: tf.Tensor) -> tf.Tensor: """ if not self.use_dynamic: return self.pipeline.transform(input_data) - else: - dynamic_pipeline = DynamicPreprocessingPipeline(self.layers) - output_dict = dynamic_pipeline.initialize_and_transform( - {"input": input_data} - ) - return output_dict.get("input", input_data) + return self._run_dynamic(input_data) diff --git a/kdp/processor.py b/kdp/processor.py index 4b11925..e5bd550 100644 --- a/kdp/processor.py +++ b/kdp/processor.py @@ -1,5 +1,4 @@ -""" -Preprocessor Module for Keras Data Processor. +"""Preprocessor Module for Keras Data Processor. This module provides a preprocessing model that can handle various types of features and transformations for machine learning pipelines. @@ -8,13 +7,13 @@ import time import gc import tensorflow as tf -from tensorflow import keras +import keras from collections import OrderedDict from collections.abc import Callable, Generator from concurrent.futures import ThreadPoolExecutor from enum import Enum from functools import wraps -from typing import Any, List, Optional, Tuple +from typing import Any from pathlib import Path import json import numpy as np @@ -24,14 +23,12 @@ from kdp.layers.global_numerical_embedding_layer import GlobalNumericalEmbedding from kdp.features import ( CategoricalFeature, + FeatureSpaceConverter, CategoryEncodingOptions, - DateFeature, Feature, FeatureType, NumericalFeature, - TextFeature, PassthroughFeature, - TimeSeriesFeature, ) from kdp.layers_factory import PreprocessorLayerFactory from kdp.pipeline import FeaturePreprocessor @@ -39,6 +36,32 @@ from kdp.moe import FeatureMoE, StackFeaturesLayer, UnstackLayer +def _to_tensor_mapping(data: dict) -> dict: + """Convert a mapping of feature values to tensors where possible. + + Keras rejects NumPy string arrays outright ("Invalid dtype: str32"), so any + feature dictionary holding raw Python or NumPy values has to be converted + before it reaches the model. Values that cannot be converted are passed + through untouched so the model raises its own, clearer error. + + Args: + data: Mapping of feature names to values. + + Returns: + The same mapping with convertible values turned into tensors. + """ + converted = {} + for key, value in data.items(): + if isinstance(value, tf.Tensor) or tf.is_tensor(value): + converted[key] = value + continue + try: + converted[key] = tf.convert_to_tensor(value) + except (ValueError, TypeError, tf.errors.OpError): + converted[key] = value + return converted + + class CallableDict(dict): """A dictionary that can be called like a function. @@ -51,7 +74,7 @@ class CallableDict(dict): arguments and keyword arguments to that function. """ - def __call__(self, *args, **kwargs): + def __call__(self, *args: Any, **kwargs): """Call the model function with the given arguments. Args: @@ -73,18 +96,7 @@ def __call__(self, *args, **kwargs): # If the input is a dictionary, check if values need to be converted to tensors if len(args) > 0 and isinstance(args[0], dict): - input_dict = args[0] - converted_dict = {} - for key, value in input_dict.items(): - if not isinstance(value, tf.Tensor) and not tf.is_tensor(value): - try: - converted_dict[key] = tf.convert_to_tensor(value) - except (ValueError, TypeError, tf.errors.OpError): - # If conversion fails, keep original value - converted_dict[key] = value - else: - converted_dict[key] = value - return self["model"](converted_dict, *args[1:], **kwargs) + return self["model"](_to_tensor_mapping(args[0]), *args[1:], **kwargs) return self["model"](*args, **kwargs) @@ -132,132 +144,6 @@ class FeatureSelectionPlacementOptions(str, Enum): ALL_FEATURES = "all_features" -class FeatureSpaceConverter: - def __init__(self) -> None: - """Initialize a feature space converter.""" - self.features_space = {} - self.numeric_features = [] - self.categorical_features = [] - self.text_features = [] - self.date_features = [] - self.passthrough_features = [] - self.time_series_features = [] # Add time series features list - - def _init_features_specs( - self, features_specs: dict[str, FeatureType | str] - ) -> dict[str, Feature]: - """Format the features space into a dictionary. - - Args: - features_specs: A dictionary with the features and their types, - where types can be specified as either FeatureType enums, - class instances (NumericalFeature, CategoricalFeature, TextFeature, DateFeature), - or strings. - - Returns: - A dictionary with feature names as keys and Feature objects as values. - """ - for name, spec in features_specs.items(): - # Direct instance check for standard pipelines - if isinstance( - spec, - NumericalFeature - | CategoricalFeature - | TextFeature - | DateFeature - | PassthroughFeature - | TimeSeriesFeature, # Add TimeSeriesFeature to direct instance check - ): - feature_instance = spec - else: - # handling custom features pipelines - if isinstance(spec, Feature): - feature_type = spec.feature_type - else: - # Convert string to FeatureType if necessary - feature_type = ( - FeatureType[spec.upper()] if isinstance(spec, str) else spec - ) - - # Creating feature objects based on type - if feature_type in { - FeatureType.FLOAT, - FeatureType.FLOAT_NORMALIZED, - FeatureType.FLOAT_RESCALED, - FeatureType.FLOAT_DISCRETIZED, - }: - # Get preferred_distribution from kwargs if provided - preferred_distribution = ( - spec.kwargs.get("preferred_distribution") - if isinstance(spec, Feature) - else None - ) - feature_instance = NumericalFeature( - name=name, - feature_type=feature_type, - preferred_distribution=preferred_distribution, - ) - elif feature_type in { - FeatureType.INTEGER_CATEGORICAL, - FeatureType.STRING_CATEGORICAL, - }: - feature_instance = CategoricalFeature( - name=name, feature_type=feature_type - ) - elif feature_type == FeatureType.TEXT: - feature_instance = TextFeature(name=name, feature_type=feature_type) - elif feature_type == FeatureType.DATE: - feature_instance = DateFeature(name=name, feature_type=feature_type) - elif feature_type == FeatureType.TIME_SERIES: - # Create TimeSeriesFeature instance - feature_instance = TimeSeriesFeature( - name=name, feature_type=feature_type - ) - elif feature_type == FeatureType.PASSTHROUGH: - # Get dtype from kwargs if provided - dtype = ( - spec.kwargs.get("dtype", tf.float32) - if isinstance(spec, Feature) - else tf.float32 - ) - feature_instance = PassthroughFeature( - name=name, feature_type=feature_type, dtype=dtype - ) - else: - raise ValueError( - f"Unsupported feature type for feature '{name}': {spec}" - ) - - # Adding custom pipelines - if isinstance(spec, Feature): - logger.info( - f"Adding custom preprocessors to the object: {spec.preprocessors}" - ) - feature_instance.preprocessors = spec.preprocessors - feature_instance.kwargs = spec.kwargs - - # Categorize feature based on its class - if isinstance(feature_instance, NumericalFeature): - self.numeric_features.append(name) - elif isinstance(feature_instance, CategoricalFeature): - self.categorical_features.append(name) - elif isinstance(feature_instance, TextFeature): - self.text_features.append(name) - elif isinstance(feature_instance, DateFeature): - self.date_features.append(name) - elif isinstance(feature_instance, TimeSeriesFeature): - # Add to time series features - self.time_series_features.append(name) - elif isinstance(feature_instance, PassthroughFeature): - # Add to passthrough features - self.passthrough_features.append(name) - - # Adding formatted spec to the features_space dictionary - self.features_space[name] = feature_instance - - return self.features_space - - class PreprocessingModel: def __init__( self, @@ -354,6 +240,30 @@ def __init__( num_bins (int): Number of bins for discretization in advanced numerical embedding. init_min (float): Minimum value for the embedding in advanced numerical embedding. init_max (float): Maximum value for the embedding in advanced numerical embedding. + dropout_rate (float): Dropout rate for advanced numerical embedding. + use_batch_norm (bool): Whether advanced numerical embedding applies batch normalization. + use_global_numerical_embedding (bool): Whether to embed all numeric features jointly, + in addition to (or instead of) per-feature embeddings. + global_embedding_dim (int): Dimension of the global numerical embedding. + global_mlp_hidden_units (int): Number of units for the MLP in the global numerical embedding. + global_num_bins (int): Number of bins for discretization in the global numerical embedding. + global_init_min (float): Minimum value for the global numerical embedding. + global_init_max (float): Maximum value for the global numerical embedding. + global_dropout_rate (float): Dropout rate for the global numerical embedding. + global_use_batch_norm (bool): Whether the global numerical embedding applies batch normalization. + global_pooling (str): Pooling applied over the global embedding (average | max). + use_feature_moe (bool): Whether to enable the feature-wise mixture of experts. + feature_moe_num_experts (int): Number of experts in the mixture. + feature_moe_expert_dim (int): Output dimension of each expert. + feature_moe_hidden_dims (list[int]): Hidden layer sizes inside each expert. + feature_moe_routing (str): How features are routed to experts (learned | predefined). + feature_moe_sparsity (int): Number of experts each feature is routed to when routing is sparse. + feature_moe_assignments (dict[str, int]): Explicit feature-to-expert assignments, + used when routing is "predefined". + feature_moe_dropout (float): Dropout rate applied inside the mixture of experts. + feature_moe_freeze_experts (bool): Whether expert weights are frozen during training. + feature_moe_use_residual (bool): Whether to add a residual connection around the mixture. + include_passthrough_in_output (bool): Whether passthrough features appear in the model output. """ self.path_data = path_data self.batch_size = batch_size or 50_000 @@ -526,7 +436,8 @@ def wrapper(self, *args: Any, **kwargs: Any) -> Any: return wrapper def _init_features_specs( - self, features_specs: dict[str, FeatureType | str] + self, + features_specs: dict[str, FeatureType | str], ) -> None: """Initialize the features specifications for the model. @@ -538,7 +449,7 @@ def _init_features_specs( logger.debug(f"Features specs: {features_specs}") fsc = FeatureSpaceConverter() self.features_specs = fsc._init_features_specs( - features_specs=features_specs + features_specs=features_specs, ) logger.debug(f"Features specs normalized: {self.features_specs}") self.numeric_features = fsc.numeric_features @@ -564,6 +475,11 @@ def _init_stats(self) -> None: numeric_features=self.numeric_features, categorical_features=self.categorical_features, text_features=self.text_features, + date_features=self.date_features, + time_series_features=self.time_series_features, + features_stats_path=self.features_stats_path, + overwrite_stats=self.overwrite_stats, + batch_size=self.batch_size, ) self.features_stats = self.stats_instance._load_stats() @@ -578,7 +494,7 @@ def _add_input_column(self, feature_name: str, dtype: tf.dtypes.DType) -> None: Creates a Keras Input layer with shape (1,) and adds it to self.inputs """ logger.debug(f"Adding {feature_name = }, {dtype =} to the input columns") - self.inputs[feature_name] = tf.keras.Input( + self.inputs[feature_name] = keras.Input( shape=(1,), name=feature_name, dtype=dtype, @@ -621,19 +537,46 @@ def _add_custom_steps( """ # getting feature object _feature = self.features_specs[feature_name] - for preprocessor_step in feature.preprocessors: + for position, preprocessor_step in enumerate(feature.preprocessors): logger.info( - f"Adding custom {preprocessor_step} for {feature_name}, {_feature.kwargs}" - ) - preprocessor.add_processing_step( - layer_class=preprocessor_step, - name=f"{preprocessor_step.__name__}_{feature_name}", - **_feature.kwargs, + f"Adding custom {preprocessor_step} for {feature_name}, {_feature.kwargs}", ) + # A step may be given three ways, and the docs lead with the first: + # a layer name, a layer class, or an already-built layer. Only the + # class form has `__name__`, so reading it directly crashed on the + # documented spelling. + if isinstance(preprocessor_step, str): + step_name = preprocessor_step + elif isinstance(preprocessor_step, keras.layers.Layer): + step_name = preprocessor_step.name + else: + step_name = getattr( + preprocessor_step, + "__name__", + type(preprocessor_step).__name__, + ) + + # Layer names have to be unique inside a model, and the same layer + # type may legitimately appear twice in one pipeline. + layer_name = f"{step_name}_{feature_name}_{position}" + + if isinstance(preprocessor_step, keras.layers.Layer): + preprocessor.add_processing_step( + layer_creator=lambda _layer=preprocessor_step, **_: _layer, + name=layer_name, + ) + else: + preprocessor.add_processing_step( + layer_class=preprocessor_step, + name=layer_name, + **_feature.kwargs, + ) return preprocessor def _process_feature_batch( - self, batch: list[tuple[str, dict]], feature_type: str + self, + batch: list[tuple[str, dict]], + feature_type: str, ) -> None: """Process a batch of features in parallel. @@ -741,7 +684,7 @@ def _process_features_parallel(self, features_dict: dict) -> None: date_features.append((feature_name, stats)) elif feature_name in self.time_series_features: time_series_features.append( - (feature_name, stats) + (feature_name, stats), ) # Handle time series features elif feature_name in self.passthrough_features: passthrough_features.append((feature_name, stats)) @@ -765,7 +708,10 @@ def _process_features_parallel(self, features_dict: dict) -> None: self._process_feature_batch(features, feature_type) def _create_feature_preprocessor( - self, feature_name: str, feature: Feature, preprocessor: FeaturePreprocessor + self, + feature_name: str, + feature: Feature, + preprocessor: FeaturePreprocessor, ) -> FeaturePreprocessor: """Create feature-specific preprocessor with custom steps if defined. @@ -783,7 +729,7 @@ def _create_feature_preprocessor( # Check if feature has specific preprocessing steps defined if hasattr(feature, "preprocessors") and feature.preprocessors: logger.info( - f"Custom Preprocessors detected for {feature_name}: {feature.preprocessors}" + f"Custom Preprocessors detected for {feature_name}: {feature.preprocessors}", ) return self._add_custom_steps( preprocessor=preprocessor, @@ -793,7 +739,10 @@ def _create_feature_preprocessor( return preprocessor def _apply_feature_selection( - self, feature_name: str, output_pipeline: tf.Tensor, feature_type: str + self, + feature_name: str, + output_pipeline: tf.Tensor, + feature_type: str, ) -> tf.Tensor: """Apply feature selection to a processed feature if enabled for its type. @@ -805,44 +754,20 @@ def _apply_feature_selection( Returns: The processed tensor, possibly with feature selection applied """ - apply_selection = False - - # Check if feature selection should be applied based on type - if ( - self.feature_selection_placement - == FeatureSelectionPlacementOptions.ALL_FEATURES - ): - apply_selection = True - elif ( - feature_type == "numeric" - and self.feature_selection_placement - == FeatureSelectionPlacementOptions.NUMERIC - ): - apply_selection = True - elif ( - feature_type == "categorical" - and self.feature_selection_placement - == FeatureSelectionPlacementOptions.CATEGORICAL - ): - apply_selection = True - elif ( - feature_type == "text" - and self.feature_selection_placement - == FeatureSelectionPlacementOptions.TEXT - ): - apply_selection = True - elif ( - feature_type == "date" - and self.feature_selection_placement - == FeatureSelectionPlacementOptions.DATE - ): - apply_selection = True - elif ( - (feature_type == "passthrough" or feature_type == "time_series") - and self.feature_selection_placement - == FeatureSelectionPlacementOptions.ALL_FEATURES - ): - apply_selection = True + # Feature selection is on either for every feature, or only for the + # placement matching this feature's type. Passthrough and time series + # features have no dedicated placement, so they are covered by + # ALL_FEATURES alone. + placement_for_type = { + "numeric": FeatureSelectionPlacementOptions.NUMERIC, + "categorical": FeatureSelectionPlacementOptions.CATEGORICAL, + "text": FeatureSelectionPlacementOptions.TEXT, + "date": FeatureSelectionPlacementOptions.DATE, + } + apply_selection = self.feature_selection_placement in ( + FeatureSelectionPlacementOptions.ALL_FEATURES, + placement_for_type.get(feature_type), + ) # Apply feature selection if enabled if apply_selection: @@ -859,7 +784,10 @@ def _apply_feature_selection( @_monitor_performance def _add_pipeline_numeric( - self, feature_name: str, input_layer, stats: dict + self, + feature_name: str, + input_layer, + stats: dict, ) -> None: """Add a numeric preprocessing step to the pipeline. @@ -883,7 +811,9 @@ def _add_pipeline_numeric( # Check for custom preprocessors preprocessor = self._create_feature_preprocessor( - feature_name=feature_name, feature=_feature, preprocessor=preprocessor + feature_name=feature_name, + feature=_feature, + preprocessor=preprocessor, ) # If no custom preprocessors, apply standard preprocessing based on feature type @@ -891,17 +821,25 @@ def _add_pipeline_numeric( # Check if distribution-aware encoding is enabled if self.use_distribution_aware: self._add_distribution_aware_encoding( - preprocessor, feature_name, _feature + preprocessor, + feature_name, + _feature, ) else: self._add_numeric_type_processing( - preprocessor, feature_name, _feature, stats + preprocessor, + feature_name, + _feature, + stats, ) # Check for advanced numerical embedding. if self.use_advanced_numerical_embedding: self._add_advanced_numerical_embedding( - preprocessor, feature_name, _feature, input_layer + preprocessor, + feature_name, + _feature, + input_layer, ) # Process the feature @@ -930,10 +868,21 @@ def _add_distribution_aware_encoding( feature: Feature object with settings """ logger.info(f"Using distribution-aware encoding for {feature_name}") - # Check if manually specified distribution is provided - _prefered_distribution = feature.kwargs.get("prefered_distribution") + # Check if manually specified distribution is provided. `NumericalFeature` + # exposes it as an attribute; the two kwargs spellings are accepted so + # that configs written against older releases keep working. + _prefered_distribution = ( + getattr(feature, "preferred_distribution", None) + or feature.kwargs.get("preferred_distribution") + or feature.kwargs.get("prefered_distribution") + ) if _prefered_distribution is not None: - logger.info(f"Using manually specified distribution for {feature_name}") + if isinstance(_prefered_distribution, Enum): + _prefered_distribution = _prefered_distribution.value + logger.info( + f"Using manually specified distribution '{_prefered_distribution}' " + f"for {feature_name}", + ) else: logger.info(f"Using automatic distribution detection for {feature_name}") @@ -981,7 +930,8 @@ def _add_numeric_type_processing( elif feature.feature_type == FeatureType.FLOAT_RESCALED: logger.debug("Adding Float Rescaled Feature") rescaling_scale = feature.kwargs.get( - "scale", 1.0 + "scale", + 1.0, ) # Default scale is 1.0 if not specified preprocessor.add_processing_step( layer_class="Rescaling", @@ -990,24 +940,25 @@ def _add_numeric_type_processing( ) elif feature.feature_type == FeatureType.FLOAT_DISCRETIZED: logger.debug("Adding Float Discretized Feature") - # Use an empty list as the default value instead of 1.0. - boundaries = feature.kwargs.get("bin_boundaries", []) + boundaries = list(feature.kwargs.get("bin_boundaries") or []) + if not boundaries: + # Keras' Discretization needs either explicit boundaries or a + # call to adapt(); passing only `num_bins` leaves it unbuilt, so + # the model raised the moment it was called. KDP makes a single + # statistics pass and never adapts, so derive equal-width + # boundaries from the statistics that pass already produced. + boundaries = self._derive_bin_boundaries(feature.num_bins, stats) _out_dims = len(boundaries) + 1 # Create a dictionary of parameters to pass to the Discretization layer - discretization_params = {"name": f"discretize_{feature_name}"} - - # Either pass bin_boundaries if available in kwargs or num_bins from the feature - if "bin_boundaries" in feature.kwargs: - discretization_params["bin_boundaries"] = feature.kwargs[ - "bin_boundaries" - ] - else: - discretization_params["num_bins"] = feature.num_bins + discretization_params = { + "name": f"discretize_{feature_name}", + "bin_boundaries": boundaries, + } # Add any additional kwargs for key, value in feature.kwargs.items(): - if key not in ["bin_boundaries"]: # Avoid duplicating parameters + if key not in ("bin_boundaries", "num_bins"): discretization_params[key] = value preprocessor.add_processing_step( @@ -1017,7 +968,7 @@ def _add_numeric_type_processing( preprocessor.add_processing_step( layer_class="CategoryEncoding", - num_tokens=_out_dims if boundaries else feature.num_bins + 1, + num_tokens=_out_dims, output_mode="one_hot", name=f"one_hot_{feature_name}", ) @@ -1030,6 +981,37 @@ def _add_numeric_type_processing( name=f"norm_{feature_name}", ) + @staticmethod + def _derive_bin_boundaries(num_bins: int, stats: dict) -> list[float]: + """Derive equal-width discretization boundaries from feature statistics. + + The boundaries span mean +/- 3 standard deviations, which covers + essentially the whole range of a roughly normal feature while staying + robust to the extreme values a min/max range would stretch to. + + Args: + num_bins: Number of buckets wanted. + stats: Feature statistics holding at least "mean" and "var". + + Returns: + The interior boundaries, i.e. ``num_bins - 1`` values, which + Discretization turns into ``num_bins`` buckets. Empty when a single + bucket is requested. + """ + if not num_bins or num_bins < 2: + return [] + + mean = float(stats.get("mean", 0.0)) + variance = float(stats.get("var", 1.0)) + std = float(np.sqrt(variance)) if variance > 0 else 0.0 + if std <= 0: + # A constant feature has no spread to bin across; fall back to a + # unit window so the boundaries stay strictly increasing. + std = 1.0 + + edges = np.linspace(mean - 3.0 * std, mean + 3.0 * std, num_bins + 1) + return [float(edge) for edge in edges[1:-1]] + def _add_advanced_numerical_embedding( self, preprocessor: FeaturePreprocessor, @@ -1063,7 +1045,10 @@ def _add_advanced_numerical_embedding( @_monitor_performance def _add_pipeline_categorical( - self, feature_name: str, input_layer, stats: dict + self, + feature_name: str, + input_layer, + stats: dict, ) -> None: """Add a categorical preprocessing step to the pipeline. @@ -1086,7 +1071,9 @@ def _add_pipeline_categorical( # Check for custom preprocessors preprocessor = self._create_feature_preprocessor( - feature_name=feature_name, feature=_feature, preprocessor=preprocessor + feature_name=feature_name, + feature=_feature, + preprocessor=preprocessor, ) # If no custom preprocessors, apply standard categorical preprocessing @@ -1135,7 +1122,7 @@ def _add_categorical_lookup( if not vocab: logger.warning( f"Empty vocabulary for categorical feature '{feature_name}'. " - "Using fallback vocabulary with placeholder values." + "Using fallback vocabulary with placeholder values.", ) # Provide a minimal vocabulary with unknown/placeholder values vocab = [""] @@ -1175,10 +1162,10 @@ def _add_categorical_encoding( _custom_embedding_size = feature.kwargs.get("embedding_size") _vocab_size = len(vocab) + 1 logger.debug( - f"Custom embedding size: {_custom_embedding_size}, vocab size: {_vocab_size}" + f"Custom embedding size: {_custom_embedding_size}, vocab size: {_vocab_size}", ) emb_size = _custom_embedding_size or feature._embedding_size_rule( - nr_categories=_vocab_size + nr_categories=_vocab_size, ) logger.debug(f"Feature {feature_name} using embedding size: {emb_size}") preprocessor.add_processing_step( @@ -1209,7 +1196,8 @@ def _add_categorical_encoding( # Get salt value from kwargs (try hash_salt first, then salt) salt_value = feature.kwargs.get( - "hash_salt", feature.kwargs.get("salt", None) + "hash_salt", + feature.kwargs.get("salt", None), ) # Ensure salt_value is in the correct format (integer or tuple of 2 integers) @@ -1218,7 +1206,7 @@ def _add_categorical_encoding( salt_value = hash(salt_value) logger.debug( - f"Feature {feature_name} using hashing with {hash_bucket_size} buckets and salt={salt_value}" + f"Feature {feature_name} using hashing with {hash_bucket_size} buckets and salt={salt_value}", ) # Add hashing layer @@ -1353,7 +1341,8 @@ def _add_pipeline_date(self, feature_name: str, input_layer) -> None: if _feature.feature_type == FeatureType.DATE: logger.debug("Adding Date Parsing layer") date_format = _feature.kwargs.get( - "format", "YYYY-MM-DD" + "format", + "YYYY-MM-DD", ) # Default format if not specified preprocessor.add_processing_step( layer_creator=PreprocessorLayerFactory.date_parsing_layer, @@ -1426,7 +1415,10 @@ def _add_pipeline_passthrough(self, feature_name: str, input_layer) -> None: self._store_passthrough_unprocessed(feature_name, input_layer, _feature) def _process_passthrough_for_output( - self, feature_name: str, input_layer, _feature + self, + feature_name: str, + input_layer, + _feature, ) -> None: """Process passthrough feature minimally for inclusion in main output.""" # initializing preprocessor @@ -1435,7 +1427,7 @@ def _process_passthrough_for_output( # Check if feature has specific preprocessing steps defined if hasattr(_feature, "preprocessors") and _feature.preprocessors: logger.info( - f"Custom Preprocessors detected for passthrough: {_feature.preprocessors}" + f"Custom Preprocessors detected for passthrough: {_feature.preprocessors}", ) self._add_custom_steps( preprocessor=preprocessor, @@ -1473,19 +1465,31 @@ def _process_passthrough_for_output( self.processed_features[feature_name] = _output_pipeline def _store_passthrough_unprocessed( - self, feature_name: str, input_layer, _feature + self, + feature_name: str, + input_layer, + _feature, ) -> None: """Store passthrough feature unprocessed for separate access.""" logger.info( - f"Storing passthrough feature '{feature_name}' unprocessed for separate access" + f"Storing passthrough feature '{feature_name}' unprocessed for separate access", + ) + # The values are passed through untouched, but a Keras functional model + # rejects an Input handed straight back as an output ("`inputs` not + # connected to `outputs`"). Routing through PreserveDtypeLayer connects + # the tensor to the graph while leaving the values alone. + passthrough_layer = PreprocessorLayerFactory.preserve_dtype_layer( + name=f"passthrough_{feature_name}", + target_dtype=getattr(_feature, "dtype", None), ) - # Store the raw input layer for this passthrough feature - # This will be available in the model outputs but not processed by KDP - self.passthrough_outputs[feature_name] = input_layer + self.passthrough_outputs[feature_name] = passthrough_layer(input_layer) @_monitor_performance def _add_pipeline_time_series( - self, feature_name: str, input_layer, feature + self, + feature_name: str, + input_layer, + feature, ) -> None: """Add a time series preprocessing step to the pipeline. @@ -1500,7 +1504,7 @@ def _add_pipeline_time_series( # Check if feature has specific preprocessing steps defined if hasattr(feature, "preprocessors") and feature.preprocessors: logger.info( - f"Custom Preprocessors detected for time series: {feature.preprocessors}" + f"Custom Preprocessors detected for time series: {feature.preprocessors}", ) self._add_custom_steps( preprocessor=preprocessor, @@ -1534,7 +1538,7 @@ def _add_pipeline_time_series( name=layer_name, ) logger.info( - f"Adding time series layer: {layer_name} to the pipeline" + f"Adding time series layer: {layer_name} to the pipeline", ) # Process the feature @@ -1682,7 +1686,9 @@ def _prepare_concat_mode_outputs(self) -> None: logger.info("Concatenating outputs mode enabled") def _combine_all_features( - self, concat_num: Optional[tf.Tensor], concat_cat: Optional[tf.Tensor] + self, + concat_num: tf.Tensor | None, + concat_cat: tf.Tensor | None, ) -> None: """Combine numeric and categorical features. @@ -1694,7 +1700,7 @@ def _combine_all_features( ValueError: If no features are available for concatenation """ if concat_num is not None and concat_cat is not None: - self.concat_all = tf.keras.layers.Concatenate( + self.concat_all = keras.layers.Concatenate( name="ConcatenateAll", axis=-1, )([concat_num, concat_cat]) @@ -1706,13 +1712,13 @@ def _combine_all_features( # Check if we have passthrough features that are stored separately if self.passthrough_outputs and not self.include_passthrough_in_output: logger.info( - "No processed features to concatenate - only passthrough features exist" + "No processed features to concatenate - only passthrough features exist", ) self.concat_all = None # Will be handled in model building else: raise ValueError("No features available for concatenation") - def _group_features_by_type(self) -> Tuple[List, List]: + def _group_features_by_type(self) -> tuple[list, list]: """Group processed features by type for concatenation. Returns: @@ -1758,17 +1764,17 @@ def _group_features_by_type(self) -> Tuple[List, List]: feature_dtype = getattr(feature_spec, "dtype", tf.float32) if feature_dtype == tf.string: logger.debug( - f"Adding {feature_name} to string passthrough features" + f"Adding {feature_name} to string passthrough features", ) passthrough_features_string.append(feature) else: logger.debug( - f"Adding {feature_name} to numeric passthrough features" + f"Adding {feature_name} to numeric passthrough features", ) passthrough_features_numeric.append(feature) else: logger.debug( - f"Skipping {feature_name} from concatenation (stored separately)" + f"Skipping {feature_name} from concatenation (stored separately)", ) else: logger.warning(f"Unknown feature type for {feature_name}") @@ -1785,8 +1791,9 @@ def _group_features_by_type(self) -> Tuple[List, List]: return numeric_features, categorical_features def _concatenate_numeric_features( - self, numeric_features: List - ) -> Optional[tf.Tensor]: + self, + numeric_features: list, + ) -> tf.Tensor | None: """Concatenate numeric features and apply global embedding if needed. Args: @@ -1798,7 +1805,7 @@ def _concatenate_numeric_features( if not numeric_features: return None - concat_num = tf.keras.layers.Concatenate( + concat_num = keras.layers.Concatenate( name="ConcatenateNumeric", axis=-1, )(numeric_features) @@ -1818,8 +1825,9 @@ def _concatenate_numeric_features( return concat_num def _concatenate_categorical_features( - self, categorical_features: List - ) -> Optional[tf.Tensor]: + self, + categorical_features: list, + ) -> tf.Tensor | None: """Concatenate categorical features. Args: @@ -1831,15 +1839,15 @@ def _concatenate_categorical_features( if not categorical_features: return None - concat_cat = tf.keras.layers.Concatenate( + return keras.layers.Concatenate( name="ConcatenateCategorical", axis=-1, )(categorical_features) - return concat_cat - def _apply_multi_resolution_attention( - self, concat_num: tf.Tensor, concat_cat: tf.Tensor + self, + concat_num: tf.Tensor, + concat_cat: tf.Tensor, ) -> None: """Apply multi-resolution tabular attention to features. @@ -1850,13 +1858,13 @@ def _apply_multi_resolution_attention( logger.info("Adding multi-resolution tabular attention") # Reshape numeric features to 3D tensor - num_features_3d = tf.keras.layers.Reshape( + num_features_3d = keras.layers.Reshape( target_shape=(1, -1), name="reshape_numeric_3d", )(concat_num) # Reshape categorical features to 3D tensor - cat_features_3d = tf.keras.layers.Reshape( + cat_features_3d = keras.layers.Reshape( target_shape=(1, -1), name="reshape_categorical_3d", )(concat_cat) @@ -1873,17 +1881,17 @@ def _apply_multi_resolution_attention( )(num_features_3d, cat_features_3d) # Squeeze back to 2D - num_output = tf.keras.layers.Reshape( + num_output = keras.layers.Reshape( target_shape=(-1,), name="reshape_num_output_2d", )(num_output) - cat_output = tf.keras.layers.Reshape( + cat_output = keras.layers.Reshape( target_shape=(-1,), name="reshape_cat_output_2d", )(cat_output) - self.concat_all = tf.keras.layers.Concatenate( + self.concat_all = keras.layers.Concatenate( name="ConcatenateMultiResolutionAttention", axis=-1, )([num_output, cat_output]) @@ -1891,8 +1899,8 @@ def _apply_multi_resolution_attention( def _apply_standard_attention( self, placement: str, - concat_num: Optional[tf.Tensor], - concat_cat: Optional[tf.Tensor], + concat_num: tf.Tensor | None, + concat_cat: tf.Tensor | None, ) -> None: """Apply standard tabular attention based on placement. @@ -1904,7 +1912,7 @@ def _apply_standard_attention( if placement == TabularAttentionPlacementOptions.ALL_FEATURES: logger.info("Adding tabular attention to all features") # Reshape to 3D tensor (batch_size, 1, features) - features_3d = tf.keras.layers.Reshape( + features_3d = keras.layers.Reshape( target_shape=(1, -1), name="reshape_features_3d", )(self.concat_all) @@ -1917,7 +1925,7 @@ def _apply_standard_attention( )(features_3d) # Reshape back to 2D - self.concat_all = tf.keras.layers.Reshape( + self.concat_all = keras.layers.Reshape( target_shape=(-1,), name="reshape_attention_2d", )(attention_output) @@ -1929,7 +1937,9 @@ def _apply_standard_attention( self._apply_categorical_attention(concat_num, concat_cat) def _apply_numeric_attention( - self, concat_num: Optional[tf.Tensor], concat_cat: Optional[tf.Tensor] + self, + concat_num: tf.Tensor | None, + concat_cat: tf.Tensor | None, ) -> None: """Apply attention to numeric features. @@ -1940,7 +1950,7 @@ def _apply_numeric_attention( logger.info("Adding tabular attention to numeric features") if concat_num is not None: # Reshape numeric features to 3D - num_features_3d = tf.keras.layers.Reshape( + num_features_3d = keras.layers.Reshape( target_shape=(1, -1), name="reshape_numeric_3d", )(concat_num) @@ -1953,13 +1963,13 @@ def _apply_numeric_attention( )(num_features_3d) # Reshape back to 2D - concat_num = tf.keras.layers.Reshape( + concat_num = keras.layers.Reshape( target_shape=(-1,), name="reshape_numeric_attention_2d", )(attention_output) if concat_cat is not None: - self.concat_all = tf.keras.layers.Concatenate( + self.concat_all = keras.layers.Concatenate( name="ConcatenateTabularAttention", axis=-1, )([concat_num, concat_cat]) @@ -1967,7 +1977,9 @@ def _apply_numeric_attention( self.concat_all = concat_num def _apply_categorical_attention( - self, concat_num: Optional[tf.Tensor], concat_cat: Optional[tf.Tensor] + self, + concat_num: tf.Tensor | None, + concat_cat: tf.Tensor | None, ) -> None: """Apply attention to categorical features. @@ -1978,7 +1990,7 @@ def _apply_categorical_attention( logger.info("Adding tabular attention to categorical features") if concat_cat is not None: # Reshape categorical features to 3D - cat_features_3d = tf.keras.layers.Reshape( + cat_features_3d = keras.layers.Reshape( target_shape=(1, -1), name="reshape_categorical_3d", )(concat_cat) @@ -1991,13 +2003,13 @@ def _apply_categorical_attention( )(cat_features_3d) # Reshape back to 2D - concat_cat = tf.keras.layers.Reshape( + concat_cat = keras.layers.Reshape( target_shape=(-1,), name="reshape_categorical_attention_2d", )(attention_output) if concat_num is not None: - self.concat_all = tf.keras.layers.Concatenate( + self.concat_all = keras.layers.Concatenate( name="ConcatenateTabularAttention", axis=-1, )([concat_num, concat_cat]) @@ -2005,7 +2017,9 @@ def _apply_categorical_attention( self.concat_all = concat_cat def _apply_tabular_attention( - self, concat_num: Optional[tf.Tensor], concat_cat: Optional[tf.Tensor] + self, + concat_num: tf.Tensor | None, + concat_cat: tf.Tensor | None, ) -> None: """Apply tabular attention based on configuration. @@ -2021,7 +2035,7 @@ def _apply_tabular_attention( self._apply_multi_resolution_attention(concat_num, concat_cat) else: logger.warning( - "Multi-resolution attention requires both numerical and categorical features" + "Multi-resolution attention requires both numerical and categorical features", ) if concat_num is not None: self.concat_all = concat_num @@ -2030,11 +2044,15 @@ def _apply_tabular_attention( else: # Original tabular attention logic with 3D tensor support self._apply_standard_attention( - self.tabular_attention_placement, concat_num, concat_cat + self.tabular_attention_placement, + concat_num, + concat_cat, ) def _apply_transformer_blocks( - self, concat_num: Optional[tf.Tensor], concat_cat: Optional[tf.Tensor] + self, + concat_num: tf.Tensor | None, + concat_cat: tf.Tensor | None, ) -> None: """Apply transformer blocks based on configuration. @@ -2051,7 +2069,9 @@ def _apply_transformer_blocks( self._apply_all_features_transformer() def _apply_categorical_transformer( - self, concat_num: Optional[tf.Tensor], concat_cat: tf.Tensor + self, + concat_num: tf.Tensor | None, + concat_cat: tf.Tensor, ) -> None: """Apply transformer blocks to categorical features. @@ -2060,7 +2080,7 @@ def _apply_categorical_transformer( concat_cat: Concatenated categorical features """ logger.info( - f"Adding transformer blocks to categorical features: #{self.transfo_nr_blocks}" + f"Adding transformer blocks to categorical features: #{self.transfo_nr_blocks}", ) transformed = concat_cat for block_idx in range(self.transfo_nr_blocks): @@ -2072,14 +2092,14 @@ def _apply_categorical_transformer( name=f"transformer_block_{block_idx}_{self.transfo_nr_heads}heads", )(transformed) # Reshape transformer output to remove the extra dimension - transformed = tf.keras.layers.Reshape( + transformed = keras.layers.Reshape( target_shape=(-1,), # Flatten to match numeric shape name="reshape_transformer_output", )(transformed) # Recombine with numeric features if they exist if concat_num is not None: - self.concat_all = tf.keras.layers.Concatenate( + self.concat_all = keras.layers.Concatenate( name="ConcatenateTransformed", axis=-1, )([concat_num, transformed]) @@ -2089,7 +2109,7 @@ def _apply_categorical_transformer( def _apply_all_features_transformer(self) -> None: """Apply transformer blocks to all features.""" logger.info( - f"Adding transformer blocks to all features: #{self.transfo_nr_blocks}" + f"Adding transformer blocks to all features: #{self.transfo_nr_blocks}", ) for block_idx in range(self.transfo_nr_blocks): self.concat_all = PreprocessorLayerFactory.transformer_block_layer( @@ -2107,7 +2127,7 @@ def _prepare_dict_mode_outputs(self) -> None: self._apply_feature_moe_dict_mode() outputs = OrderedDict( - [(k, None) for k in self.inputs if k in self.processed_features] + [(k, None) for k in self.inputs if k in self.processed_features], ) outputs.update(OrderedDict(self.processed_features)) self.outputs = outputs @@ -2120,14 +2140,14 @@ def _apply_feature_moe_dict_mode(self) -> None: and updates the processed_features dictionary with the enhanced versions. """ logger.info( - f"Applying Feature-wise Mixture of Experts (dict mode) with {self.feature_moe_num_experts} experts" + f"Applying Feature-wise Mixture of Experts (dict mode) with {self.feature_moe_num_experts} experts", ) # Get feature names and corresponding processed features feature_names = [] individual_features = [] - for feature_name in self.inputs.keys(): + for feature_name in self.inputs: if feature_name in self.processed_features: feature_names.append(feature_name) individual_features.append(self.processed_features[feature_name]) @@ -2138,7 +2158,7 @@ def _apply_feature_moe_dict_mode(self) -> None: # Stack the features along a new axis stacked_features = StackFeaturesLayer(name="stacked_features_for_moe_dict")( - individual_features + individual_features, ) # Create the Feature MoE layer @@ -2165,7 +2185,7 @@ def _apply_feature_moe_dict_mode(self) -> None: for i, feature_name in enumerate(feature_names): feature_output = unstacked_outputs[i] # Add a projection layer for this feature - projection = tf.keras.layers.Dense( + projection = keras.layers.Dense( self.feature_moe_expert_dim, activation="relu", name=f"{feature_name}_moe_projection_dict", @@ -2176,16 +2196,15 @@ def _apply_feature_moe_dict_mode(self) -> None: logger.info("Feature MoE applied successfully in dict mode") - def _apply_feature_moe(self): - """ - Enhances the combined feature representation using Feature-wise Mixture of Experts (MoE) + def _apply_feature_moe(self) -> None: + """Enhances the combined feature representation using Feature-wise Mixture of Experts (MoE) in concatenated output mode. This method creates a Feature MoE layer that routes features to different experts based on their content, improving the overall representational power. """ logger.info( - f"Applying Feature-wise Mixture of Experts (concat mode) with {self.feature_moe_num_experts} experts" + f"Applying Feature-wise Mixture of Experts (concat mode) with {self.feature_moe_num_experts} experts", ) # Check if we have concatenated features to work with @@ -2199,9 +2218,7 @@ def _apply_feature_moe(self): output_dims = [] for feature_type in ["numeric", "categorical"]: if feature_type in self.processed_features_dims: - for feature_name, dims in self.processed_features_dims[ - feature_type - ].items(): + for dims in self.processed_features_dims[feature_type].values(): if dims is not None: output_dims.append(dims) @@ -2224,7 +2241,7 @@ def _apply_feature_moe(self): return # Set equal dimensions for all features if actual dimensions are not available - feature_dim = keras.backend.int_shape(self.concat_all)[-1] // total_features + feature_dim = int(self.concat_all.shape[-1]) // total_features output_dims = [feature_dim] * total_features # Store these calculated dimensions for future use @@ -2236,19 +2253,21 @@ def _apply_feature_moe(self): if hasattr(self, "numeric_features") and self.numeric_features: for feature_name in self.numeric_features: if hasattr(self, f"pipeline_{feature_name}") and hasattr( - getattr(self, f"pipeline_{feature_name}"), "output" + getattr(self, f"pipeline_{feature_name}"), + "output", ): feature_outputs.append( - getattr(self, f"pipeline_{feature_name}").output + getattr(self, f"pipeline_{feature_name}").output, ) if hasattr(self, "categorical_features") and self.categorical_features: for feature_name in self.categorical_features: if hasattr(self, f"pipeline_{feature_name}") and hasattr( - getattr(self, f"pipeline_{feature_name}"), "output" + getattr(self, f"pipeline_{feature_name}"), + "output", ): feature_outputs.append( - getattr(self, f"pipeline_{feature_name}").output + getattr(self, f"pipeline_{feature_name}").output, ) # If we couldn't get individual features, we'll split the concatenated tensor @@ -2265,7 +2284,7 @@ def _apply_feature_moe(self): # Stack the features for the MoE layer stacked_features = StackFeaturesLayer(name="stacked_features_for_moe")( - feature_outputs + feature_outputs, ) # Create and apply the Feature MoE layer @@ -2281,7 +2300,7 @@ def _apply_feature_moe(self): # Concatenate the processed features back together self.concat_all = keras.layers.Concatenate(axis=-1, name="concat_moe_features")( - unstacked_features + unstacked_features, ) @_monitor_performance @@ -2300,9 +2319,40 @@ def _cleanup_intermediate_tensors(self) -> None: gc.collect() # Clear backend session to free GPU memory if using GPU - tf.keras.backend.clear_session() + keras.backend.clear_session() @_monitor_performance + def _requires_dataset_statistics(self) -> bool: + """Report whether any configured feature needs a pass over the data. + + Numeric features need moments and text features need a vocabulary. + Categorical features need a vocabulary unless they are hashed with an + explicit bucket count -- hashing exists precisely so the data need not + be scanned. Date, passthrough and time series features derive + everything from their inputs. + + Returns: + True when the statistics pass is required. + """ + if self.numeric_features or self.text_features: + return True + + for feature_name in self.categorical_features: + feature = self.features_specs.get(feature_name) + if feature is None: + return True + if ( + getattr(feature, "category_encoding", None) + != CategoryEncodingOptions.HASHING + ): + return True + # Without an explicit size, the bucket count is derived from the + # vocabulary, which still requires the statistics pass. + if "hash_bucket_size" not in getattr(feature, "kwargs", {}): + return True + + return False + def build_preprocessor(self) -> dict: """Building preprocessing model. @@ -2321,18 +2371,39 @@ def build_preprocessor(self) -> dict: # Validate inputs if not self.features_specs: raise ValueError( - "No features specified. Please provide features_specs." + "No features specified. Please provide features_specs.", ) # preparing statistics if they do not exist if not self.features_stats or self.overwrite_stats: - logger.info("No input features_stats detected !") - if not hasattr(self, "stats_instance"): - raise ValueError( - "stats_instance not initialized. Cannot calculate features stats." + if not self._requires_dataset_statistics(): + # Hashing sizes its own buckets and dates parse themselves; + # a model built only from such features has nothing to learn + # from the data, so it should not demand a dataset. + logger.info( + "No feature requires dataset statistics; skipping the " + "statistics pass.", + ) + self.features_stats = self.features_stats or {} + else: + logger.info("No input features_stats detected !") + if self.path_data is None: + raise ValueError( + "Statistics are needed to build this preprocessor " + "(numeric, text, or vocabulary-based categorical " + "features are present) but neither `features_stats` " + "nor `path_data` was provided. Pass `path_data` " + "pointing at your CSV data, or supply precomputed " + "`features_stats`.", + ) + if not hasattr(self, "stats_instance"): + raise ValueError( + "stats_instance not initialized. Cannot calculate features stats.", + ) + self.features_stats = self.stats_instance.main() + logger.debug( + f"Features Stats were calculated: {self.features_stats}", ) - self.features_stats = self.stats_instance.main() - logger.debug(f"Features Stats were calculated: {self.features_stats}") # Set up inputs for all feature types BEFORE processing them for feature_name in ( @@ -2354,23 +2425,28 @@ def build_preprocessor(self) -> dict: feature_stats = None if feature_name in self.numeric_features: feature_stats = self.features_stats.get( - "numeric_stats", {} + "numeric_stats", + {}, ).get(feature_name, {}) elif feature_name in self.categorical_features: feature_stats = self.features_stats.get( - "categorical_stats", {} + "categorical_stats", + {}, ).get(feature_name, {}) elif feature_name in self.text_features: feature_stats = self.features_stats.get("text", {}).get( - feature_name, {} + feature_name, + {}, ) elif feature_name in self.date_features: feature_stats = self.features_stats.get("date", {}).get( - feature_name, {} + feature_name, + {}, ) elif feature_name in self.time_series_features: feature_stats = self.features_stats.get( - "time_series", {} + "time_series", + {}, ).get(feature_name, {}) if feature_stats: @@ -2379,19 +2455,19 @@ def build_preprocessor(self) -> dict: # Final fallback based on feature type if isinstance(feature, PassthroughFeature): dtype = getattr(feature, "dtype", tf.float32) - elif feature.feature_type in [ + elif feature.feature_type in ( FeatureType.STRING_CATEGORICAL, FeatureType.TEXT, - ]: - dtype = tf.string - elif feature.feature_type == FeatureType.DATE: + FeatureType.DATE, + ): dtype = tf.string else: dtype = tf.float32 self._add_input_column(feature_name=feature_name, dtype=dtype) self._add_input_signature( - feature_name=feature_name, dtype=dtype + feature_name=feature_name, + dtype=dtype, ) # Process features in batches by type @@ -2407,7 +2483,8 @@ def build_preprocessor(self) -> dict: categorical_stats = self.features_stats.get("categorical_stats", {}) text_stats = self.features_stats.get("text", {}) time_series_stats = self.features_stats.get( - "time_series", {} + "time_series", + {}, ) # Add time series stats for f_name in self.numeric_features: @@ -2456,9 +2533,9 @@ def build_preprocessor(self) -> dict: and not self.include_passthrough_in_output ): logger.info( - "Only passthrough features detected - creating passthrough-only model" + "Only passthrough features detected - creating passthrough-only model", ) - self.model = tf.keras.Model( + self.model = keras.Model( inputs=self.inputs, outputs=self.passthrough_outputs, name="preprocessor", @@ -2466,7 +2543,7 @@ def build_preprocessor(self) -> dict: _output_dims = "passthrough_only" elif self.concat_all is None: raise ValueError( - "No features were concatenated. Check if features were properly processed." + "No features were concatenated. Check if features were properly processed.", ) else: # Determine outputs based on passthrough settings @@ -2480,13 +2557,14 @@ def build_preprocessor(self) -> dict: "passthrough": self.passthrough_outputs, } logger.info( - f"Creating model with separate passthrough outputs: {list(self.passthrough_outputs.keys())}" + "Creating model with separate passthrough outputs: " + f"{list(self.passthrough_outputs.keys())}", ) else: # Standard concat output model_outputs = self.concat_all - self.model = tf.keras.Model( + self.model = keras.Model( inputs=self.inputs, outputs=model_outputs, name="preprocessor", @@ -2500,7 +2578,7 @@ def build_preprocessor(self) -> dict: else: # DICT mode if not self.outputs and not self.passthrough_outputs: raise ValueError( - "No outputs were created. Check if features were properly processed." + "No outputs were created. Check if features were properly processed.", ) # Include passthrough outputs in dict mode if they exist @@ -2508,16 +2586,16 @@ def build_preprocessor(self) -> dict: if self.passthrough_outputs and not self.include_passthrough_in_output: final_outputs.update(self.passthrough_outputs) logger.info( - f"Adding passthrough outputs to dict mode: {list(self.passthrough_outputs.keys())}" + f"Adding passthrough outputs to dict mode: {list(self.passthrough_outputs.keys())}", ) elif not final_outputs: # Only passthrough features exist final_outputs = self.passthrough_outputs logger.info( - "Only passthrough features detected - creating passthrough-only dict model" + "Only passthrough features detected - creating passthrough-only dict model", ) - self.model = tf.keras.Model( + self.model = keras.Model( inputs=self.inputs, outputs=final_outputs, name="preprocessor", @@ -2537,7 +2615,8 @@ def build_preprocessor(self) -> dict: "categorical": self.features_stats.get("categorical", {}), "text": self.features_stats.get("text", {}), "time_series": self.features_stats.get( - "time_series", {} + "time_series", + {}, ), # Add time series stats } @@ -2551,7 +2630,7 @@ def build_preprocessor(self) -> dict: "signature": self.signature, "output_dims": _output_dims, "feature_stats": feature_stats, - } + }, ) except Exception as e: @@ -2573,7 +2652,7 @@ def save_model(self, save_path: str) -> None: """ if not hasattr(self, "model") or self.model is None: raise ValueError( - "Model must be built before saving. Call build_preprocessor() first." + "Model must be built before saving. Call build_preprocessor() first.", ) # Create the directory if it doesn't exist @@ -2608,7 +2687,7 @@ def save_model(self, save_path: str) -> None: # Save metadata as JSON metadata_path = save_path / "metadata.json" - with open(metadata_path, "w") as f: + with metadata_path.open("w") as f: json.dump(metadata, f, indent=2, default=str) logger.info(f"Model metadata saved to {metadata_path}") @@ -2639,11 +2718,11 @@ def load_model(load_path: str) -> tuple: raise ValueError(f"Metadata file {metadata_path} does not exist") # Load the model - loaded_model = tf.keras.models.load_model(str(model_path)) + loaded_model = keras.saving.load_model(str(model_path)) logger.info(f"Model loaded from {model_path}") # Load metadata - with open(metadata_path, "r") as f: + with metadata_path.open() as f: metadata = json.load(f) logger.info(f"Model metadata loaded from {metadata_path}") @@ -2663,7 +2742,7 @@ def batch_predict(self, dataset: tf.data.Dataset) -> Generator: """ if not hasattr(self, "model") or self.model is None: raise ValueError( - "Model must be built before prediction. Call build_preprocessor() first." + "Model must be built before prediction. Call build_preprocessor() first.", ) # Process each batch of data @@ -2703,7 +2782,29 @@ def get_feature_importances(self) -> dict: return feature_importances - def _validate_time_series_inference_data(self, data): + @staticmethod + def _is_value_sequence(value) -> bool: + """Report whether a value carries several time steps rather than one. + + Tensors count: `TimeSeriesInferenceFormatter` hands back tensors when + asked to, and rejecting them meant KDP refused the output of its own + companion formatter. + + Args: + value: The value supplied for a time series feature. + + Returns: + True when the value holds a sequence of time steps. + """ + if isinstance(value, list | tuple | np.ndarray): + return True + if tf.is_tensor(value): + rank = value.shape.rank + return rank is None or rank >= 1 + # pandas Series and anything else sized, but not a bare string. + return hasattr(value, "__len__") and not isinstance(value, str | bytes) + + def _validate_time_series_inference_data(self, data) -> bool: """Validate that the provided data meets minimum requirements for time series inference. Args: @@ -2728,16 +2829,13 @@ def _validate_time_series_inference_data(self, data): if not time_series_features: return True - # Convert data to DataFrame if it's a dict + # Each time series feature needs a sequence of past values, not a scalar. if isinstance(data, dict): for key, value in data.items(): - if ( - not isinstance(value, (list, np.ndarray)) - and key in time_series_features - ): + if key in time_series_features and not self._is_value_sequence(value): raise ValueError( f"Time series feature '{key}' requires historical context. " - f"Please provide a list or array of values, not a single value." + f"Please provide a list or array of values, not a single value.", ) # For each time series feature, check that we have enough data @@ -2745,20 +2843,26 @@ def _validate_time_series_inference_data(self, data): feature = self.features_specs[feature_name] # Check grouping column exists if needed - if hasattr(feature, "group_by") and feature.group_by: - if isinstance(data, dict) and feature.group_by not in data: - raise ValueError( - f"Time series feature '{feature_name}' requires grouping by " - f"'{feature.group_by}', but this column is not in the data." - ) + if ( + getattr(feature, "group_by", None) + and isinstance(data, dict) + and feature.group_by not in data + ): + raise ValueError( + f"Time series feature '{feature_name}' requires grouping by " + f"'{feature.group_by}', but this column is not in the data.", + ) # Check sorting column exists if needed - if hasattr(feature, "sort_by") and feature.sort_by: - if isinstance(data, dict) and feature.sort_by not in data: - raise ValueError( - f"Time series feature '{feature_name}' requires sorting by " - f"'{feature.sort_by}', but this column is not in the data." - ) + if ( + getattr(feature, "sort_by", None) + and isinstance(data, dict) + and feature.sort_by not in data + ): + raise ValueError( + f"Time series feature '{feature_name}' requires sorting by " + f"'{feature.sort_by}', but this column is not in the data.", + ) # Calculate minimum required history min_history = 1 # Default minimum @@ -2802,17 +2906,17 @@ def _validate_time_series_inference_data(self, data): # Check data size if it's a dict with lists/arrays if isinstance(data, dict) and feature_name in data: feature_data = data[feature_name] - if isinstance(feature_data, (list, np.ndarray)): + if isinstance(feature_data, list | np.ndarray): data_length = len(feature_data) if data_length < min_history: raise ValueError( f"Time series feature '{feature_name}' requires at least {min_history} " - f"historical data points, but only {data_length} were provided." + f"historical data points, but only {data_length} were provided.", ) return True - def predict(self, data, **kwargs): + def predict(self, data, **kwargs) -> Any: """Predict using the preprocessor model. Args: @@ -2825,21 +2929,39 @@ def predict(self, data, **kwargs): # Validate time series inference data self._validate_time_series_inference_data(data) + # Keras cannot consume NumPy string arrays, which every categorical, + # text or date feature produces when the caller passes a plain dict. + if isinstance(data, dict): + data = _to_tensor_mapping(data) + # Call the model's predict method return self.model.predict(data, **kwargs) # Define serializable custom layers -@tf.keras.utils.register_keras_serializable(package="kdp.processor") +@keras.saving.register_keras_serializable(package="kdp.processor") class SplitLayer(keras.layers.Layer): """Custom layer to split a tensor into individual features based on dimensions.""" def __init__(self, feature_dims, **kwargs): + """Initialize the SplitLayer. + + See the class docstring for the accepted arguments and what + each one controls. + """ super().__init__(**kwargs) self.feature_dims = feature_dims - def call(self, inputs): + def call(self, inputs) -> list: # Handle case where feature_dims is None or empty + """Apply the layer to its inputs. + + Args: + inputs: Input tensor to process. + + Returns: + The transformed tensor. + """ if not self.feature_dims: # Return the input as a single feature if no dimensions are provided return [inputs] @@ -2852,12 +2974,12 @@ def call(self, inputs): start_indices.append(start_indices[-1] + dim) # Create [(start_idx, dim), ...] format - split_indices = list(zip(start_indices, self.feature_dims)) + split_indices = list(zip(start_indices, self.feature_dims, strict=False)) return [inputs[:, i : i + dim] for i, dim in split_indices] # Handle case where feature_dims is already a list of tuples (i, dim) if ( - isinstance(self.feature_dims[0], (list, tuple)) + isinstance(self.feature_dims[0], list | tuple) and len(self.feature_dims[0]) == 2 ): return [inputs[:, i : i + dim] for i, dim in self.feature_dims] @@ -2865,27 +2987,40 @@ def call(self, inputs): # If we get here, feature_dims is in an invalid format raise ValueError( f"Invalid feature_dims format: {self.feature_dims}. " - "Expected a list of integers or a list of (index, dimension) tuples." + "Expected a list of integers or a list of (index, dimension) tuples.", ) - def get_config(self): + def get_config(self) -> dict: + """Return the configuration needed to re-create this layer. + + Returns: + The layer configuration. + """ config = super().get_config() config.update({"feature_dims": self.feature_dims}) return config - def compute_output_shape(self, input_shape): + def compute_output_shape(self, input_shape) -> tuple: # Return a list of shapes for each split + """Compute the output shape for a given input shape. + + Args: + input_shape: Shape of the input tensor. + + Returns: + The corresponding output shape. + """ if not self.feature_dims: return [input_shape] elif isinstance(self.feature_dims[0], int): return [(input_shape[0], dim) for dim in self.feature_dims] elif ( - isinstance(self.feature_dims[0], (list, tuple)) + isinstance(self.feature_dims[0], list | tuple) and len(self.feature_dims[0]) == 2 ): return [(input_shape[0], dim) for _, dim in self.feature_dims] else: raise ValueError( f"Invalid feature_dims format: {self.feature_dims}. " - "Expected a list of integers or a list of (index, dimension) tuples." + "Expected a list of integers or a list of (index, dimension) tuples.", ) diff --git a/kdp/stats.py b/kdp/stats.py index 51c34e3..d36c7e7 100644 --- a/kdp/stats.py +++ b/kdp/stats.py @@ -1,4 +1,5 @@ import json +import math import os from concurrent.futures import ThreadPoolExecutor, as_completed from pathlib import Path @@ -10,10 +11,12 @@ from kdp.features import ( CategoricalFeature, + FeatureSpaceConverter, FeatureType, NumericalFeature, TimeSeriesFeature, ) +from kdp.layers.date_parsing_layer import DateParsingLayer MAX_WORKERS = os.cpu_count() or 4 @@ -100,18 +103,19 @@ def update(self, new_values: tf.Tensor) -> None: self.values.assign(updated_values) elif new_values.dtype == tf.int32: updated_values = tf.unique( - tf.concat([self.int_values, new_values], axis=0) + tf.concat([self.int_values, new_values], axis=0), )[0] self.int_values.assign(updated_values) else: raise ValueError( - f"Unsupported data type for categorical features: {new_values.dtype}" + f"Unsupported data type for categorical features: {new_values.dtype}", ) def get_unique_values(self) -> list: """Returns the unique categorical values accumulated so far.""" all_values = tf.concat( - [self.values, tf.strings.as_string(self.int_values)], axis=0 + [self.values, tf.strings.as_string(self.int_values)], + axis=0, ) return tf.unique(all_values)[0].numpy().tolist() @@ -144,7 +148,7 @@ def update(self, new_texts: tf.Tensor) -> None: """ if new_texts.dtype != tf.string: raise ValueError( - f"Unsupported data type for text features: {new_texts.dtype}" + f"Unsupported data type for text features: {new_texts.dtype}", ) # Split each string into words and flatten the list @@ -162,8 +166,11 @@ def get_unique_words(self) -> list: Returns: list of str: Unique words accumulated. """ - unique_words = self.words.value().numpy().tolist() - return unique_words + # `.numpy().tolist()` yields `bytes`; decode so the vocabulary is made of + # real `str` values. Keeping `bytes` here breaks Keras serialization of the + # TextVectorization layer and corrupts the vocabulary when the stats are + # round-tripped through JSON. + return [word.decode("utf-8") for word in self.words.value().numpy().tolist()] class DateAccumulator: @@ -178,19 +185,37 @@ def __init__(self): self.day_of_week_sin_accumulator = WelfordAccumulator() self.day_of_week_cos_accumulator = WelfordAccumulator() - @tf.function def update(self, dates: tf.Tensor) -> None: """Updates the accumulators with new date values. Args: - dates: A tensor of shape [batch_size, 3] where each row contains [year, month, day_of_week]. + dates: Either a tensor of date strings (``YYYY-MM-DD`` or ``YYYY/MM/DD``), + as read from a CSV column, or an already parsed numeric tensor of + shape ``[batch_size, >=4]`` whose columns are + ``[year, month, day_of_month, day_of_week]``. + + Raises: + ValueError: If a parsed numeric tensor does not carry the four + expected date components. """ - year = dates[:, 0] - month = dates[:, 1] - day_of_week = dates[:, 2] + if dates.dtype == tf.string: + parsed = DateParsingLayer()(tf.reshape(dates, [-1, 1])) + else: + parsed = tf.convert_to_tensor(dates) + if parsed.shape.rank != 2 or parsed.shape[-1] < 4: + raise ValueError( + "Parsed date tensors must have shape [batch_size, >=4] with " + "columns [year, month, day_of_month, day_of_week], got shape " + f"{parsed.shape}.", + ) + + parsed = tf.cast(parsed, tf.float32) + year = parsed[:, 0] + month = parsed[:, 1] + day_of_week = parsed[:, 3] # Cyclical encoding - pi = tf.math.pi + pi = tf.constant(math.pi, dtype=tf.float32) month_sin = tf.math.sin(2 * pi * month / 12) month_cos = tf.math.cos(2 * pi * month / 12) day_of_week_sin = tf.math.sin(2 * pi * day_of_week / 7) @@ -202,7 +227,6 @@ def update(self, dates: tf.Tensor) -> None: self.day_of_week_sin_accumulator.update(day_of_week_sin) self.day_of_week_cos_accumulator.update(day_of_week_cos) - @property def mean(self) -> dict: """Returns the mean statistics for date features.""" return { @@ -213,7 +237,6 @@ def mean(self) -> dict: "day_of_week_cos": self.day_of_week_cos_accumulator.mean.numpy(), } - @property def variance(self) -> dict: """Returns the variance statistics for date features.""" return { @@ -262,6 +285,27 @@ def __init__( self.date_features = date_features or [] self.time_series_features = time_series_features or [] self.features_specs = features_specs or {} + + # `features_specs` is documented as the easier alternative to listing the + # features by type, but nothing derived those lists from it: an instance + # built that way created no accumulators and `main()` returned {}, which + # is what made `auto_configure` produce empty recommendations. + if self.features_specs and not any( + ( + self.numeric_features, + self.categorical_features, + self.text_features, + self.date_features, + self.time_series_features, + ), + ): + converter = FeatureSpaceConverter() + self.features_specs = converter._init_features_specs(self.features_specs) + self.numeric_features = converter.numeric_features + self.categorical_features = converter.categorical_features + self.text_features = converter.text_features + self.date_features = converter.date_features + self.time_series_features = converter.time_series_features self.features_stats_path = features_stats_path or "features_stats.json" self.overwrite_stats = overwrite_stats self.batch_size = batch_size @@ -288,6 +332,18 @@ def _get_csv_file_pattern(self, path) -> str: str: File pattern that always has *.csv at the end """ + if path is None: + raise ValueError( + "`path_data` is required to compute statistics but was None. " + "Pass a path to a CSV file or to a directory of CSV files.", + ) + if not isinstance(path, str | os.PathLike): + raise TypeError( + "`path_data` must be a path to CSV data (a str or PathLike), " + f"got {type(path).__name__}. In-memory frames are not read " + "directly -- write the data to a CSV first and pass its path.", + ) + file_path = Path(path) # Check if the path is a directory if file_path.suffix: @@ -395,13 +451,13 @@ def _process_time_series_data(self) -> dict: if not has_feature: logger.warning( - f"Feature '{feature_name}' not found in the dataset. Skipping statistics calculation." + f"Feature '{feature_name}' not found in the dataset. Skipping statistics calculation.", ) continue # Prepare for grouped processing if grouping is specified if feature.group_by and feature.group_by in list( - dataset.element_spec.keys() + dataset.element_spec.keys(), ): # Process data by groups group_data = {} @@ -429,11 +485,11 @@ def _process_time_series_data(self) -> dict: if sort_keys is not None: group_data[group_key].append( - (sort_keys[i], feature_values[i]) + (sort_keys[i], feature_values[i]), ) else: group_data[group_key].append( - (i, feature_values[i]) + (i, feature_values[i]), ) # Use index as sort key # Create a separate accumulator for each group and process them @@ -443,7 +499,8 @@ def _process_time_series_data(self) -> dict: # Sort if sort_by is specified if feature.sort_by: pairs.sort( - key=lambda x: x[0], reverse=not feature.sort_ascending + key=lambda x: x[0], + reverse=not feature.sort_ascending, ) # Extract sorted values @@ -481,7 +538,7 @@ def _process_time_series_data(self) -> dict: "count": int( sum( acc.count.numpy() for acc in group_accumulators.values() - ) + ), ), "dtype": feature.dtype.name if hasattr(feature.dtype, "name") @@ -498,7 +555,7 @@ def _process_time_series_data(self) -> dict: accumulator = WelfordAccumulator() if feature.sort_by and feature.sort_by in list( - dataset.element_spec.keys() + dataset.element_spec.keys(), ): # Process in a streaming fashion to avoid memory issues # Create buffer for sorting that can be processed in chunks @@ -525,7 +582,8 @@ def _process_time_series_data(self) -> dict: # Extract values and update accumulator sorted_values = [pair[1] for pair in buffer] sorted_tensor = tf.constant( - sorted_values, dtype=tf.float32 + sorted_values, + dtype=tf.float32, ) accumulator.update(sorted_tensor) @@ -535,7 +593,8 @@ def _process_time_series_data(self) -> dict: # Process any remaining items in buffer if buffer: buffer.sort( - key=lambda x: x[0], reverse=not feature.sort_ascending + key=lambda x: x[0], + reverse=not feature.sort_ascending, ) sorted_values = [pair[1] for pair in buffer] sorted_tensor = tf.constant(sorted_values, dtype=tf.float32) @@ -610,7 +669,9 @@ def _process_batch_parallel(self, batch: tf.Tensor) -> None: raise def _compute_feature_stats_parallel( - self, feature_type: str, features: list[str] + self, + feature_type: str, + features: list[str], ) -> dict[str, Any]: """Compute statistics for a group of features in parallel. @@ -711,25 +772,29 @@ def _compute_final_statistics(self) -> dict[str, dict]: # Compute numeric statistics if self.numeric_features: stats["numeric_stats"] = self._compute_feature_stats_parallel( - "numeric", self.numeric_features + "numeric", + self.numeric_features, ) # Compute categorical statistics if self.categorical_features: stats["categorical_stats"] = self._compute_feature_stats_parallel( - "categorical", self.categorical_features + "categorical", + self.categorical_features, ) # Compute text statistics if self.text_features: stats["text"] = self._compute_feature_stats_parallel( - "text", self.text_features + "text", + self.text_features, ) # Compute date statistics if self.date_features: stats["date"] = self._compute_feature_stats_parallel( - "date", self.date_features + "date", + self.date_features, ) # Compute time series statistics @@ -767,7 +832,9 @@ def _custom_serializer(obj) -> Any: elif isinstance(obj, np.floating): return float(obj) # Convert numpy float to Python float elif isinstance(obj, bytes): - return str(obj) + # `str(b"foo")` would serialize the *repr* ("b'foo'") and silently + # corrupt vocabularies on reload, so decode explicitly. + return obj.decode("utf-8") elif isinstance(obj, np.ndarray): return obj.tolist() # Convert numpy arrays to lists logger.debug(f"Type {type(obj)} is not serializable") @@ -783,6 +850,36 @@ def _save_stats(self) -> None: json.dump(self.features_stats, f, default=self._custom_serializer) logger.info("features_stats saved ") + @staticmethod + def _has_repr_encoded_vocabulary(features_stats: dict) -> bool: + """Detect vocabularies saved as Python byte reprs by older releases. + + Args: + features_stats: Statistics loaded from disk. + + Returns: + True when any vocabulary entry looks like ``b'value'`` rather than + the value itself. + """ + for stats_group in features_stats.values(): + if not isinstance(stats_group, dict): + continue + for feature_stats in stats_group.values(): + if not isinstance(feature_stats, dict): + continue + vocabulary = feature_stats.get("vocab") + if not isinstance(vocabulary, list): + continue + for entry in vocabulary: + if ( + isinstance(entry, str) + and len(entry) >= 3 + and entry.startswith(("b'", 'b"')) + and entry[-1] == entry[1] + ): + return True + return False + def _load_stats(self) -> dict: """Loads serialized features stats from a file, with custom handling for TensorFlow dtypes. @@ -796,11 +893,27 @@ def _load_stats(self) -> dict: stats_path = Path(self.features_stats_path) if stats_path.is_file(): logger.info( - f"Found columns statistics, loading as features_stats: {self.features_stats_path}" + f"Found columns statistics, loading as features_stats: {self.features_stats_path}", ) with stats_path.open() as f: self.features_stats = json.load(f) + # Statistics written before the bytes-decoding fix hold repr strings + # ("b'paris'") instead of the real categories. Such a file still + # loads and still builds a model, but every category then misses the + # vocabulary and encodes identically to an unseen value -- silent, + # total signal loss. Recompute instead of trusting it. + if self._has_repr_encoded_vocabulary(self.features_stats): + logger.warning( + f"{self.features_stats_path} was written by an older KDP " + "release whose vocabularies were stored as byte reprs " + "(\"b'value'\"). Reusing it would silently map every " + "category to the out-of-vocabulary slot, so the statistics " + "are being recomputed from the data.", + ) + self.features_stats = {} + return self.features_stats + # Convert dtype strings back to TensorFlow dtype objects for stats_type in ( self.features_stats.values() @@ -808,7 +921,7 @@ def _load_stats(self) -> dict: for _, feature_stats in stats_type.items(): if "dtype" in feature_stats: feature_stats["dtype"] = tf.dtypes.as_dtype( - feature_stats["dtype"] + feature_stats["dtype"], ) logger.info("features_stats loaded ") else: @@ -828,8 +941,7 @@ def main(self) -> dict: return stats def recommend_model_configuration(self) -> dict: - """ - Analyze the computed dataset statistics and provide recommendations for optimal preprocessing. + """Analyze the computed dataset statistics and provide recommendations for optimal preprocessing. This method leverages the ModelAdvisor to analyze feature characteristics and suggest the best preprocessing strategies, layer configurations, and model parameters. @@ -850,10 +962,10 @@ def recommend_model_configuration(self) -> dict: recommendations = recommend_model_configuration(self.features_stats) logger.info( - "Generated model configuration recommendations based on dataset statistics" + "Generated model configuration recommendations based on dataset statistics", ) logger.info( - f"Recommended configuration for {len(recommendations.get('features', {}))} features" + f"Recommended configuration for {len(recommendations.get('features', {}))} features", ) return recommendations diff --git a/kdp/time_series/inference.py b/kdp/time_series/inference.py index e34694e..bf98c88 100644 --- a/kdp/time_series/inference.py +++ b/kdp/time_series/inference.py @@ -1,7 +1,7 @@ import pandas as pd import numpy as np import tensorflow as tf -from typing import Dict, List, Union, Optional, Any +from typing import Any from kdp.features import FeatureType, TimeSeriesFeature from kdp.inference.base import InferenceFormatter @@ -40,7 +40,7 @@ def is_time_series_preprocessor(self) -> bool: """ return len(self.time_series_features) > 0 - def _identify_time_series_features(self) -> Dict[str, TimeSeriesFeature]: + def _identify_time_series_features(self) -> dict[str, TimeSeriesFeature]: """Identify all time series features in the preprocessor. Returns: @@ -57,7 +57,7 @@ def _identify_time_series_features(self) -> Dict[str, TimeSeriesFeature]: return time_series_features - def _calculate_min_history_requirements(self) -> Dict[str, Dict[str, Any]]: + def _calculate_min_history_requirements(self) -> dict[str, dict[str, Any]]: """Calculate minimum history requirements for each time series feature. Returns: @@ -79,7 +79,8 @@ def _calculate_min_history_requirements(self) -> Dict[str, Dict[str, Any]]: lags = feature.lag_config.get("lags", []) if lags: feature_req["min_history"] = max( - feature_req["min_history"], max(lags) + feature_req["min_history"], + max(lags), ) # Check rolling statistics @@ -89,7 +90,8 @@ def _calculate_min_history_requirements(self) -> Dict[str, Dict[str, Any]]: ): window_size = feature.rolling_stats_config.get("window_size", 1) feature_req["min_history"] = max( - feature_req["min_history"], window_size + feature_req["min_history"], + window_size, ) # Check differencing @@ -105,7 +107,8 @@ def _calculate_min_history_requirements(self) -> Dict[str, Dict[str, Any]]: periods = feature.moving_average_config.get("periods", []) if periods: feature_req["min_history"] = max( - feature_req["min_history"], max(periods) + feature_req["min_history"], + max(periods), ) # Check wavelet transform @@ -122,17 +125,22 @@ def _calculate_min_history_requirements(self) -> Dict[str, Dict[str, Any]]: def prepare_inference_data( self, - data: Union[Dict, pd.DataFrame], - historical_data: Optional[Union[Dict, pd.DataFrame]] = None, - fill_missing: bool = True, + data: dict | pd.DataFrame, + historical_data: dict | pd.DataFrame | None = None, + fill_missing: bool = True, # noqa: ARG002 - inert, see docstring to_tensors: bool = False, - ) -> Union[Dict, Dict[str, tf.Tensor]]: + ) -> dict | dict[str, tf.Tensor]: """Prepare time series data for inference based on preprocessor requirements. Args: data: The new data to make predictions on historical_data: Optional historical data to provide context for time series - fill_missing: Whether to attempt to fill missing values/context + fill_missing: Accepted for backwards compatibility and currently + inert. This formatter never fabricates history: when the data is + too short for the configured lookback it raises so the caller can + supply real context. Missing *values* inside otherwise sufficient + history are handled in-graph by `MissingValueHandlerLayer`, via + the feature's `missing_value_config`. to_tensors: Whether to convert the output to TensorFlow tensors Returns: @@ -153,7 +161,8 @@ def prepare_inference_data( historical_dict = self._convert_to_dict(historical_data) # Combine historical and new data combined_data = self._combine_historical_and_new( - historical_dict, inference_data + historical_dict, + inference_data, ) else: # Check if inference data itself has enough history @@ -172,7 +181,7 @@ def prepare_inference_data( return formatted_data - def _check_inference_data_sufficiency(self, data: Dict) -> None: + def _check_inference_data_sufficiency(self, data: dict) -> None: """Check if inference data itself has enough history for each feature. Args: @@ -184,7 +193,7 @@ def _check_inference_data_sufficiency(self, data: Dict) -> None: for feature_name, requirements in self.min_history_requirements.items(): if feature_name not in data: raise ValueError( - f"Time series feature '{feature_name}' is missing from input data" + f"Time series feature '{feature_name}' is missing from input data", ) # Check that data length is sufficient @@ -193,10 +202,10 @@ def _check_inference_data_sufficiency(self, data: Dict) -> None: raise ValueError( f"Time series feature '{feature_name}' requires at least " f"{requirements['min_history']} data points, but only " - f"{data_length} were provided. Please provide historical data." + f"{data_length} were provided. Please provide historical data.", ) - def _combine_historical_and_new(self, historical: Dict, new_data: Dict) -> Dict: + def _combine_historical_and_new(self, historical: dict, new_data: dict) -> dict: """Combine historical and new data for time series features. Args: @@ -232,7 +241,7 @@ def _combine_historical_and_new(self, historical: Dict, new_data: Dict) -> Dict: return combined - def _sort_by_time_and_group(self, data: Dict) -> Dict: + def _sort_by_time_and_group(self, data: dict) -> dict: """Sort time series data by time and group. Args: @@ -246,7 +255,7 @@ def _sort_by_time_and_group(self, data: Dict) -> Dict: sort_columns = set() group_columns = set() - for feature_name, requirements in self.min_history_requirements.items(): + for requirements in self.min_history_requirements.values(): if requirements["sort_by"]: needs_sorting = True sort_columns.add(requirements["sort_by"]) @@ -318,13 +327,13 @@ def describe_requirements(self) -> str: for feature_name, reqs in self.min_history_requirements.items(): feature_req = [f" - {feature_name}:"] feature_req.append( - f" * Minimum history: {reqs['min_history']} data points" + f" * Minimum history: {reqs['min_history']} data points", ) if reqs["sort_by"]: feature_req.append( f" * Must be sorted by: {reqs['sort_by']} " - + f"({'ascending' if reqs['sort_ascending'] else 'descending'})" + f"({'ascending' if reqs['sort_ascending'] else 'descending'})", ) if reqs["group_by"]: @@ -335,8 +344,11 @@ def describe_requirements(self) -> str: return "\n".join(requirements) def format_for_incremental_prediction( - self, current_history: Dict, new_row: Dict, to_tensors: bool = False - ) -> Union[Dict, Dict[str, tf.Tensor]]: + self, + current_history: dict, + new_row: dict, + to_tensors: bool = False, + ) -> dict | dict[str, tf.Tensor]: """Format data for incremental time series prediction. This is useful for forecasting scenarios where each new prediction @@ -356,40 +368,66 @@ def format_for_incremental_prediction( # Combine and prepare the data return self.prepare_inference_data( - new_dict, history_dict, to_tensors=to_tensors + new_dict, + history_dict, + to_tensors=to_tensors, ) def generate_multi_step_forecast( self, - history: Dict, - future_dates: List, - group_id: Optional[str] = None, - steps: int = 1, + history: dict, + future_dates: list, + group_id: str | None = None, + steps: int | None = None, ) -> pd.DataFrame: - """Generate data frames for multi-step forecasting. + """Generate a placeholder frame for multi-step forecasting. - This method prepares a sequence of data frames for multi-step forecasting - where each prediction becomes part of the history for the next step. + The returned frame carries one row per forecast step, with the sort + column filled from ``future_dates`` and every time series feature set to + NaN. Callers fill each row in turn with their model's prediction, so the + row becomes part of the history for the following step. Args: - history: Historical data dictionary or DataFrame - future_dates: List of dates for future predictions - group_id: Optional group identifier (e.g., store_id) if using grouped time series - steps: Number of steps to forecast + history: Historical data dictionary or DataFrame. It is validated + against the minimum history each configured feature needs. + future_dates: List of dates for future predictions. + group_id: Optional group identifier (e.g. store_id) if using grouped + time series. + steps: Number of steps to forecast. Defaults to every date in + ``future_dates``. Returns: - DataFrame with placeholder rows for each future step + DataFrame with placeholder rows for each future step. + + Raises: + ValueError: If the preprocessor has no time series features, if the + feature has no sort column, if ``steps`` asks for more rows than + ``future_dates`` provides, or if ``history`` is too short for the + configured lookback. """ if not self.time_series_features: raise ValueError("No time series features found in the preprocessor") + future_dates = list(future_dates) + if steps is not None: + if steps > len(future_dates): + raise ValueError( + f"Requested {steps} forecast steps but only " + f"{len(future_dates)} future date(s) were supplied.", + ) + future_dates = future_dates[:steps] + + # The history has to satisfy the same lookback the preprocessor needs, + # otherwise the frame built here cannot be fed back through it. + self._check_history_requirements(self._convert_to_dict(history)) + # Get the first time series feature to determine sort and group columns feature_name = next(iter(self.time_series_features)) requirements = self.min_history_requirements[feature_name] if not requirements["sort_by"]: raise ValueError( - f"Time series feature '{feature_name}' has no sort_by column specified" + f"Time series feature '{feature_name}' has no sort_by column specified", ) # Create a DataFrame of future dates diff --git a/mkdocs.yml b/mkdocs.yml index 958ba59..bb49ca2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -1,6 +1,10 @@ site_name: KDP - Keras Data Preprocessor site_url: https://unicolab.github.io/keras-data-processor docs_dir: docs +# Contributor-facing notes about the docs tree itself, not a site page. +# mkdocs would otherwise treat it as a second index. +exclude_docs: | + README.md theme: name: "material" icon: @@ -43,6 +47,7 @@ nav: - 🎯 Quick Start: getting-started/quick-start.md - 🌍 Why KDP?: getting-started/motivation.md - 🧩 Architecture Overview: getting-started/architecture.md + - ⬆️ Upgrading from 1.11.x: getting-started/upgrading.md - 📚 Feature Types: - 🛠️ Features Overview: features/overview.md - 🔢 Numerical Features: features/numerical-features.md @@ -50,6 +55,7 @@ nav: - 📝 Text Features: features/text-features.md - 📅 Date Features: features/date-features.md - 📊 Time Series Features: features/time_series_features.md + - 🔮 Time Series Inference: time_series_inference.md - ➕ Cross Features: features/cross-features.md - 🔄 Passthrough Features: features/passthrough-features.md - 🧠 Advanced Features: @@ -166,7 +172,7 @@ extra: - icon: fontawesome/brands/github link: https://github.com/UnicoLab/keras-data-processor - icon: fontawesome/brands/python - link: https://pypi.org/project/keras-data-processor/ + link: https://pypi.org/project/kdp/ extra_javascript: - assets/js/fix-image-paths.js diff --git a/poetry.lock b/poetry.lock index cfec430..a1ceda2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 2.3.3 and should not be changed by hand. [[package]] name = "absl-py" @@ -6,6 +6,7 @@ version = "2.3.1" description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "absl_py-2.3.1-py3-none-any.whl", hash = "sha256:eeecf07f0c2a93ace0772c92e596ace6d3d3996c042b2128459aaae2a76de11d"}, {file = "absl_py-2.3.1.tar.gz", hash = "sha256:a97820526f7fbfd2ec1bce83f3f25e3a14840dac0d8e02a0b71cd75db3f77fc9"}, @@ -17,6 +18,8 @@ version = "0.1.4" description = "Disable App Nap on macOS >= 10.9" optional = false python-versions = ">=3.6" +groups = ["dev"] +markers = "platform_system == \"Darwin\"" files = [ {file = "appnope-0.1.4-py2.py3-none-any.whl", hash = "sha256:502575ee11cd7a28c0205f379b525beefebab9d161b7c964670864014ed7213c"}, {file = "appnope-0.1.4.tar.gz", hash = "sha256:1de3860566df9caf38f01f86f65e0e13e379af54f9e4bee1e66b48f2efffd1ee"}, @@ -28,6 +31,7 @@ version = "3.0.0" description = "Annotate AST trees with source code positions" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "asttokens-3.0.0-py3-none-any.whl", hash = "sha256:e3078351a059199dd5138cb1c706e6430c05eff2ff136af5eb4790f9d28932e2"}, {file = "asttokens-3.0.0.tar.gz", hash = "sha256:0dcd8baa8d62b0c1d118b399b2ddba3c4aff271d0d7a9e0d4c1681c79035bbc7"}, @@ -43,6 +47,7 @@ version = "1.6.3" description = "An AST unparser for Python" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "astunparse-1.6.3-py2.py3-none-any.whl", hash = "sha256:c2652417f2c8b5bb325c885ae329bdf3f86424075c4fd1a128674bc6fba4b8e8"}, {file = "astunparse-1.6.3.tar.gz", hash = "sha256:5ad93a8456f0d084c3456d059fd9a92cce667963232cbf763eac3bc5b7940872"}, @@ -58,13 +63,14 @@ version = "2.17.0" description = "Internationalization utilities" optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "babel-2.17.0-py3-none-any.whl", hash = "sha256:4d0b53093fdfb4b21c92b5213dba5a1b23885afa8383709427046b21c366e5f2"}, {file = "babel-2.17.0.tar.gz", hash = "sha256:0c54cffb19f690cdcc52a3b50bcbf71e07a808d1c80d549f2459b9d2cf0afb9d"}, ] [package.extras] -dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] +dev = ["backports.zoneinfo ; python_version < \"3.9\"", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata ; sys_platform == \"win32\""] [[package]] name = "backrefs" @@ -72,6 +78,7 @@ version = "5.9" description = "A wrapper around re and regex that adds additional back references." optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "backrefs-5.9-py310-none-any.whl", hash = "sha256:db8e8ba0e9de81fcd635f440deab5ae5f2591b54ac1ebe0550a2ca063488cd9f"}, {file = "backrefs-5.9-py311-none-any.whl", hash = "sha256:6907635edebbe9b2dc3de3a2befff44d74f30a4562adbb8b36f21252ea19c5cf"}, @@ -91,6 +98,7 @@ version = "2025.7.14" description = "Python package for providing Mozilla's CA Bundle." optional = false python-versions = ">=3.7" +groups = ["main", "doc"] files = [ {file = "certifi-2025.7.14-py3-none-any.whl", hash = "sha256:6b31f564a415d79ee77df69d757bb49a5bb53bd9f756cbbe24394ffd6fc1f4b2"}, {file = "certifi-2025.7.14.tar.gz", hash = "sha256:8ea99dbdfaaf2ba2f9bac77b9249ef62ec5218e7c2b2e903378ed5fccf765995"}, @@ -102,6 +110,8 @@ version = "1.17.1" description = "Foreign Function Interface for Python calling C code." optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "implementation_name == \"pypy\"" files = [ {file = "cffi-1.17.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:df8b1c11f177bc2313ec4b2d46baec87a5f3e71fc8b45dab2ee7cae86d9aba14"}, {file = "cffi-1.17.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:8f2cdc858323644ab277e9bb925ad72ae0e67f69e804f4898c070998d50b1a67"}, @@ -181,6 +191,7 @@ version = "3.4.0" description = "Validate configuration and produce human readable error messages." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "cfgv-3.4.0-py2.py3-none-any.whl", hash = "sha256:b7265b1f29fd3316bfcd2b330d63d024f2bfd8bcb8b0272f8e19a504856c48f9"}, {file = "cfgv-3.4.0.tar.gz", hash = "sha256:e52591d4c5f5dead8e0f673fb16db7949d2cfb3f7da4582893288f0ded8fe560"}, @@ -192,6 +203,7 @@ version = "3.4.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." optional = false python-versions = ">=3.7" +groups = ["main", "doc"] files = [ {file = "charset_normalizer-3.4.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7c48ed483eb946e6c04ccbe02c6b4d1d48e51944b6db70f697e089c193404941"}, {file = "charset_normalizer-3.4.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b2d318c11350e10662026ad0eb71bb51c7812fc8590825304ae0bdd4ac283acd"}, @@ -293,6 +305,7 @@ version = "8.1.8" description = "Composable command line interface toolkit" optional = false python-versions = ">=3.7" +groups = ["doc"] files = [ {file = "click-8.1.8-py3-none-any.whl", hash = "sha256:63c132bbbed01578a06712a2d1f497bb62d9c1c0d329b7903a866228027263b2"}, {file = "click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a"}, @@ -307,10 +320,12 @@ version = "0.4.6" description = "Cross-platform colored terminal text." optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["main", "dev", "doc"] files = [ {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +markers = {main = "sys_platform == \"win32\"", dev = "sys_platform == \"win32\""} [[package]] name = "comm" @@ -318,6 +333,7 @@ version = "0.2.3" description = "Jupyter Python Comm implementation, for usage in ipykernel, xeus-python etc." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "comm-0.2.3-py3-none-any.whl", hash = "sha256:c615d91d75f7f04f095b30d1c1711babd43bdc6419c1be9886a85f2f4e489417"}, {file = "comm-0.2.3.tar.gz", hash = "sha256:2dc8048c10962d55d7ad693be1e7045d891b7ce8d999c97963a5e3e99c055971"}, @@ -326,12 +342,184 @@ files = [ [package.extras] test = ["pytest"] +[[package]] +name = "contourpy" +version = "1.3.2" +description = "Python library for calculating contours of 2D quadrilateral grids" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +markers = "python_version == \"3.10\"" +files = [ + {file = "contourpy-1.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:ba38e3f9f330af820c4b27ceb4b9c7feee5fe0493ea53a8720f4792667465934"}, + {file = "contourpy-1.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:dc41ba0714aa2968d1f8674ec97504a8f7e334f48eeacebcaa6256213acb0989"}, + {file = "contourpy-1.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9be002b31c558d1ddf1b9b415b162c603405414bacd6932d031c5b5a8b757f0d"}, + {file = "contourpy-1.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8d2e74acbcba3bfdb6d9d8384cdc4f9260cae86ed9beee8bd5f54fee49a430b9"}, + {file = "contourpy-1.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e259bced5549ac64410162adc973c5e2fb77f04df4a439d00b478e57a0e65512"}, + {file = "contourpy-1.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad687a04bc802cbe8b9c399c07162a3c35e227e2daccf1668eb1f278cb698631"}, + {file = "contourpy-1.3.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cdd22595308f53ef2f891040ab2b93d79192513ffccbd7fe19be7aa773a5e09f"}, + {file = "contourpy-1.3.2-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b4f54d6a2defe9f257327b0f243612dd051cc43825587520b1bf74a31e2f6ef2"}, + {file = "contourpy-1.3.2-cp310-cp310-win32.whl", hash = "sha256:f939a054192ddc596e031e50bb13b657ce318cf13d264f095ce9db7dc6ae81c0"}, + {file = "contourpy-1.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:c440093bbc8fc21c637c03bafcbef95ccd963bc6e0514ad887932c18ca2a759a"}, + {file = "contourpy-1.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6a37a2fb93d4df3fc4c0e363ea4d16f83195fc09c891bc8ce072b9d084853445"}, + {file = "contourpy-1.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b7cd50c38f500bbcc9b6a46643a40e0913673f869315d8e70de0438817cb7773"}, + {file = "contourpy-1.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6658ccc7251a4433eebd89ed2672c2ed96fba367fd25ca9512aa92a4b46c4f1"}, + {file = "contourpy-1.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:70771a461aaeb335df14deb6c97439973d253ae70660ca085eec25241137ef43"}, + {file = "contourpy-1.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65a887a6e8c4cd0897507d814b14c54a8c2e2aa4ac9f7686292f9769fcf9a6ab"}, + {file = "contourpy-1.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3859783aefa2b8355697f16642695a5b9792e7a46ab86da1118a4a23a51a33d7"}, + {file = "contourpy-1.3.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:eab0f6db315fa4d70f1d8ab514e527f0366ec021ff853d7ed6a2d33605cf4b83"}, + {file = "contourpy-1.3.2-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:d91a3ccc7fea94ca0acab82ceb77f396d50a1f67412efe4c526f5d20264e6ecd"}, + {file = "contourpy-1.3.2-cp311-cp311-win32.whl", hash = "sha256:1c48188778d4d2f3d48e4643fb15d8608b1d01e4b4d6b0548d9b336c28fc9b6f"}, + {file = "contourpy-1.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:5ebac872ba09cb8f2131c46b8739a7ff71de28a24c869bcad554477eb089a878"}, + {file = "contourpy-1.3.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:4caf2bcd2969402bf77edc4cb6034c7dd7c0803213b3523f111eb7460a51b8d2"}, + {file = "contourpy-1.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:82199cb78276249796419fe36b7386bd8d2cc3f28b3bc19fe2454fe2e26c4c15"}, + {file = "contourpy-1.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:106fab697af11456fcba3e352ad50effe493a90f893fca6c2ca5c033820cea92"}, + {file = "contourpy-1.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d14f12932a8d620e307f715857107b1d1845cc44fdb5da2bc8e850f5ceba9f87"}, + {file = "contourpy-1.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:532fd26e715560721bb0d5fc7610fce279b3699b018600ab999d1be895b09415"}, + {file = "contourpy-1.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b383144cf2d2c29f01a1e8170f50dacf0eac02d64139dcd709a8ac4eb3cfe"}, + {file = "contourpy-1.3.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:c49f73e61f1f774650a55d221803b101d966ca0c5a2d6d5e4320ec3997489441"}, + {file = "contourpy-1.3.2-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:3d80b2c0300583228ac98d0a927a1ba6a2ba6b8a742463c564f1d419ee5b211e"}, + {file = "contourpy-1.3.2-cp312-cp312-win32.whl", hash = "sha256:90df94c89a91b7362e1142cbee7568f86514412ab8a2c0d0fca72d7e91b62912"}, + {file = "contourpy-1.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:8c942a01d9163e2e5cfb05cb66110121b8d07ad438a17f9e766317bcb62abf73"}, + {file = "contourpy-1.3.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:de39db2604ae755316cb5967728f4bea92685884b1e767b7c24e983ef5f771cb"}, + {file = "contourpy-1.3.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:3f9e896f447c5c8618f1edb2bafa9a4030f22a575ec418ad70611450720b5b08"}, + {file = "contourpy-1.3.2-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:71e2bd4a1c4188f5c2b8d274da78faab884b59df20df63c34f74aa1813c4427c"}, + {file = "contourpy-1.3.2-cp313-cp313-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:de425af81b6cea33101ae95ece1f696af39446db9682a0b56daaa48cfc29f38f"}, + {file = "contourpy-1.3.2-cp313-cp313-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:977e98a0e0480d3fe292246417239d2d45435904afd6d7332d8455981c408b85"}, + {file = "contourpy-1.3.2-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:434f0adf84911c924519d2b08fc10491dd282b20bdd3fa8f60fd816ea0b48841"}, + {file = "contourpy-1.3.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c66c4906cdbc50e9cba65978823e6e00b45682eb09adbb78c9775b74eb222422"}, + {file = "contourpy-1.3.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:8b7fc0cd78ba2f4695fd0a6ad81a19e7e3ab825c31b577f384aa9d7817dc3bef"}, + {file = "contourpy-1.3.2-cp313-cp313-win32.whl", hash = "sha256:15ce6ab60957ca74cff444fe66d9045c1fd3e92c8936894ebd1f3eef2fff075f"}, + {file = "contourpy-1.3.2-cp313-cp313-win_amd64.whl", hash = "sha256:e1578f7eafce927b168752ed7e22646dad6cd9bca673c60bff55889fa236ebf9"}, + {file = "contourpy-1.3.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0475b1f6604896bc7c53bb070e355e9321e1bc0d381735421a2d2068ec56531f"}, + {file = "contourpy-1.3.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:c85bb486e9be652314bb5b9e2e3b0d1b2e643d5eec4992c0fbe8ac71775da739"}, + {file = "contourpy-1.3.2-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:745b57db7758f3ffc05a10254edd3182a2a83402a89c00957a8e8a22f5582823"}, + {file = "contourpy-1.3.2-cp313-cp313t-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:970e9173dbd7eba9b4e01aab19215a48ee5dd3f43cef736eebde064a171f89a5"}, + {file = "contourpy-1.3.2-cp313-cp313t-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c6c4639a9c22230276b7bffb6a850dfc8258a2521305e1faefe804d006b2e532"}, + {file = "contourpy-1.3.2-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cc829960f34ba36aad4302e78eabf3ef16a3a100863f0d4eeddf30e8a485a03b"}, + {file = "contourpy-1.3.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:d32530b534e986374fc19eaa77fcb87e8a99e5431499949b828312bdcd20ac52"}, + {file = "contourpy-1.3.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:e298e7e70cf4eb179cc1077be1c725b5fd131ebc81181bf0c03525c8abc297fd"}, + {file = "contourpy-1.3.2-cp313-cp313t-win32.whl", hash = "sha256:d0e589ae0d55204991450bb5c23f571c64fe43adaa53f93fc902a84c96f52fe1"}, + {file = "contourpy-1.3.2-cp313-cp313t-win_amd64.whl", hash = "sha256:78e9253c3de756b3f6a5174d024c4835acd59eb3f8e2ca13e775dbffe1558f69"}, + {file = "contourpy-1.3.2-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:fd93cc7f3139b6dd7aab2f26a90dde0aa9fc264dbf70f6740d498a70b860b82c"}, + {file = "contourpy-1.3.2-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:107ba8a6a7eec58bb475329e6d3b95deba9440667c4d62b9b6063942b61d7f16"}, + {file = "contourpy-1.3.2-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:ded1706ed0c1049224531b81128efbd5084598f18d8a2d9efae833edbd2b40ad"}, + {file = "contourpy-1.3.2-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:5f5964cdad279256c084b69c3f412b7801e15356b16efa9d78aa974041903da0"}, + {file = "contourpy-1.3.2-pp311-pypy311_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:49b65a95d642d4efa8f64ba12558fcb83407e58a2dfba9d796d77b63ccfcaff5"}, + {file = "contourpy-1.3.2-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:8c5acb8dddb0752bf252e01a3035b21443158910ac16a3b0d20e7fed7d534ce5"}, + {file = "contourpy-1.3.2.tar.gz", hash = "sha256:b6945942715a034c671b7fc54f9588126b0b8bf23db2696e3ca8328f3ff0ab54"}, +] + +[package.dependencies] +numpy = ">=1.23" + +[package.extras] +bokeh = ["bokeh", "selenium"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["bokeh", "contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.15.0)", "types-Pillow"] +test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] + +[[package]] +name = "contourpy" +version = "1.3.3" +description = "Python library for calculating contours of 2D quadrilateral grids" +optional = false +python-versions = ">=3.11" +groups = ["dev"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "contourpy-1.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:709a48ef9a690e1343202916450bc48b9e51c049b089c7f79a267b46cffcdaa1"}, + {file = "contourpy-1.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:23416f38bfd74d5d28ab8429cc4d63fa67d5068bd711a85edb1c3fb0c3e2f381"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:929ddf8c4c7f348e4c0a5a3a714b5c8542ffaa8c22954862a46ca1813b667ee7"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:9e999574eddae35f1312c2b4b717b7885d4edd6cb46700e04f7f02db454e67c1"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0bf67e0e3f482cb69779dd3061b534eb35ac9b17f163d851e2a547d56dba0a3a"}, + {file = "contourpy-1.3.3-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:51e79c1f7470158e838808d4a996fa9bac72c498e93d8ebe5119bc1e6becb0db"}, + {file = "contourpy-1.3.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:598c3aaece21c503615fd59c92a3598b428b2f01bfb4b8ca9c4edeecc2438620"}, + {file = "contourpy-1.3.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:322ab1c99b008dad206d406bb61d014cf0174df491ae9d9d0fac6a6fda4f977f"}, + {file = "contourpy-1.3.3-cp311-cp311-win32.whl", hash = "sha256:fd907ae12cd483cd83e414b12941c632a969171bf90fc937d0c9f268a31cafff"}, + {file = "contourpy-1.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:3519428f6be58431c56581f1694ba8e50626f2dd550af225f82fb5f5814d2a42"}, + {file = "contourpy-1.3.3-cp311-cp311-win_arm64.whl", hash = "sha256:15ff10bfada4bf92ec8b31c62bf7c1834c244019b4a33095a68000d7075df470"}, + {file = "contourpy-1.3.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:b08a32ea2f8e42cf1d4be3169a98dd4be32bafe4f22b6c4cb4ba810fa9e5d2cb"}, + {file = "contourpy-1.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:556dba8fb6f5d8742f2923fe9457dbdd51e1049c4a43fd3986a0b14a1d815fc6"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:92d9abc807cf7d0e047b95ca5d957cf4792fcd04e920ca70d48add15c1a90ea7"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b2e8faa0ed68cb29af51edd8e24798bb661eac3bd9f65420c1887b6ca89987c8"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:626d60935cf668e70a5ce6ff184fd713e9683fb458898e4249b63be9e28286ea"}, + {file = "contourpy-1.3.3-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d00e655fcef08aba35ec9610536bfe90267d7ab5ba944f7032549c55a146da1"}, + {file = "contourpy-1.3.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:451e71b5a7d597379ef572de31eeb909a87246974d960049a9848c3bc6c41bf7"}, + {file = "contourpy-1.3.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:459c1f020cd59fcfe6650180678a9993932d80d44ccde1fa1868977438f0b411"}, + {file = "contourpy-1.3.3-cp312-cp312-win32.whl", hash = "sha256:023b44101dfe49d7d53932be418477dba359649246075c996866106da069af69"}, + {file = "contourpy-1.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:8153b8bfc11e1e4d75bcb0bff1db232f9e10b274e0929de9d608027e0d34ff8b"}, + {file = "contourpy-1.3.3-cp312-cp312-win_arm64.whl", hash = "sha256:07ce5ed73ecdc4a03ffe3e1b3e3c1166db35ae7584be76f65dbbe28a7791b0cc"}, + {file = "contourpy-1.3.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:177fb367556747a686509d6fef71d221a4b198a3905fe824430e5ea0fda54eb5"}, + {file = "contourpy-1.3.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d002b6f00d73d69333dac9d0b8d5e84d9724ff9ef044fd63c5986e62b7c9e1b1"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:348ac1f5d4f1d66d3322420f01d42e43122f43616e0f194fc1c9f5d830c5b286"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:655456777ff65c2c548b7c454af9c6f33f16c8884f11083244b5819cc214f1b5"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:644a6853d15b2512d67881586bd03f462c7ab755db95f16f14d7e238f2852c67"}, + {file = "contourpy-1.3.3-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4debd64f124ca62069f313a9cb86656ff087786016d76927ae2cf37846b006c9"}, + {file = "contourpy-1.3.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a15459b0f4615b00bbd1e91f1b9e19b7e63aea7483d03d804186f278c0af2659"}, + {file = "contourpy-1.3.3-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ca0fdcd73925568ca027e0b17ab07aad764be4706d0a925b89227e447d9737b7"}, + {file = "contourpy-1.3.3-cp313-cp313-win32.whl", hash = "sha256:b20c7c9a3bf701366556e1b1984ed2d0cedf999903c51311417cf5f591d8c78d"}, + {file = "contourpy-1.3.3-cp313-cp313-win_amd64.whl", hash = "sha256:1cadd8b8969f060ba45ed7c1b714fe69185812ab43bd6b86a9123fe8f99c3263"}, + {file = "contourpy-1.3.3-cp313-cp313-win_arm64.whl", hash = "sha256:fd914713266421b7536de2bfa8181aa8c699432b6763a0ea64195ebe28bff6a9"}, + {file = "contourpy-1.3.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:88df9880d507169449d434c293467418b9f6cbe82edd19284aa0409e7fdb933d"}, + {file = "contourpy-1.3.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:d06bb1f751ba5d417047db62bca3c8fde202b8c11fb50742ab3ab962c81e8216"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e4e6b05a45525357e382909a4c1600444e2a45b4795163d3b22669285591c1ae"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ab3074b48c4e2cf1a960e6bbeb7f04566bf36b1861d5c9d4d8ac04b82e38ba20"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6c3d53c796f8647d6deb1abe867daeb66dcc8a97e8455efa729516b997b8ed99"}, + {file = "contourpy-1.3.3-cp313-cp313t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:50ed930df7289ff2a8d7afeb9603f8289e5704755c7e5c3bbd929c90c817164b"}, + {file = "contourpy-1.3.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:4feffb6537d64b84877da813a5c30f1422ea5739566abf0bd18065ac040e120a"}, + {file = "contourpy-1.3.3-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2b7e9480ffe2b0cd2e787e4df64270e3a0440d9db8dc823312e2c940c167df7e"}, + {file = "contourpy-1.3.3-cp313-cp313t-win32.whl", hash = "sha256:283edd842a01e3dcd435b1c5116798d661378d83d36d337b8dde1d16a5fc9ba3"}, + {file = "contourpy-1.3.3-cp313-cp313t-win_amd64.whl", hash = "sha256:87acf5963fc2b34825e5b6b048f40e3635dd547f590b04d2ab317c2619ef7ae8"}, + {file = "contourpy-1.3.3-cp313-cp313t-win_arm64.whl", hash = "sha256:3c30273eb2a55024ff31ba7d052dde990d7d8e5450f4bbb6e913558b3d6c2301"}, + {file = "contourpy-1.3.3-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fde6c716d51c04b1c25d0b90364d0be954624a0ee9d60e23e850e8d48353d07a"}, + {file = "contourpy-1.3.3-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:cbedb772ed74ff5be440fa8eee9bd49f64f6e3fc09436d9c7d8f1c287b121d77"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:22e9b1bd7a9b1d652cd77388465dc358dafcd2e217d35552424aa4f996f524f5"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a22738912262aa3e254e4f3cb079a95a67132fc5a063890e224393596902f5a4"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:afe5a512f31ee6bd7d0dda52ec9864c984ca3d66664444f2d72e0dc4eb832e36"}, + {file = "contourpy-1.3.3-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f64836de09927cba6f79dcd00fdd7d5329f3fccc633468507079c829ca4db4e3"}, + {file = "contourpy-1.3.3-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:1fd43c3be4c8e5fd6e4f2baeae35ae18176cf2e5cced681cca908addf1cdd53b"}, + {file = "contourpy-1.3.3-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:6afc576f7b33cf00996e5c1102dc2a8f7cc89e39c0b55df93a0b78c1bd992b36"}, + {file = "contourpy-1.3.3-cp314-cp314-win32.whl", hash = "sha256:66c8a43a4f7b8df8b71ee1840e4211a3c8d93b214b213f590e18a1beca458f7d"}, + {file = "contourpy-1.3.3-cp314-cp314-win_amd64.whl", hash = "sha256:cf9022ef053f2694e31d630feaacb21ea24224be1c3ad0520b13d844274614fd"}, + {file = "contourpy-1.3.3-cp314-cp314-win_arm64.whl", hash = "sha256:95b181891b4c71de4bb404c6621e7e2390745f887f2a026b2d99e92c17892339"}, + {file = "contourpy-1.3.3-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:33c82d0138c0a062380332c861387650c82e4cf1747aaa6938b9b6516762e772"}, + {file = "contourpy-1.3.3-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:ea37e7b45949df430fe649e5de8351c423430046a2af20b1c1961cae3afcda77"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d304906ecc71672e9c89e87c4675dc5c2645e1f4269a5063b99b0bb29f232d13"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:ca658cd1a680a5c9ea96dc61cdbae1e85c8f25849843aa799dfd3cb370ad4fbe"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_26_s390x.manylinux_2_28_s390x.whl", hash = "sha256:ab2fd90904c503739a75b7c8c5c01160130ba67944a7b77bbf36ef8054576e7f"}, + {file = "contourpy-1.3.3-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7301b89040075c30e5768810bc96a8e8d78085b47d8be6e4c3f5a0b4ed478a0"}, + {file = "contourpy-1.3.3-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:2a2a8b627d5cc6b7c41a4beff6c5ad5eb848c88255fda4a8745f7e901b32d8e4"}, + {file = "contourpy-1.3.3-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:fd6ec6be509c787f1caf6b247f0b1ca598bef13f4ddeaa126b7658215529ba0f"}, + {file = "contourpy-1.3.3-cp314-cp314t-win32.whl", hash = "sha256:e74a9a0f5e3fff48fb5a7f2fd2b9b70a3fe014a67522f79b7cca4c0c7e43c9ae"}, + {file = "contourpy-1.3.3-cp314-cp314t-win_amd64.whl", hash = "sha256:13b68d6a62db8eafaebb8039218921399baf6e47bf85006fd8529f2a08ef33fc"}, + {file = "contourpy-1.3.3-cp314-cp314t-win_arm64.whl", hash = "sha256:b7448cb5a725bb1e35ce88771b86fba35ef418952474492cf7c764059933ff8b"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:cd5dfcaeb10f7b7f9dc8941717c6c2ade08f587be2226222c12b25f0483ed497"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:0c1fc238306b35f246d61a1d416a627348b5cf0648648a031e14bb8705fcdfe8"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:70f9aad7de812d6541d29d2bbf8feb22ff7e1c299523db288004e3157ff4674e"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5ed3657edf08512fc3fe81b510e35c2012fbd3081d2e26160f27ca28affec989"}, + {file = "contourpy-1.3.3-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:3d1a3799d62d45c18bafd41c5fa05120b96a28079f2393af559b843d1a966a77"}, + {file = "contourpy-1.3.3.tar.gz", hash = "sha256:083e12155b210502d0bca491432bb04d56dc3432f95a979b429f2848c3dbe880"}, +] + +[package.dependencies] +numpy = ">=1.25" + +[package.extras] +bokeh = ["bokeh", "selenium"] +docs = ["furo", "sphinx (>=7.2)", "sphinx-copybutton"] +mypy = ["bokeh", "contourpy[bokeh,docs]", "docutils-stubs", "mypy (==1.17.0)", "types-Pillow"] +test = ["Pillow", "contourpy[test-no-images]", "matplotlib"] +test-no-images = ["pytest", "pytest-cov", "pytest-rerunfailures", "pytest-xdist", "wurlitzer"] + [[package]] name = "coverage" version = "7.10.1" description = "Code coverage measurement for Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "coverage-7.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1c86eb388bbd609d15560e7cc0eb936c102b6f43f31cf3e58b4fd9afe28e1372"}, {file = "coverage-7.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6b4ba0f488c1bdb6bd9ba81da50715a372119785458831c73428a8566253b86b"}, @@ -427,7 +615,23 @@ files = [ tomli = {version = "*", optional = true, markers = "python_full_version <= \"3.11.0a6\" and extra == \"toml\""} [package.extras] -toml = ["tomli"] +toml = ["tomli ; python_full_version <= \"3.11.0a6\""] + +[[package]] +name = "cycler" +version = "0.12.1" +description = "Composable style cycles" +optional = false +python-versions = ">=3.8" +groups = ["dev"] +files = [ + {file = "cycler-0.12.1-py3-none-any.whl", hash = "sha256:85cef7cff222d8644161529808465972e51340599459b8ac3ccbac5a854e0d30"}, + {file = "cycler-0.12.1.tar.gz", hash = "sha256:88bb128f02ba341da8ef447245a9e138fae777f6a23943da4540077d3601eb1c"}, +] + +[package.extras] +docs = ["ipython", "matplotlib", "numpydoc", "sphinx"] +tests = ["pytest", "pytest-cov", "pytest-xdist"] [[package]] name = "debugpy" @@ -435,6 +639,7 @@ version = "1.8.15" description = "An implementation of the Debug Adapter Protocol for Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "debugpy-1.8.15-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:e9a8125c85172e3ec30985012e7a81ea5e70bbb836637f8a4104f454f9b06c97"}, {file = "debugpy-1.8.15-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7fd0b6b5eccaa745c214fd240ea82f46049d99ef74b185a3517dad3ea1ec55d9"}, @@ -470,6 +675,7 @@ version = "5.2.1" description = "Decorators for Humans" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "decorator-5.2.1-py3-none-any.whl", hash = "sha256:d316bb415a2d9e2d2b3abcc4084c6502fc09240e292cd76a76afc106a1c8e04a"}, {file = "decorator-5.2.1.tar.gz", hash = "sha256:65f266143752f734b0a7cc83c46f4618af75b8c5911b00ccb61d0ac9b6da0360"}, @@ -481,6 +687,7 @@ version = "0.4.0" description = "Distribution utilities" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "distlib-0.4.0-py2.py3-none-any.whl", hash = "sha256:9659f7d87e46584a30b5780e43ac7a2143098441670ff0a49d5f9034c54a6c16"}, {file = "distlib-0.4.0.tar.gz", hash = "sha256:feec40075be03a04501a973d81f633735b4b69f98b05450592310c0f401a4e0d"}, @@ -492,6 +699,8 @@ version = "1.3.0" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" +groups = ["dev"] +markers = "python_version == \"3.10\"" files = [ {file = "exceptiongroup-1.3.0-py3-none-any.whl", hash = "sha256:4d111e6e0c13d0644cad6ddaa7ed0261a0b36971f6d23e7ec9b4b9097da78a10"}, {file = "exceptiongroup-1.3.0.tar.gz", hash = "sha256:b241f5885f560bc56a59ee63ca4c6a8bfa46ae4ad651af316d4e81817bb9fd88"}, @@ -509,6 +718,7 @@ version = "2.1.1" description = "execnet: rapid multi-Python deployment" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "execnet-2.1.1-py3-none-any.whl", hash = "sha256:26dee51f1b80cebd6d0ca8e74dd8745419761d3bef34163928cbebbdc4749fdc"}, {file = "execnet-2.1.1.tar.gz", hash = "sha256:5189b52c6121c24feae288166ab41b32549c7e2348652736540b9e6e7d4e72e3"}, @@ -523,13 +733,14 @@ version = "2.2.0" description = "Get the currently executing AST node of a frame, and other information" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "executing-2.2.0-py2.py3-none-any.whl", hash = "sha256:11387150cad388d62750327a53d3339fad4888b39a6fe233c3afbb54ecffd3aa"}, {file = "executing-2.2.0.tar.gz", hash = "sha256:5d108c028108fe2551d1a7b2e8b713341e2cb4fc0aa7dcf966fa4327a5226755"}, ] [package.extras] -tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich"] +tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipython", "littleutils", "pytest", "rich ; python_version >= \"3.11\""] [[package]] name = "filelock" @@ -537,6 +748,7 @@ version = "3.18.0" description = "A platform independent file lock." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "filelock-3.18.0-py3-none-any.whl", hash = "sha256:c401f4f8377c4464e6db25fff06205fd89bdd83b65eb0488ed1b160f780e21de"}, {file = "filelock-3.18.0.tar.gz", hash = "sha256:adbc88eabb99d2fec8c9c1b229b171f18afa655400173ddc653d5d01501fb9f2"}, @@ -545,7 +757,7 @@ files = [ [package.extras] docs = ["furo (>=2024.8.6)", "sphinx (>=8.1.3)", "sphinx-autodoc-typehints (>=3)"] testing = ["covdefaults (>=2.3)", "coverage (>=7.6.10)", "diff-cover (>=9.2.1)", "pytest (>=8.3.4)", "pytest-asyncio (>=0.25.2)", "pytest-cov (>=6)", "pytest-mock (>=3.14)", "pytest-timeout (>=2.3.1)", "virtualenv (>=20.28.1)"] -typing = ["typing-extensions (>=4.12.2)"] +typing = ["typing-extensions (>=4.12.2) ; python_version < \"3.11\""] [[package]] name = "flatbuffers" @@ -553,17 +765,92 @@ version = "25.2.10" description = "The FlatBuffers serialization format for Python" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "flatbuffers-25.2.10-py2.py3-none-any.whl", hash = "sha256:ebba5f4d5ea615af3f7fd70fc310636fbb2bbd1f566ac0a23d98dd412de50051"}, {file = "flatbuffers-25.2.10.tar.gz", hash = "sha256:97e451377a41262f8d9bd4295cc836133415cc03d8cb966410a4af92eb00d26e"}, ] +[[package]] +name = "fonttools" +version = "4.63.0" +description = "Tools to manipulate font files" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "fonttools-4.63.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e3297a6a4059b4acc3a1e9a8b04741f240a80044eef08ebd32e8b5bcdddce75b"}, + {file = "fonttools-4.63.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b1cd75a03ad8cb5bc40c90bfde68c0c47de423aa19e5c0f362b43520645eea94"}, + {file = "fonttools-4.63.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c0425b277a59cff3d80ca42162a8de360f318438a2ac83570842a678d826d579"}, + {file = "fonttools-4.63.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d7e5c9973aa04c95650c96e5f5ad865fbf42d62079163ecfab1e01cbc2504c22"}, + {file = "fonttools-4.63.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cb014d58140a38135f16064c74c652ed57aa0b75cbf8bb59cac821f7edb5334e"}, + {file = "fonttools-4.63.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:032038247a96c1690f9f31e377c389383c902531b085aa4e4dabd6f57f870e69"}, + {file = "fonttools-4.63.0-cp310-cp310-win32.whl", hash = "sha256:a8b33a82979e0a6a34ff435cc81317be1f95ec1ebb7a3a2d1c8a6a54f02ae44e"}, + {file = "fonttools-4.63.0-cp310-cp310-win_amd64.whl", hash = "sha256:0c18358a155d75034911c5ee397a5b44cd19dd325dbb8b35fb60bf421d6a72ac"}, + {file = "fonttools-4.63.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2b8ae05d9eacf6081414d759c0a352769ac28ce31280d6bb8e77b03f9e3c449f"}, + {file = "fonttools-4.63.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:79cdc9f567aec74a72918fd060283911406750cbc9fd28c1316023deb6ce31a9"}, + {file = "fonttools-4.63.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2c14b4fd138c4bafcca294765c547914e1aa431ae1ca94ab99d8db08c958bd3b"}, + {file = "fonttools-4.63.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:d76ac49f929aecaf82d83250b8347e099d7aecba0f4726c1d9b6df3b8bb5fe18"}, + {file = "fonttools-4.63.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:dcf076a4474fe0d7367e5bbf5b052c7284fa1feca729c04176ce513521afd8a0"}, + {file = "fonttools-4.63.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:7dd683fef0663e9f0f45cf541d788d24caa3ec9db50796b588e1757d8b3bc007"}, + {file = "fonttools-4.63.0-cp311-cp311-win32.whl", hash = "sha256:afefc1ed0a59785a7fb06ea7e1678e849c193e1e387db783579bc7b3056fcfcb"}, + {file = "fonttools-4.63.0-cp311-cp311-win_amd64.whl", hash = "sha256:063e08bd17bd5a90127a14123de0d6a952dbc847695fd98b63c043d58057f90c"}, + {file = "fonttools-4.63.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:37dd23e621e3b0aef1baa70a303b80aaf38449632cfc8fd2a55fb285bbccfc02"}, + {file = "fonttools-4.63.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:a9faff9e0c1f76f9fd55899d2ce785832efebab37eb8ae13995853aef178bef0"}, + {file = "fonttools-4.63.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ef3048ef05dbb552b89817713d9cac912e00d0fde4a3105c00d29e52e10c89af"}, + {file = "fonttools-4.63.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:58dc6bb86a78d782f00f9190ca02c119cf5bbe2807536e361e18d42019f877d8"}, + {file = "fonttools-4.63.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ee08ebfa58f6e1aeff5697ab9582105bb620008c1caafb681e4c557e7483027b"}, + {file = "fonttools-4.63.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:27fdc65af8da6f88b9c6121c47a464cbe359fcfff7ff6fc2d37a1f395d755b78"}, + {file = "fonttools-4.63.0-cp312-cp312-win32.whl", hash = "sha256:af2fd1664d00a397d75f806985ddb36282091c2131a73a6485c23b4a34722263"}, + {file = "fonttools-4.63.0-cp312-cp312-win_amd64.whl", hash = "sha256:59ac449f8cca9b4ffa08d2e7bbadad87ce710d69d1eda5c3c1ce579baa987272"}, + {file = "fonttools-4.63.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:cd7e9857e5e63738b9d9fd707bc1f59c8b09e5177726d23664db393c59bb08bd"}, + {file = "fonttools-4.63.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c2a2a42198b696a6f48fad91709afb55176e66a5e566131219dba372fb7f8c59"}, + {file = "fonttools-4.63.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1e874792a8212b44583ea02189d9e693906b2f78b261f372f95d6c563210ac1d"}, + {file = "fonttools-4.63.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:22135da48a348785c5e2d5d2d9d6bec5ed44adacbaeb9db12d9493bf6c6bfa68"}, + {file = "fonttools-4.63.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ccf41f2efdf56994d22d73bef4ced1052161958169428d06ba9724ea9e9a64be"}, + {file = "fonttools-4.63.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:9ced0bd02ac751dd6319b0da88aaef24414e3b0dbc32bb4f24944821a3741a27"}, + {file = "fonttools-4.63.0-cp313-cp313-win32.whl", hash = "sha256:85be818f5506e8a7753153def2c9550178f0ecae6a47b5e0e8dbb23f7cc90380"}, + {file = "fonttools-4.63.0-cp313-cp313-win_amd64.whl", hash = "sha256:ba04cb5891d4c0c21b6da95eda8d7b090021508a294fff33464fc7d241e0856b"}, + {file = "fonttools-4.63.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:fd1e3094f42d806d3d7c79162fc59e5910fcbe3a7360c385b8da969bc4493745"}, + {file = "fonttools-4.63.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6e528da43bc3791085f8cb6141b1d13e459226790240340fcbb4625649238b03"}, + {file = "fonttools-4.63.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6b2248c5decb223562f7902ff6325077a073f608ee8e33e88ad88db734eb9f49"}, + {file = "fonttools-4.63.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:308f957cdeaf8abe4e5f2f124902ef405448af92c90f80e302a3b771c2e6116b"}, + {file = "fonttools-4.63.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:bf00f21eb5fb721dbaf73d1e9da6d02a1af7768f2ebcf9798be98beab8ba90f6"}, + {file = "fonttools-4.63.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:c1aaa4b9c75798400ac043ce04d74e7830376c85095a5a6ed7cba2f17a266bf4"}, + {file = "fonttools-4.63.0-cp314-cp314-win32.whl", hash = "sha256:22693918177bd9ceabec4736d338045f357769416fc6b0b2508eefef75b08616"}, + {file = "fonttools-4.63.0-cp314-cp314-win_amd64.whl", hash = "sha256:7d782fac32985914c351556f68ac0855391572bcd87de50e05970d3cd4c96fc5"}, + {file = "fonttools-4.63.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:6db5140a60a5d731d21ec076745b40a310607731b0a565b50776393188649001"}, + {file = "fonttools-4.63.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7d76edbff9014094dbf03bd2d074709dfa6ec7aba13d838c937a2b33d2d6a86e"}, + {file = "fonttools-4.63.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0eac00b9118c3c2f87d272e45341871c5b3066baa3c86897fa634a7c3fb59096"}, + {file = "fonttools-4.63.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51394295f1a51de8b5f30bdb1e1b9a4231536c7064ef5c6e211eec19fa36036f"}, + {file = "fonttools-4.63.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:9e12f105d2b6342c559c298afb674006bb2893afc7102dcf8a1b55b0486b4e40"}, + {file = "fonttools-4.63.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:796f27556dbe094c4824f75ca85267e4df776c79036c8441469a4df37038c196"}, + {file = "fonttools-4.63.0-cp314-cp314t-win32.whl", hash = "sha256:948428a275741f0b64b113c955425a953314f4b9ab9997f73a72c83e68e569c8"}, + {file = "fonttools-4.63.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6d4741eb179121cab9eea4cb2393d24492373a260d7945006358c08cfbf45419"}, + {file = "fonttools-4.63.0-py3-none-any.whl", hash = "sha256:445af2eab030a16b9171ea8bdda7ebf7d96bda2df88ee182a464252f6e05e20d"}, + {file = "fonttools-4.63.0.tar.gz", hash = "sha256:caeb583deeb5168e694b65cda8b4ee62abedfa66cf88488734466f2366b9c4e0"}, +] + +[package.extras] +all = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "lxml (>=4.0)", "lz4 (>=1.7.4.2)", "matplotlib", "munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\"", "skia-pathops (>=0.5.0)", "sympy", "uharfbuzz (>=0.45.0)", "unicodedata2 (>=17.0.0) ; python_version <= \"3.14\"", "xattr ; sys_platform == \"darwin\"", "zopfli (>=0.1.4)"] +graphite = ["lz4 (>=1.7.4.2)"] +interpolatable = ["munkres ; platform_python_implementation == \"PyPy\"", "pycairo", "scipy ; platform_python_implementation != \"PyPy\""] +lxml = ["lxml (>=4.0)"] +pathops = ["skia-pathops (>=0.5.0)"] +plot = ["matplotlib"] +repacker = ["uharfbuzz (>=0.45.0)"] +symfont = ["sympy"] +type1 = ["xattr ; sys_platform == \"darwin\""] +unicode = ["unicodedata2 (>=17.0.0) ; python_version <= \"3.14\""] +woff = ["brotli (>=1.0.1) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\"", "zopfli (>=0.1.4)"] + [[package]] name = "gast" version = "0.6.0" description = "Python AST that abstracts the underlying Python version" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,>=2.7" +groups = ["main"] files = [ {file = "gast-0.6.0-py3-none-any.whl", hash = "sha256:52b182313f7330389f72b069ba00f174cfe2a06411099547288839c6cbafbd54"}, {file = "gast-0.6.0.tar.gz", hash = "sha256:88fc5300d32c7ac6ca7b515310862f71e6fdf2c029bbec7c66c0f5dd47b6b1fb"}, @@ -575,6 +862,7 @@ version = "2.1.0" description = "Copy your docs directly to the gh-pages branch." optional = false python-versions = "*" +groups = ["doc"] files = [ {file = "ghp-import-2.1.0.tar.gz", hash = "sha256:9c535c4c61193c2df8871222567d7fd7e5014d835f97dc7b7439069e2413d343"}, {file = "ghp_import-2.1.0-py3-none-any.whl", hash = "sha256:8337dd7b50877f163d4c0289bc1f1c7f127550241988d568c1db512c4324a619"}, @@ -592,6 +880,7 @@ version = "0.2.0" description = "pasta is an AST-based Python refactoring library" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "google-pasta-0.2.0.tar.gz", hash = "sha256:c9f2c8dfc8f96d0d5808299920721be30c9eec37f2389f28904f454565c8a16e"}, {file = "google_pasta-0.2.0-py2-none-any.whl", hash = "sha256:4612951da876b1a10fe3960d7226f0c7682cf901e16ac06e473b267a5afa8954"}, @@ -607,6 +896,7 @@ version = "0.25.5" description = "Signatures for entire Python programs. Extract the structure, the frame, the skeleton of your project, to generate API documentation or find breaking changes in your API." optional = false python-versions = ">=3.7" +groups = ["doc"] files = [ {file = "griffe-0.25.5-py3-none-any.whl", hash = "sha256:1fb9edff48e66d4873014a2ebf21aca5f271d0006a4c937826e3cf592ffb3706"}, {file = "griffe-0.25.5.tar.gz", hash = "sha256:11ea3403ef0560a1cbcf7f302eb5d21cf4c1d8ed3f8a16a75aa9f6f458caf3f1"}, @@ -624,6 +914,7 @@ version = "1.74.0" description = "HTTP/2-based RPC framework" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "grpcio-1.74.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:85bd5cdf4ed7b2d6438871adf6afff9af7096486fcf51818a81b77ef4dd30907"}, {file = "grpcio-1.74.0-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:68c8ebcca945efff9d86d8d6d7bfb0841cf0071024417e2d7f45c5e46b5b08eb"}, @@ -687,6 +978,7 @@ version = "3.14.0" description = "Read and write HDF5 files from Python" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "h5py-3.14.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:24df6b2622f426857bda88683b16630014588a0e4155cba44e872eb011c4eaed"}, {file = "h5py-3.14.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6ff2389961ee5872de697054dd5a033b04284afc3fb52dc51d94561ece2c10c6"}, @@ -725,6 +1017,7 @@ version = "2.6.12" description = "File identification library for Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "identify-2.6.12-py2.py3-none-any.whl", hash = "sha256:ad9672d5a72e0d2ff7c5c8809b62dfa60458626352fb0eb7b55e69bdc45334a2"}, {file = "identify-2.6.12.tar.gz", hash = "sha256:d8de45749f1efb108badef65ee8386f0f7bb19a7f26185f74de6367bffbaf0e6"}, @@ -739,6 +1032,7 @@ version = "3.10" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.6" +groups = ["main", "doc"] files = [ {file = "idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3"}, {file = "idna-3.10.tar.gz", hash = "sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9"}, @@ -753,6 +1047,7 @@ version = "8.7.0" description = "Read metadata from Python packages" optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "importlib_metadata-8.7.0-py3-none-any.whl", hash = "sha256:e5dd1551894c77868a30651cef00984d50e1002d06942a7101d34870c5f02afd"}, {file = "importlib_metadata-8.7.0.tar.gz", hash = "sha256:d13b81ad223b890aa16c5471f2ac3056cf76c5f10f82d6f9292f0b415f389000"}, @@ -762,12 +1057,12 @@ files = [ zipp = ">=3.20" [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] perf = ["ipython"] -test = ["flufl.flake8", "importlib_resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] +test = ["flufl.flake8", "importlib_resources (>=1.3) ; python_version < \"3.9\"", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.9.2)"] type = ["pytest-mypy"] [[package]] @@ -776,16 +1071,14 @@ version = "6.5.2" description = "Read resources from Python packages" optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "importlib_resources-6.5.2-py3-none-any.whl", hash = "sha256:789cfdc3ed28c78b67a06acb8126751ced69a3d5f79c095a98298cd8a760ccec"}, {file = "importlib_resources-6.5.2.tar.gz", hash = "sha256:185f87adef5bcc288449d98fb4fba07cea78bc036455dd44c5fc4a2fe78fed2c"}, ] -[package.dependencies] -zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} - [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] @@ -798,6 +1091,7 @@ version = "2.1.0" description = "brain-dead simple config-ini parsing" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "iniconfig-2.1.0-py3-none-any.whl", hash = "sha256:9deba5723312380e77435581c6bf4935c94cbfab9b1ed33ef8d238ea168eb760"}, {file = "iniconfig-2.1.0.tar.gz", hash = "sha256:3abbd2e30b36733fee78f9c7f7308f2d0050e88f0087fd25c2645f63c773e1c7"}, @@ -809,6 +1103,7 @@ version = "6.30.0" description = "IPython Kernel for Jupyter" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "ipykernel-6.30.0-py3-none-any.whl", hash = "sha256:fd2936e55c4a1c2ee8b1e5fa6a372b8eecc0ab1338750dee76f48fa5cca1301e"}, {file = "ipykernel-6.30.0.tar.gz", hash = "sha256:b7b808ddb2d261aae2df3a26ff3ff810046e6de3dfbc6f7de8c98ea0a6cb632c"}, @@ -842,6 +1137,7 @@ version = "8.18.1" description = "IPython: Productive Interactive Computing" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "ipython-8.18.1-py3-none-any.whl", hash = "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397"}, {file = "ipython-8.18.1.tar.gz", hash = "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27"}, @@ -858,7 +1154,6 @@ prompt-toolkit = ">=3.0.41,<3.1.0" pygments = ">=2.4.0" stack-data = "*" traitlets = ">=5" -typing-extensions = {version = "*", markers = "python_version < \"3.10\""} [package.extras] all = ["black", "curio", "docrepr", "exceptiongroup", "ipykernel", "ipyparallel", "ipywidgets", "matplotlib", "matplotlib (!=3.2.0)", "nbconvert", "nbformat", "notebook", "numpy (>=1.22)", "pandas", "pickleshare", "pytest (<7)", "pytest (<7.1)", "pytest-asyncio (<0.22)", "qtconsole", "setuptools (>=18.5)", "sphinx (>=1.3)", "sphinx-rtd-theme", "stack-data", "testpath", "trio", "typing-extensions"] @@ -879,6 +1174,7 @@ version = "0.19.2" description = "An autocompletion tool for Python that can be used for text editors." optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "jedi-0.19.2-py2.py3-none-any.whl", hash = "sha256:a8ef22bde8490f57fe5c7681a3c83cb58874daf72b4784de3cce5b6ef6edb5b9"}, {file = "jedi-0.19.2.tar.gz", hash = "sha256:4770dc3de41bde3966b02eb84fbcf557fb33cce26ad23da12c742fb50ecb11f0"}, @@ -898,6 +1194,7 @@ version = "3.1.6" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" +groups = ["dev", "doc"] files = [ {file = "jinja2-3.1.6-py3-none-any.whl", hash = "sha256:85ece4451f492d0c13c5dd7c13a64681a86afae63a5f347908daf103ce6d2f67"}, {file = "jinja2-3.1.6.tar.gz", hash = "sha256:0137fb05990d35f1275a587e9aee6d56da821fc83491a0fb838183be43f66d6d"}, @@ -909,19 +1206,31 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "joblib" +version = "1.5.3" +description = "Lightweight pipelining with Python functions" +optional = false +python-versions = ">=3.9" +groups = ["dev"] +files = [ + {file = "joblib-1.5.3-py3-none-any.whl", hash = "sha256:5fc3c5039fc5ca8c0276333a188bbd59d6b7ab37fe6632daa76bc7f9ec18e713"}, + {file = "joblib-1.5.3.tar.gz", hash = "sha256:8561a3269e6801106863fd0d6d84bb737be9e7631e33aaed3fb9ce5953688da3"}, +] + [[package]] name = "jupyter-client" version = "8.6.3" description = "Jupyter protocol implementation and client libraries" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "jupyter_client-8.6.3-py3-none-any.whl", hash = "sha256:e8a19cc986cc45905ac3362915f410f3af85424b4c0905e94fa5f2cb08e8f23f"}, {file = "jupyter_client-8.6.3.tar.gz", hash = "sha256:35b3a0947c4a6e9d589eb97d7d4cd5e90f910ee73101611f01283732bd6d9419"}, ] [package.dependencies] -importlib-metadata = {version = ">=4.8.3", markers = "python_version < \"3.10\""} jupyter-core = ">=4.12,<5.0.dev0 || >=5.1.dev0" python-dateutil = ">=2.8.2" pyzmq = ">=23.0" @@ -930,7 +1239,7 @@ traitlets = ">=5.3" [package.extras] docs = ["ipykernel", "myst-parser", "pydata-sphinx-theme", "sphinx (>=4)", "sphinx-autodoc-typehints", "sphinxcontrib-github-alt", "sphinxcontrib-spelling"] -test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko", "pre-commit", "pytest (<8.2.0)", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] +test = ["coverage", "ipykernel (>=6.14)", "mypy", "paramiko ; sys_platform == \"win32\"", "pre-commit", "pytest (<8.2.0)", "pytest-cov", "pytest-jupyter[client] (>=0.4.1)", "pytest-timeout"] [[package]] name = "jupyter-core" @@ -938,6 +1247,7 @@ version = "5.8.1" description = "Jupyter core package. A base package on which Jupyter projects rely." optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "jupyter_core-5.8.1-py3-none-any.whl", hash = "sha256:c28d268fc90fb53f1338ded2eb410704c5449a358406e8a948b75706e24863d0"}, {file = "jupyter_core-5.8.1.tar.gz", hash = "sha256:0a5f9706f70e64786b75acba995988915ebd4601c8a52e534a40b51c95f59941"}, @@ -958,6 +1268,7 @@ version = "3.10.0" description = "Multi-backend Keras" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "keras-3.10.0-py3-none-any.whl", hash = "sha256:c095a6bf90cd50defadf73d4859ff794fad76b775357ef7bd1dbf96388dae7d3"}, {file = "keras-3.10.0.tar.gz", hash = "sha256:6e9100bf66eaf6de4b7f288d34ef9bb8b5dcdd62f42c64cfd910226bb34ad2d2"}, @@ -973,12 +1284,140 @@ optree = "*" packaging = "*" rich = "*" +[[package]] +name = "kiwisolver" +version = "1.5.0" +description = "A fast implementation of the Cassowary constraint solver" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "kiwisolver-1.5.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:32cc0a5365239a6ea0c6ed461e8838d053b57e397443c0ca894dcc8e388d4374"}, + {file = "kiwisolver-1.5.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:cc0b66c1eec9021353a4b4483afb12dfd50e3669ffbb9152d6842eb34c7e29fd"}, + {file = "kiwisolver-1.5.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:86e0287879f75621ae85197b0877ed2f8b7aa57b511c7331dce2eb6f4de7d476"}, + {file = "kiwisolver-1.5.0-cp310-cp310-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:62f59da443c4f4849f73a51a193b1d9d258dcad0c41bc4d1b8fb2bcc04bfeb22"}, + {file = "kiwisolver-1.5.0-cp310-cp310-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9190426b7aa26c5229501fa297b8d0653cfd3f5a36f7990c264e157cbf886b3b"}, + {file = "kiwisolver-1.5.0-cp310-cp310-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:c8277104ded0a51e699c8c3aff63ce2c56d4ed5519a5f73e0fd7057f959a2b9e"}, + {file = "kiwisolver-1.5.0-cp310-cp310-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8f9baf6f0a6e7571c45c8863010b45e837c3ee1c2c77fcd6ef423be91b21fedb"}, + {file = "kiwisolver-1.5.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cff8e5383db4989311f99e814feeb90c4723eb4edca425b9d5d9c3fefcdd9537"}, + {file = "kiwisolver-1.5.0-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:ebae99ed6764f2b5771c522477b311be313e8841d2e0376db2b10922daebbba4"}, + {file = "kiwisolver-1.5.0-cp310-cp310-musllinux_1_2_s390x.whl", hash = "sha256:d5cd5189fc2b6a538b75ae45433140c4823463918f7b1617c31e68b085c0022c"}, + {file = "kiwisolver-1.5.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:f42c23db5d1521218a3276bb08666dcb662896a0be7347cba864eca45ff64ede"}, + {file = "kiwisolver-1.5.0-cp310-cp310-win_amd64.whl", hash = "sha256:94eff26096eb5395136634622515b234ecb6c9979824c1f5004c6e3c3c85ccd2"}, + {file = "kiwisolver-1.5.0-cp310-cp310-win_arm64.whl", hash = "sha256:dd952e03bfbb096cfe2dd35cd9e00f269969b67536cb4370994afc20ff2d0875"}, + {file = "kiwisolver-1.5.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9eed0f7edbb274413b6ee781cca50541c8c0facd3d6fd289779e494340a2b85c"}, + {file = "kiwisolver-1.5.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3c4923e404d6bcd91b6779c009542e5647fef32e4a5d75e115e3bbac6f2335eb"}, + {file = "kiwisolver-1.5.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:0df54df7e686afa55e6f21fb86195224a6d9beb71d637e8d7920c95cf0f89aac"}, + {file = "kiwisolver-1.5.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:2517e24d7315eb51c10664cdb865195df38ab74456c677df67bb47f12d088a27"}, + {file = "kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ff710414307fefa903e0d9bdf300972f892c23477829f49504e59834f4195398"}, + {file = "kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:6176c1811d9d5a04fa391c490cc44f451e240697a16977f11c6f722efb9041db"}, + {file = "kiwisolver-1.5.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:50847dca5d197fcbd389c805aa1a1cf32f25d2e7273dc47ab181a517666b68cc"}, + {file = "kiwisolver-1.5.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:01808c6d15f4c3e8559595d6d1fe6411c68e4a3822b4b9972b44473b24f4e679"}, + {file = "kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:f1f9f4121ec58628c96baa3de1a55a4e3a333c5102c8e94b64e23bf7b2083309"}, + {file = "kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:b7d335370ae48a780c6e6a6bbfa97342f563744c39c35562f3f367665f5c1de2"}, + {file = "kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:800ee55980c18545af444d93fdd60c56b580db5cc54867d8cbf8a1dc0829938c"}, + {file = "kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_s390x.whl", hash = "sha256:c438f6ca858697c9ab67eb28246c92508af972e114cac34e57a6d4ba17a3ac08"}, + {file = "kiwisolver-1.5.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:8c63c91f95173f9c2a67c7c526b2cea976828a0e7fced9cdcead2802dc10f8a4"}, + {file = "kiwisolver-1.5.0-cp311-cp311-win_amd64.whl", hash = "sha256:beb7f344487cdcb9e1efe4b7a29681b74d34c08f0043a327a74da852a6749e7b"}, + {file = "kiwisolver-1.5.0-cp311-cp311-win_arm64.whl", hash = "sha256:ad4ae4ffd1ee9cd11357b4c66b612da9888f4f4daf2f36995eda64bd45370cac"}, + {file = "kiwisolver-1.5.0-cp312-cp312-macosx_10_13_universal2.whl", hash = "sha256:4e9750bc21b886308024f8a54ccb9a2cc38ac9fa813bf4348434e3d54f337ff9"}, + {file = "kiwisolver-1.5.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:72ec46b7eba5b395e0a7b63025490d3214c11013f4aacb4f5e8d6c3041829588"}, + {file = "kiwisolver-1.5.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ed3a984b31da7481b103f68776f7128a89ef26ed40f4dc41a2223cda7fb24819"}, + {file = "kiwisolver-1.5.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bb5136fb5352d3f422df33f0c879a1b0c204004324150cc3b5e3c4f310c9049f"}, + {file = "kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b2af221f268f5af85e776a73d62b0845fc8baf8ef0abfae79d29c77d0e776aaf"}, + {file = "kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:b0f172dc8ffaccb8522d7c5d899de00133f2f1ca7b0a49b7da98e901de87bf2d"}, + {file = "kiwisolver-1.5.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6ab8ba9152203feec73758dad83af9a0bbe05001eb4639e547207c40cfb52083"}, + {file = "kiwisolver-1.5.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:cdee07c4d7f6d72008d3f73b9bf027f4e11550224c7c50d8df1ae4a37c1402a6"}, + {file = "kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7c60d3c9b06fb23bd9c6139281ccbdc384297579ae037f08ae90c69f6845c0b1"}, + {file = "kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:e315e5ec90d88e140f57696ff85b484ff68bb311e36f2c414aa4286293e6dee0"}, + {file = "kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:1465387ac63576c3e125e5337a6892b9e99e0627d52317f3ca79e6930d889d15"}, + {file = "kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_s390x.whl", hash = "sha256:530a3fd64c87cffa844d4b6b9768774763d9caa299e9b75d8eca6a4423b31314"}, + {file = "kiwisolver-1.5.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:1d9daea4ea6b9be74fe2f01f7fbade8d6ffab263e781274cffca0dba9be9eec9"}, + {file = "kiwisolver-1.5.0-cp312-cp312-win_amd64.whl", hash = "sha256:f18c2d9782259a6dc132fdc7a63c168cbc74b35284b6d75c673958982a378384"}, + {file = "kiwisolver-1.5.0-cp312-cp312-win_arm64.whl", hash = "sha256:f7c7553b13f69c1b29a5bde08ddc6d9d0c8bfb84f9ed01c30db25944aeb852a7"}, + {file = "kiwisolver-1.5.0-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:fd40bb9cd0891c4c3cb1ddf83f8bbfa15731a248fdc8162669405451e2724b09"}, + {file = "kiwisolver-1.5.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:c0e1403fd7c26d77c1f03e096dc58a5c726503fa0db0456678b8668f76f521e3"}, + {file = "kiwisolver-1.5.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:dda366d548e89a90d88a86c692377d18d8bd64b39c1fb2b92cb31370e2896bbd"}, + {file = "kiwisolver-1.5.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:332b4f0145c30b5f5ad9374881133e5aa64320428a57c2c2b61e9d891a51c2f3"}, + {file = "kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0c50b89ffd3e1a911c69a1dd3de7173c0cd10b130f56222e57898683841e4f96"}, + {file = "kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:4db576bb8c3ef9365f8b40fe0f671644de6736ae2c27a2c62d7d8a1b4329f099"}, + {file = "kiwisolver-1.5.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:0b85aad90cea8ac6797a53b5d5f2e967334fa4d1149f031c4537569972596cb8"}, + {file = "kiwisolver-1.5.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:d36ca54cb4c6c4686f7cbb7b817f66f5911c12ddb519450bbe86707155028f87"}, + {file = "kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:38f4a703656f493b0ad185211ccfca7f0386120f022066b018eb5296d8613e23"}, + {file = "kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:3ac2360e93cb41be81121755c6462cff3beaa9967188c866e5fce5cf13170859"}, + {file = "kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:c95cab08d1965db3d84a121f1c7ce7479bdd4072c9b3dafd8fecce48a2e6b902"}, + {file = "kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:fc20894c3d21194d8041a28b65622d5b86db786da6e3cfe73f0c762951a61167"}, + {file = "kiwisolver-1.5.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:7a32f72973f0f950c1920475d5c5ea3d971b81b6f0ec53b8d0a956cc965f22e0"}, + {file = "kiwisolver-1.5.0-cp313-cp313-win_amd64.whl", hash = "sha256:0bf3acf1419fa93064a4c2189ac0b58e3be7872bf6ee6177b0d4c63dc4cea276"}, + {file = "kiwisolver-1.5.0-cp313-cp313-win_arm64.whl", hash = "sha256:fa8eb9ecdb7efb0b226acec134e0d709e87a909fa4971a54c0c4f6e88635484c"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-macosx_10_13_universal2.whl", hash = "sha256:db485b3847d182b908b483b2ed133c66d88d49cacf98fd278fadafe11b4478d1"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:be12f931839a3bdfe28b584db0e640a65a8bcbc24560ae3fdb025a449b3d754e"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:16b85d37c2cbb3253226d26e64663f755d88a03439a9c47df6246b35defbdfb7"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:4432b835675f0ea7414aab3d37d119f7226d24869b7a829caeab49ebda407b0c"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:1b0feb50971481a2cc44d94e88bdb02cdd497618252ae226b8eb1201b957e368"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:56fa888f10d0f367155e76ce849fa1166fc9730d13bd2d65a2aa13b6f5424489"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:940dda65d5e764406b9fb92761cbf462e4e63f712ab60ed98f70552e496f3bf1"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-manylinux_2_39_riscv64.whl", hash = "sha256:89fc958c702ee9a745e4700378f5d23fddbc46ff89e8fdbf5395c24d5c1452a3"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:9027d773c4ff81487181a925945743413f6069634d0b122d0b37684ccf4f1e18"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_ppc64le.whl", hash = "sha256:5b233ea3e165e43e35dba1d2b8ecc21cf070b45b65ae17dd2747d2713d942021"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_riscv64.whl", hash = "sha256:ce9bf03dad3b46408c08649c6fbd6ca28a9fce0eb32fdfffa6775a13103b5310"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_s390x.whl", hash = "sha256:fc4d3f1fb9ca0ae9f97b095963bc6326f1dbfd3779d6679a1e016b9baaa153d3"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:f443b4825c50a51ee68585522ab4a1d1257fac65896f282b4c6763337ac9f5d2"}, + {file = "kiwisolver-1.5.0-cp313-cp313t-win_arm64.whl", hash = "sha256:893ff3a711d1b515ba9da14ee090519bad4610ed1962fbe298a434e8c5f8db53"}, + {file = "kiwisolver-1.5.0-cp314-cp314-macosx_10_15_universal2.whl", hash = "sha256:8df31fe574b8b3993cc61764f40941111b25c2d9fea13d3ce24a49907cd2d615"}, + {file = "kiwisolver-1.5.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:1d49a49ac4cbfb7c1375301cd1ec90169dfeae55ff84710d782260ce77a75a02"}, + {file = "kiwisolver-1.5.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:0cbe94b69b819209a62cb27bdfa5dc2a8977d8de2f89dfd97ba4f53ed3af754e"}, + {file = "kiwisolver-1.5.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:80aa065ffd378ff784822a6d7c3212f2d5f5e9c3589614b5c228b311fd3063ac"}, + {file = "kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4e7f886f47ab881692f278ae901039a234e4025a68e6dfab514263a0b1c4ae05"}, + {file = "kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:5060731cc3ed12ca3a8b57acd4aeca5bbc2f49216dd0bec1650a1acd89486bcd"}, + {file = "kiwisolver-1.5.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:7a4aa69609f40fce3cbc3f87b2061f042eee32f94b8f11db707b66a26461591a"}, + {file = "kiwisolver-1.5.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:d168fda2dbff7b9b5f38e693182d792a938c31db4dac3a80a4888de603c99554"}, + {file = "kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:413b820229730d358efd838ecbab79902fe97094565fdc80ddb6b0a18c18a581"}, + {file = "kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:5124d1ea754509b09e53738ec185584cc609aae4a3b510aaf4ed6aa047ef9303"}, + {file = "kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:e4415a8db000bf49a6dd1c478bf70062eaacff0f462b92b0ba68791a905861f9"}, + {file = "kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_s390x.whl", hash = "sha256:d618fd27420381a4f6044faa71f46d8bfd911bd077c555f7138ed88729bfbe79"}, + {file = "kiwisolver-1.5.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5092eb5b1172947f57d6ea7d89b2f29650414e4293c47707eb499ec07a0ac796"}, + {file = "kiwisolver-1.5.0-cp314-cp314-win_amd64.whl", hash = "sha256:d76e2d8c75051d58177e762164d2e9ab92886534e3a12e795f103524f221dd8e"}, + {file = "kiwisolver-1.5.0-cp314-cp314-win_arm64.whl", hash = "sha256:fa6248cd194edff41d7ea9425ced8ca3a6f838bfb295f6f1d6e6bb694a8518df"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-macosx_10_15_universal2.whl", hash = "sha256:d1ffeb80b5676463d7a7d56acbe8e37a20ce725570e09549fe738e02ca6b7e1e"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:bc4d8e252f532ab46a1de9349e2d27b91fce46736a9eedaa37beaca66f574ed4"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:6783e069732715ad0c3ce96dbf21dbc2235ab0593f2baf6338101f70371f4028"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e7c4c09a490dc4d4a7f8cbee56c606a320f9dc28cf92a7157a39d1ce7676a657"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:2a075bd7bd19c70cf67c8badfa36cf7c5d8de3c9ddb8420c51e10d9c50e94920"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:bdd3e53429ff02aa319ba59dfe4ceeec345bf46cf180ec2cf6fd5b942e7975e9"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:3cdcb35dc9d807259c981a85531048ede628eabcffb3239adf3d17463518992d"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:70d593af6a6ca332d1df73d519fddb5148edb15cd90d5f0155e3746a6d4fcc65"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:377815a8616074cabbf3f53354e1d040c35815a134e01d7614b7692e4bf8acfa"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_ppc64le.whl", hash = "sha256:0255a027391d52944eae1dbb5d4cc5903f57092f3674e8e544cdd2622826b3f0"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_riscv64.whl", hash = "sha256:012b1eb16e28718fa782b5e61dc6f2da1f0792ca73bd05d54de6cb9561665fc9"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_s390x.whl", hash = "sha256:0e3aafb33aed7479377e5e9a82e9d4bf87063741fc99fc7ae48b0f16e32bdd6f"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7a116ae737f0000343218c4edf5bd45893bfeaff0993c0b215d7124c9f77646"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1dd9b0b119a350976a6d781e7278ec7aca0b201e1a9e2d23d9804afecb6ca681"}, + {file = "kiwisolver-1.5.0-cp314-cp314t-win_arm64.whl", hash = "sha256:58f812017cd2985c21fbffb4864d59174d4903dd66fa23815e74bbc7a0e2dd57"}, + {file = "kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_10_13_x86_64.whl", hash = "sha256:5ae8e62c147495b01a0f4765c878e9bfdf843412446a247e28df59936e99e797"}, + {file = "kiwisolver-1.5.0-graalpy312-graalpy250_312_native-macosx_11_0_arm64.whl", hash = "sha256:f6764a4ccab3078db14a632420930f6186058750df066b8ea2a7106df91d3203"}, + {file = "kiwisolver-1.5.0-graalpy312-graalpy250_312_native-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c31c13da98624f957b0fb1b5bae5383b2333c2c3f6793d9825dd5ce79b525cb7"}, + {file = "kiwisolver-1.5.0-graalpy312-graalpy250_312_native-win_amd64.whl", hash = "sha256:1f1489f769582498610e015a8ef2d36f28f505ab3096d0e16b4858a9ec214f57"}, + {file = "kiwisolver-1.5.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:295d9ffe712caa9f8a3081de8d32fc60191b4b51c76f02f951fd8407253528f4"}, + {file = "kiwisolver-1.5.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:51e8c4084897de9f05898c2c2a39af6318044ae969d46ff7a34ed3f96274adca"}, + {file = "kiwisolver-1.5.0-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b83af57bdddef03c01a9138034c6ff03181a3028d9a1003b301eb1a55e161a3f"}, + {file = "kiwisolver-1.5.0-pp310-pypy310_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bf4679a3d71012a7c2bf360e5cd878fbd5e4fcac0896b56393dec239d81529ed"}, + {file = "kiwisolver-1.5.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:41024ed50e44ab1a60d3fe0a9d15a4ccc9f5f2b1d814ff283c8d01134d5b81bc"}, + {file = "kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:ec4c85dc4b687c7f7f15f553ff26a98bfe8c58f5f7f0ac8905f0ba4c7be60232"}, + {file = "kiwisolver-1.5.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:12e91c215a96e39f57989c8912ae761286ac5a9584d04030ceb3368a357f017a"}, + {file = "kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:be4a51a55833dc29ab5d7503e7bcb3b3af3402d266018137127450005cdfe737"}, + {file = "kiwisolver-1.5.0-pp311-pypy311_pp73-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:daae526907e262de627d8f70058a0f64acc9e2641c164c99c8f594b34a799a16"}, + {file = "kiwisolver-1.5.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:59cd8683f575d96df5bb48f6add94afc055012c29e28124fcae2b63661b9efb1"}, + {file = "kiwisolver-1.5.0.tar.gz", hash = "sha256:d4193f3d9dc3f6f79aaed0e5637f45d98850ebf01f7ca20e69457f3e8946b66a"}, +] + [[package]] name = "libclang" version = "18.1.1" description = "Clang Python Bindings, mirrored from the official LLVM repo: https://github.com/llvm/llvm-project/tree/main/clang/bindings/python, to make the installation process easier." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "libclang-18.1.1-1-py2.py3-none-macosx_11_0_arm64.whl", hash = "sha256:0b2e143f0fac830156feb56f9231ff8338c20aecfe72b4ffe96f19e5a1dbb69a"}, {file = "libclang-18.1.1-py2.py3-none-macosx_10_9_x86_64.whl", hash = "sha256:6f14c3f194704e5d09769108f03185fce7acaf1d1ae4bbb2f30a72c2400cb7c5"}, @@ -998,6 +1437,7 @@ version = "0.7.3" description = "Python logging made (stupidly) simple" optional = false python-versions = "<4.0,>=3.5" +groups = ["main"] files = [ {file = "loguru-0.7.3-py3-none-any.whl", hash = "sha256:31a33c10c8e1e10422bfd431aeb5d351c7cf7fa671e3c4df004162264b28220c"}, {file = "loguru-0.7.3.tar.gz", hash = "sha256:19480589e77d47b8d85b2c827ad95d49bf31b0dcde16593892eb51dd18706eb6"}, @@ -1008,7 +1448,7 @@ colorama = {version = ">=0.3.4", markers = "sys_platform == \"win32\""} win32-setctime = {version = ">=1.0.0", markers = "sys_platform == \"win32\""} [package.extras] -dev = ["Sphinx (==8.1.3)", "build (==1.2.2)", "colorama (==0.4.5)", "colorama (==0.4.6)", "exceptiongroup (==1.1.3)", "freezegun (==1.1.0)", "freezegun (==1.5.0)", "mypy (==v0.910)", "mypy (==v0.971)", "mypy (==v1.13.0)", "mypy (==v1.4.1)", "myst-parser (==4.0.0)", "pre-commit (==4.0.1)", "pytest (==6.1.2)", "pytest (==8.3.2)", "pytest-cov (==2.12.1)", "pytest-cov (==5.0.0)", "pytest-cov (==6.0.0)", "pytest-mypy-plugins (==1.9.3)", "pytest-mypy-plugins (==3.1.0)", "sphinx-rtd-theme (==3.0.2)", "tox (==3.27.1)", "tox (==4.23.2)", "twine (==6.0.1)"] +dev = ["Sphinx (==8.1.3) ; python_version >= \"3.11\"", "build (==1.2.2) ; python_version >= \"3.11\"", "colorama (==0.4.5) ; python_version < \"3.8\"", "colorama (==0.4.6) ; python_version >= \"3.8\"", "exceptiongroup (==1.1.3) ; python_version >= \"3.7\" and python_version < \"3.11\"", "freezegun (==1.1.0) ; python_version < \"3.8\"", "freezegun (==1.5.0) ; python_version >= \"3.8\"", "mypy (==0.910) ; python_version < \"3.6\"", "mypy (==0.971) ; python_version == \"3.6\"", "mypy (==1.13.0) ; python_version >= \"3.8\"", "mypy (==1.4.1) ; python_version == \"3.7\"", "myst-parser (==4.0.0) ; python_version >= \"3.11\"", "pre-commit (==4.0.1) ; python_version >= \"3.9\"", "pytest (==6.1.2) ; python_version < \"3.8\"", "pytest (==8.3.2) ; python_version >= \"3.8\"", "pytest-cov (==2.12.1) ; python_version < \"3.8\"", "pytest-cov (==5.0.0) ; python_version == \"3.8\"", "pytest-cov (==6.0.0) ; python_version >= \"3.9\"", "pytest-mypy-plugins (==1.9.3) ; python_version >= \"3.6\" and python_version < \"3.8\"", "pytest-mypy-plugins (==3.1.0) ; python_version >= \"3.8\"", "sphinx-rtd-theme (==3.0.2) ; python_version >= \"3.11\"", "tox (==3.27.1) ; python_version < \"3.8\"", "tox (==4.23.2) ; python_version >= \"3.8\"", "twine (==6.0.1) ; python_version >= \"3.11\""] [[package]] name = "markdown" @@ -1016,14 +1456,12 @@ version = "3.8.2" description = "Python implementation of John Gruber's Markdown." optional = false python-versions = ">=3.9" +groups = ["main", "doc"] files = [ {file = "markdown-3.8.2-py3-none-any.whl", hash = "sha256:5c83764dbd4e00bdd94d85a19b8d55ccca20fe35b2e678a1422b380324dd5f24"}, {file = "markdown-3.8.2.tar.gz", hash = "sha256:247b9a70dd12e27f67431ce62523e675b866d254f900c4fe75ce3dda62237c45"}, ] -[package.dependencies] -importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} - [package.extras] docs = ["mdx_gh_links (>=0.2)", "mkdocs (>=1.6)", "mkdocs-gen-files", "mkdocs-literate-nav", "mkdocs-nature (>=0.6)", "mkdocs-section-index", "mkdocstrings[python]"] testing = ["coverage", "pyyaml"] @@ -1034,6 +1472,7 @@ version = "3.0.0" description = "Python port of markdown-it. Markdown parsing, done right!" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "markdown-it-py-3.0.0.tar.gz", hash = "sha256:e3f60a94fa066dc52ec76661e37c851cb232d92f9886b15cb560aaada2df8feb"}, {file = "markdown_it_py-3.0.0-py3-none-any.whl", hash = "sha256:355216845c60bd96232cd8d8c40e8f9765cc86f46880e43a8fd22dc1a1a8cab1"}, @@ -1058,6 +1497,7 @@ version = "3.0.2" description = "Safely add untrusted strings to HTML/XML markup." optional = false python-versions = ">=3.9" +groups = ["main", "dev", "doc"] files = [ {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:7e94c425039cde14257288fd61dcfb01963e658efbc0ff54f5306b06054700f8"}, {file = "MarkupSafe-3.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9e2d922824181480953426608b81967de705c3cef4d1af983af849d7bd619158"}, @@ -1122,12 +1562,161 @@ files = [ {file = "markupsafe-3.0.2.tar.gz", hash = "sha256:ee55d3edf80167e48ea11a923c7386f4669df67d7994554387f84e7d8b0a2bf0"}, ] +[[package]] +name = "matplotlib" +version = "3.10.9" +description = "Python plotting package" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +markers = "python_version == \"3.10\"" +files = [ + {file = "matplotlib-3.10.9-cp310-cp310-macosx_10_12_x86_64.whl", hash = "sha256:77210dce9cb8153dffc967efaae990543392563d5a376d4dd8539bebcb0ed217"}, + {file = "matplotlib-3.10.9-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:1e7698ac9868428e84d2c967424803b2472ff7167d9d6590d4204ed775343c3b"}, + {file = "matplotlib-3.10.9-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1aa972116abb4c9d201bf245620b433726cb6856f3bef6a78f776a00f5c92d37"}, + {file = "matplotlib-3.10.9-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ae2f11957b27ce53497dd4d7b235c4d4f1faf383dfb39d0c5beb833bff883294"}, + {file = "matplotlib-3.10.9-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b049278ddce116aaa1c1377ebf58adea909132dfce0281cf7e3a1ea9fc2e2c65"}, + {file = "matplotlib-3.10.9-cp310-cp310-win_amd64.whl", hash = "sha256:82834c3c292d24d3a8aae77cd2d20019de69d692a34a970e4fdb8d33e2ea3dda"}, + {file = "matplotlib-3.10.9-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:68cfdcede415f7c8f5577b03303dd94526cdb6d11036cecdc205e08733b2d2bb"}, + {file = "matplotlib-3.10.9-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:dfca0129678bd56379db26c52b5d77ed7de314c047492fbdc763aa7501710cfb"}, + {file = "matplotlib-3.10.9-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e436d155fa8a3399dc62683f8f5d0e2e50d25d0144a73edd73f82eec8f4abfb"}, + {file = "matplotlib-3.10.9-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:56fc0bd271b00025c6edfdc7c2dcd247372c8e1544971d62e1dc7c17367e8bf9"}, + {file = "matplotlib-3.10.9-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:a5a6104ed666402ba5106d7f36e0e0cdca4e8d7fa4d39708ca88019e2835a2eb"}, + {file = "matplotlib-3.10.9-cp311-cp311-win_amd64.whl", hash = "sha256:d730e984eddf56974c3e72b6129c7ca462ac38dc624338f4b0b23eb23ecba00f"}, + {file = "matplotlib-3.10.9-cp311-cp311-win_arm64.whl", hash = "sha256:51bf0ddbdc598e060d46c16b5590708f81a1624cefbaaf62f6a81bf9285b8c80"}, + {file = "matplotlib-3.10.9-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f0c3c28d9fbcc1fe7a03be236d73430cf6409c41fb2383a7ac52fe932b072cb1"}, + {file = "matplotlib-3.10.9-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:41cb28c2bd769aa3e98322c6ab09854cbcc52ab69d2759d681bba3e327b2b320"}, + {file = "matplotlib-3.10.9-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ae20801130378b82d647ff5047c07316295b68dc054ca6b3c13519d0ea624285"}, + {file = "matplotlib-3.10.9-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6c63ebcd8b4b169eb2f5c200552ae6b8be8999a005b6b507ed76fb8d7d674fe2"}, + {file = "matplotlib-3.10.9-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:d75d11c949914165976c621b2324f9ef162af7ebf4b057ddf95dd1dba7e5edcf"}, + {file = "matplotlib-3.10.9-cp312-cp312-win_amd64.whl", hash = "sha256:d091f9d758b34aaaaa6331d13574bf01891d903b3dec59bfff458ef7551de5d6"}, + {file = "matplotlib-3.10.9-cp312-cp312-win_arm64.whl", hash = "sha256:10cc5ce06d10231c36f40e875f3c7e8050362a4ee8f0ee5d29a6b3277d57bb42"}, + {file = "matplotlib-3.10.9-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b580440f1ff81a0e34122051a3dfabb7e4b7f9e380629929bde0eff9af72165f"}, + {file = "matplotlib-3.10.9-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:b1b745c489cd1a77a0dc1120a05dc87af9798faebc913601feb8c73d89bf2d1e"}, + {file = "matplotlib-3.10.9-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8f3bcac1ca5ed000a6f4337d47ba67dfddf37ed6a46c15fd7f014997f7bf865f"}, + {file = "matplotlib-3.10.9-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a8d66a55def891c33147ba3ba9bfcabf0b526a43764c818acbb4525e5ed0838"}, + {file = "matplotlib-3.10.9-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d843374407c4017a6403b59c6c81606773d136f3259d5b6da3131bc814542cc2"}, + {file = "matplotlib-3.10.9-cp313-cp313-win_amd64.whl", hash = "sha256:f4399f64b3e94cd500195490972ae1ee81170df1636fa15364d157d5bdd7b921"}, + {file = "matplotlib-3.10.9-cp313-cp313-win_arm64.whl", hash = "sha256:ba7b3b8ef09eab7df0e86e9ae086faa433efbfbdb46afcb3aa16aabf779469a8"}, + {file = "matplotlib-3.10.9-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:09218df8a93712bd6ea133e83a153c755448cf7868316c531cffcc43f69d1cc9"}, + {file = "matplotlib-3.10.9-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:82368699727bfb7b0182e1aa13082e3c08e092fa1a25d3e1fd92405bff96f6d4"}, + {file = "matplotlib-3.10.9-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3225f4e1edcb8c86c884ddf79ebe20ecd0a67d30188f279897554ccd8fded4dc"}, + {file = "matplotlib-3.10.9-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:de2445a0c6690d21b7eb6ce071cebad6d40a2e9bdf10d039074a96ba19797b99"}, + {file = "matplotlib-3.10.9-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:b2b9516251cb89ff618d757daec0e2ed1bf21248013844a853d87ef85ab3081d"}, + {file = "matplotlib-3.10.9-cp313-cp313t-win_amd64.whl", hash = "sha256:e9fae004b941b23ff2edcf1567a857ed77bafc8086ffa258190462328434faf8"}, + {file = "matplotlib-3.10.9-cp313-cp313t-win_arm64.whl", hash = "sha256:6b63d9c7c769b88ab81e10dc86e4e0607cf56817b9f9e6cf24b2a5f1693b8e38"}, + {file = "matplotlib-3.10.9-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:172db52c9e683f5d12eaf57f0f54834190e12581fe1cc2a19595a8f5acb4e77d"}, + {file = "matplotlib-3.10.9-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:97e35e8d39ccc85859095e01a53847432ba9a53ddf7986f7a54a11b73d0e143f"}, + {file = "matplotlib-3.10.9-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:aba1615dabe83188e19d4f75a253c6a08423e04c1425e64039f800050a69de6b"}, + {file = "matplotlib-3.10.9-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:34cf8167e023ad956c15f36302911d5406bd99a9862c1a8499ea6f7c0e015dc2"}, + {file = "matplotlib-3.10.9-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:59476c6d29d612b8e9bb6ce8c5b631be6ba8f9e3a2421f22a02b192c7dd28716"}, + {file = "matplotlib-3.10.9-cp314-cp314-win_amd64.whl", hash = "sha256:336b9acc64d309063126edcdaca00db9373af3c476bb94388fe9c5a53ad13e6f"}, + {file = "matplotlib-3.10.9-cp314-cp314-win_arm64.whl", hash = "sha256:2dc9477819ffd78ad12a20df1d9d6a6bd4fec6aaa9072681465fddca052f1456"}, + {file = "matplotlib-3.10.9-cp314-cp314t-macosx_10_13_x86_64.whl", hash = "sha256:da4e09638420548f31c354032a6250e473c68e5a4e96899b4844cf39ddea23fe"}, + {file = "matplotlib-3.10.9-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:345f6f68ecc8da0ca56fad2ea08fde1a115eda530079eca185d50a7bc3e146c6"}, + {file = "matplotlib-3.10.9-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4edcfbd8565339aa62f1cd4012f7180926fdbe71850f7b0d3c379c175cd6b66c"}, + {file = "matplotlib-3.10.9-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6be157fe17fc37cb95ac1d7374cf717ce9259616edec911a78d9d26dae8522d4"}, + {file = "matplotlib-3.10.9-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:4e42042d54db34fda4e95a7bd3e5789c2a995d2dad3eb8850232ee534092fbbf"}, + {file = "matplotlib-3.10.9-cp314-cp314t-win_amd64.whl", hash = "sha256:c27df8b3848f32a83d1767566595e43cfaa4460380974da06f4279a7ec143c39"}, + {file = "matplotlib-3.10.9-cp314-cp314t-win_arm64.whl", hash = "sha256:a49f1eadc84ca85fd72fa4e89e70e61bf86452df6f971af04b12c60761a0772c"}, + {file = "matplotlib-3.10.9-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:1872fb212a05b729e649754a72d5da61d03e0554d76e80303b6f83d1d2c0552b"}, + {file = "matplotlib-3.10.9-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:985f2238880e2e69093f588f5fe2e46771747febf0649f3cf7f7b7480875317f"}, + {file = "matplotlib-3.10.9-pp310-pypy310_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6640f75af2c6148293caa0a2b39dd806a492dd66c8a8b04035813e33d0fd2585"}, + {file = "matplotlib-3.10.9-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:42fb814efabe95c06c1994d8ab5a8385f43a249e23badd3ba931d4308e5bca20"}, + {file = "matplotlib-3.10.9-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:f76e640a5268850bfda54b5131b1b1941cc685e42c5fa98ed9f2d64038308cba"}, + {file = "matplotlib-3.10.9-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:3fc0364dfbe1d07f6d15c5ebd0c5bf89e126916e5a8667dd4a7a6e84c36653d4"}, + {file = "matplotlib-3.10.9.tar.gz", hash = "sha256:fd66508e8c6877d98e586654b608a0456db8d7e8a546eb1e2600efd957302358"}, +] + +[package.dependencies] +contourpy = ">=1.0.1" +cycler = ">=0.10" +fonttools = ">=4.22.0" +kiwisolver = ">=1.3.1" +numpy = ">=1.23" +packaging = ">=20.0" +pillow = ">=8" +pyparsing = ">=3" +python-dateutil = ">=2.7" + +[package.extras] +dev = ["meson-python (>=0.13.1,<0.17.0)", "pybind11 (>=2.13.2,!=2.13.3)", "setuptools (>=64)", "setuptools_scm (>=7,<10)"] + +[[package]] +name = "matplotlib" +version = "3.11.1" +description = "Python plotting package" +optional = false +python-versions = ">=3.11" +groups = ["dev"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "matplotlib-3.11.1-cp311-cp311-macosx_10_12_x86_64.whl", hash = "sha256:b7cf158e7add54a8d51ac9b5a84abd6d4e13ed4951b4f25f1c5139f41c2addb2"}, + {file = "matplotlib-3.11.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d2ace7273b9a5061a3b420918a16fae1f2dc5dfee1abcc13aba71b5d94b1820c"}, + {file = "matplotlib-3.11.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aee55e9041211bf84302ab55ec3965df18dd90ae19f8b58332a7feaf208bfe83"}, + {file = "matplotlib-3.11.1-cp311-cp311-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96f4bdeea33a8d15a071dbfe6d119451b1d719c733ac666d65357082901a9099"}, + {file = "matplotlib-3.11.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:b4c78ceb2f11bcac7389d305cda17aeb1f4586a857854ab5780bd3dd8dbfc407"}, + {file = "matplotlib-3.11.1-cp311-cp311-win_amd64.whl", hash = "sha256:7f33a781e12b1e53b278deb2f5373c2e55ec4f10727be3440c0cfb5cda9f944f"}, + {file = "matplotlib-3.11.1-cp311-cp311-win_arm64.whl", hash = "sha256:67e4c3cd578c65ebd81bdc09a1b6592ceafee6dfafe116dc85dfcb647b5bbb18"}, + {file = "matplotlib-3.11.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:e15ef41507f3d525f46154ac9e3ae785dacde9f20e593a25de8986267892ef74"}, + {file = "matplotlib-3.11.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:21a67b961a6d597bca54fae826cd20695ba4a6e4d05424a08da6e13e3176fd6b"}, + {file = "matplotlib-3.11.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ba8f811b8ddfac493734d6af0b2dff96919d0c28ca0d641858dab4262777c6ea"}, + {file = "matplotlib-3.11.1-cp312-cp312-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:c52f7ad20ef476806ed212380b1d54d20310c8b86bdc2c9a68b51f0024a44472"}, + {file = "matplotlib-3.11.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8b14eb22961fe865efb0e4ff167e333e428908b00115a8d800ccb65ee108e481"}, + {file = "matplotlib-3.11.1-cp312-cp312-win_amd64.whl", hash = "sha256:88a2a27dd9691ae448dfae4b26f59036be90c3c28757edd3553a29559d00859f"}, + {file = "matplotlib-3.11.1-cp312-cp312-win_arm64.whl", hash = "sha256:480194afceca4df2f137c2721227d3cba67121fbf4397b69cee7f83714b0a58a"}, + {file = "matplotlib-3.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:6771b0cd7838c6a857a7209814158c0ad09bfef878db3033dd82d70ad101f191"}, + {file = "matplotlib-3.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:2abdee5ffa2fe11b2d19f7a5c63b785fb7c28cc46c7bc1814156341d9d1a33e1"}, + {file = "matplotlib-3.11.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0a19dcf73406d3746d25a5ed42d713604c9a3e024d129b102852b0d941cb9f3"}, + {file = "matplotlib-3.11.1-cp313-cp313-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7389b77ed2ab0552f46d9a90b81b7b8e6dfcdc42adc36c37a0865799843e0e3e"}, + {file = "matplotlib-3.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:c90be0b73568da4f662afac580956a76e308437e641b4a45aa08925eeb67d95f"}, + {file = "matplotlib-3.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:68408341f2312836fbbdf6b3c78047f65b2d8752f5fd221c3e72d348f5b34f8b"}, + {file = "matplotlib-3.11.1-cp313-cp313-win_arm64.whl", hash = "sha256:0c1f44890d435c1b4ef52f701ad5828cb450ea97bcc83918fda6be74965d6cd2"}, + {file = "matplotlib-3.11.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:5e510088c27a89d53580a752f959146893563e63c330e161d159b0fee652af6f"}, + {file = "matplotlib-3.11.1-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:1524e2bdd48a93557aa47ddcfe9c225dfdd57d5a01a5c49128c20f0632980ee1"}, + {file = "matplotlib-3.11.1-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:11664c551345553db92e61cae6cf1376f138f8c47cafdf13b64b18f3e3e9e464"}, + {file = "matplotlib-3.11.1-cp313-cp313t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e1f8922ba31959cf6a9dfb51be64b7f7bc582801a3957dc0c2f3afcd3537adf"}, + {file = "matplotlib-3.11.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:83235693abde86e5e0129998f80ee39fc7f58e6d56a88fafb28a9278833e9d5f"}, + {file = "matplotlib-3.11.1-cp313-cp313t-win_amd64.whl", hash = "sha256:9a076f4fc5cdc43fdf510f5981418d25c2db4973418d9f22d8bb3dc8045ada78"}, + {file = "matplotlib-3.11.1-cp313-cp313t-win_arm64.whl", hash = "sha256:216fbb93a74add02ddb4cb38ef5348f59ac00b3e84567eaf16598772d40e150a"}, + {file = "matplotlib-3.11.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:30c492d4ba9448595b6fd8708c6725963f8148e25c0d8842948da5b05f0ee8d3"}, + {file = "matplotlib-3.11.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:ac104be2768ffdd8655db9e71b768cbb45f2b9aa7b450cf1595e8f65d3822319"}, + {file = "matplotlib-3.11.1-cp314-cp314-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6be943cb68bc6660ead58c55b3aa6366cba2ef7feb06460fbcce32360376f19f"}, + {file = "matplotlib-3.11.1-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5af0dcda57d471440a7b5b623e70e0a61003518443d9098f211a96ecfbbc25be"}, + {file = "matplotlib-3.11.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:3d3fd84082b1afbd9398466c81309e20045be20d48fe0fb18c43504d164cbbb2"}, + {file = "matplotlib-3.11.1-cp314-cp314-win_amd64.whl", hash = "sha256:9601a1e90be21e4884c53b4f3dc3ee0544654946f9975258d691f1c2e2f119c6"}, + {file = "matplotlib-3.11.1-cp314-cp314-win_arm64.whl", hash = "sha256:ae30c6109848ac0f9fa36c5d6270938487614c47ba31860bd5361266dabc5685"}, + {file = "matplotlib-3.11.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:dadfe80797174e2984aae3be0b77594a3c72d2c0a40fbd4a0de48d2728caf3ae"}, + {file = "matplotlib-3.11.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:89b193b255f4f6f7948dbcee3691f4f341ab05d9a8874a67b45ddb4182922eda"}, + {file = "matplotlib-3.11.1-cp314-cp314t-manylinux_2_26_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191163532cdefcb1571ca38a6d7e6474baccde64495783e6ba47aa07ec4b9bbb"}, + {file = "matplotlib-3.11.1-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:9fdf1c818ab05d0e74002091ddaf414478a3a449ec9d51c8976d45be7e3a01e2"}, + {file = "matplotlib-3.11.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:b937b9dba5f5f6c1e31c47abe2186c865c0914fd18f2ce0dfc39c9adcef5951d"}, + {file = "matplotlib-3.11.1-cp314-cp314t-win_amd64.whl", hash = "sha256:f2912f647f3fbe1ccf085f91e213936f9101bead81a5e670565b1f1b3712f4fb"}, + {file = "matplotlib-3.11.1-cp314-cp314t-win_arm64.whl", hash = "sha256:54d47b8ae8b579633a3902ca5b4ad6c1e132a5626d64447b2e22a66394e79987"}, + {file = "matplotlib-3.11.1-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:427258425f9a3fc4ed79a91f9e9b9aaf5a82cb6571e85dc14063cc6fbb993741"}, + {file = "matplotlib-3.11.1-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:1ac697e591c11b6ad04679a73c2d2f9980fe9d9f0311fb414a2e329706343dfb"}, + {file = "matplotlib-3.11.1-pp311-pypy311_pp73-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4b9ac2f1f607ecda2af90a5232beee2af7582fce1cc30c4b6a1b012dc21ee99"}, + {file = "matplotlib-3.11.1.tar.gz", hash = "sha256:69647db5746941c793d6e445a4cd349323ffb87d9cc958c2ad84a659b4832d30"}, +] + +[package.dependencies] +contourpy = ">=1.0.1" +cycler = ">=0.10" +fonttools = ">=4.28.2" +kiwisolver = ">=1.3.1" +numpy = ">=1.25" +packaging = ">=20.0" +pillow = ">=9" +pyparsing = ">=3" +python-dateutil = ">=2.7" + [[package]] name = "matplotlib-inline" version = "0.1.7" description = "Inline Matplotlib backend for Jupyter" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "matplotlib_inline-0.1.7-py3-none-any.whl", hash = "sha256:df192d39a4ff8f21b1895d72e6a13f5fcc5099f00fa84384e0ea28c2cc0653ca"}, {file = "matplotlib_inline-0.1.7.tar.gz", hash = "sha256:8423b23ec666be3d16e16b60bdd8ac4e86e840ebd1dd11a30b9f117f2fa0ab90"}, @@ -1142,6 +1731,7 @@ version = "0.1.2" description = "Markdown URL utilities" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8"}, {file = "mdurl-0.1.2.tar.gz", hash = "sha256:bb413d29f5eea38f31dd4754dd7377d4465116fb207585f97bf925588687c1ba"}, @@ -1153,6 +1743,7 @@ version = "1.3.4" description = "A deep merge function for 🐍." optional = false python-versions = ">=3.6" +groups = ["doc"] files = [ {file = "mergedeep-1.3.4-py3-none-any.whl", hash = "sha256:70775750742b25c0d8f36c55aed03d24c3384d17c951b3175d898bd778ef0307"}, {file = "mergedeep-1.3.4.tar.gz", hash = "sha256:0096d52e9dad9939c3d975a774666af186eda617e6ca84df4c94dec30004f2a8"}, @@ -1164,6 +1755,7 @@ version = "2.1.3" description = "Manage multiple versions of your MkDocs-powered documentation" optional = false python-versions = "*" +groups = ["doc"] files = [ {file = "mike-2.1.3-py3-none-any.whl", hash = "sha256:d90c64077e84f06272437b464735130d380703a76a5738b152932884c60c062a"}, {file = "mike-2.1.3.tar.gz", hash = "sha256:abd79b8ea483fb0275b7972825d3082e5ae67a41820f8d8a0dc7a3f49944e810"}, @@ -1189,6 +1781,7 @@ version = "1.6.1" description = "Project documentation with Markdown." optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "mkdocs-1.6.1-py3-none-any.whl", hash = "sha256:db91759624d1647f3f34aa0c3f327dd2601beae39a366d6e064c03468d35c20e"}, {file = "mkdocs-1.6.1.tar.gz", hash = "sha256:7b432f01d928c084353ab39c57282f29f92136665bdd6abf7c1ec8d822ef86f2"}, @@ -1198,7 +1791,6 @@ files = [ click = ">=7.0" colorama = {version = ">=0.4", markers = "platform_system == \"Windows\""} ghp-import = ">=1.0" -importlib-metadata = {version = ">=4.4", markers = "python_version < \"3.10\""} jinja2 = ">=2.11.1" markdown = ">=3.3.6" markupsafe = ">=2.0.1" @@ -1212,7 +1804,7 @@ watchdog = ">=2.0" [package.extras] i18n = ["babel (>=2.9.0)"] -min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4)", "ghp-import (==1.0)", "importlib-metadata (==4.4)", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] +min-versions = ["babel (==2.9.0)", "click (==7.0)", "colorama (==0.4) ; platform_system == \"Windows\"", "ghp-import (==1.0)", "importlib-metadata (==4.4) ; python_version < \"3.10\"", "jinja2 (==2.11.1)", "markdown (==3.3.6)", "markupsafe (==2.0.1)", "mergedeep (==1.3.4)", "mkdocs-get-deps (==0.2.0)", "packaging (==20.5)", "pathspec (==0.11.1)", "pyyaml (==5.1)", "pyyaml-env-tag (==0.1)", "watchdog (==2.0)"] [[package]] name = "mkdocs-autorefs" @@ -1220,6 +1812,7 @@ version = "1.4.2" description = "Automatically link across pages in MkDocs." optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "mkdocs_autorefs-1.4.2-py3-none-any.whl", hash = "sha256:83d6d777b66ec3c372a1aad4ae0cf77c243ba5bcda5bf0c6b8a2c5e7a3d89f13"}, {file = "mkdocs_autorefs-1.4.2.tar.gz", hash = "sha256:e2ebe1abd2b67d597ed19378c0fff84d73d1dbce411fce7a7cc6f161888b6749"}, @@ -1236,6 +1829,7 @@ version = "0.5.0" description = "MkDocs plugin to programmatically generate documentation pages during the build" optional = false python-versions = ">=3.7" +groups = ["doc"] files = [ {file = "mkdocs_gen_files-0.5.0-py3-none-any.whl", hash = "sha256:7ac060096f3f40bd19039e7277dd3050be9a453c8ac578645844d4d91d7978ea"}, {file = "mkdocs_gen_files-0.5.0.tar.gz", hash = "sha256:4c7cf256b5d67062a788f6b1d035e157fc1a9498c2399be9af5257d4ff4d19bc"}, @@ -1250,13 +1844,13 @@ version = "0.2.0" description = "MkDocs extension that lists all dependencies according to a mkdocs.yml file" optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "mkdocs_get_deps-0.2.0-py3-none-any.whl", hash = "sha256:2bf11d0b133e77a0dd036abeeb06dec8775e46efa526dc70667d8863eefc6134"}, {file = "mkdocs_get_deps-0.2.0.tar.gz", hash = "sha256:162b3d129c7fad9b19abfdcb9c1458a651628e4b1dea628ac68790fb3061c60c"}, ] [package.dependencies] -importlib-metadata = {version = ">=4.3", markers = "python_version < \"3.10\""} mergedeep = ">=1.3.4" platformdirs = ">=2.2.0" pyyaml = ">=5.1" @@ -1267,6 +1861,7 @@ version = "9.6.16" description = "Documentation that simply works" optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "mkdocs_material-9.6.16-py3-none-any.whl", hash = "sha256:8d1a1282b892fe1fdf77bfeb08c485ba3909dd743c9ba69a19a40f637c6ec18c"}, {file = "mkdocs_material-9.6.16.tar.gz", hash = "sha256:d07011df4a5c02ee0877496d9f1bfc986cfb93d964799b032dd99fe34c0e9d19"}, @@ -1296,6 +1891,7 @@ version = "1.3.1" description = "Extension pack for Python Markdown and MkDocs Material." optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "mkdocs_material_extensions-1.3.1-py3-none-any.whl", hash = "sha256:adff8b62700b25cb77b53358dad940f3ef973dd6db797907c49e3c2ef3ab4e31"}, {file = "mkdocs_material_extensions-1.3.1.tar.gz", hash = "sha256:10c9511cea88f568257f960358a467d12b970e1f7b2c0e5fb2bb48cab1928443"}, @@ -1307,6 +1903,7 @@ version = "1.2.2" description = "A MkDocs plugin for dynamic page redirects to prevent broken links" optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "mkdocs_redirects-1.2.2-py3-none-any.whl", hash = "sha256:7dbfa5647b79a3589da4401403d69494bd1f4ad03b9c15136720367e1f340ed5"}, {file = "mkdocs_redirects-1.2.2.tar.gz", hash = "sha256:3094981b42ffab29313c2c1b8ac3969861109f58b2dd58c45fc81cd44bfa0095"}, @@ -1321,6 +1918,7 @@ version = "0.24.3" description = "Automatic documentation from sources, for MkDocs." optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "mkdocstrings-0.24.3-py3-none-any.whl", hash = "sha256:5c9cf2a32958cd161d5428699b79c8b0988856b0d4a8c5baf8395fc1bf4087c3"}, {file = "mkdocstrings-0.24.3.tar.gz", hash = "sha256:f327b234eb8d2551a306735436e157d0a22d45f79963c60a8b585d5f7a94c1d2"}, @@ -1328,7 +1926,6 @@ files = [ [package.dependencies] click = ">=7.0" -importlib-metadata = {version = ">=4.6", markers = "python_version < \"3.10\""} Jinja2 = ">=2.11.1" Markdown = ">=3.3" MarkupSafe = ">=1.1" @@ -1337,7 +1934,6 @@ mkdocs-autorefs = ">=0.3.1" mkdocstrings-python = {version = ">=0.5.2", optional = true, markers = "extra == \"python\""} platformdirs = ">=2.2.0" pymdown-extensions = ">=6.3" -typing-extensions = {version = ">=4.1", markers = "python_version < \"3.10\""} [package.extras] crystal = ["mkdocstrings-crystal (>=0.3.4)"] @@ -1350,6 +1946,7 @@ version = "0.9.0" description = "A Python handler for mkdocstrings." optional = false python-versions = ">=3.7" +groups = ["doc"] files = [ {file = "mkdocstrings-python-0.9.0.tar.gz", hash = "sha256:da0a54d7d46523a25a5227f0ecc74b491291bd9d36fc71445bfb0ea64283e287"}, {file = "mkdocstrings_python-0.9.0-py3-none-any.whl", hash = "sha256:00e02b5d3d444f9abdec2398f9ba0c73e15deab78685f793f5801fd4d62a5b6f"}, @@ -1359,44 +1956,13 @@ files = [ griffe = ">=0.24" mkdocstrings = ">=0.20" -[[package]] -name = "ml-dtypes" -version = "0.4.1" -description = "" -optional = false -python-versions = ">=3.9" -files = [ - {file = "ml_dtypes-0.4.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:1fe8b5b5e70cd67211db94b05cfd58dace592f24489b038dc6f9fe347d2e07d5"}, - {file = "ml_dtypes-0.4.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8c09a6d11d8475c2a9fd2bc0695628aec105f97cab3b3a3fb7c9660348ff7d24"}, - {file = "ml_dtypes-0.4.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5e8f75fa371020dd30f9196e7d73babae2abd51cf59bdd56cb4f8de7e13354"}, - {file = "ml_dtypes-0.4.1-cp310-cp310-win_amd64.whl", hash = "sha256:15fdd922fea57e493844e5abb930b9c0bd0af217d9edd3724479fc3d7ce70e3f"}, - {file = "ml_dtypes-0.4.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2d55b588116a7085d6e074cf0cdb1d6fa3875c059dddc4d2c94a4cc81c23e975"}, - {file = "ml_dtypes-0.4.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e138a9b7a48079c900ea969341a5754019a1ad17ae27ee330f7ebf43f23877f9"}, - {file = "ml_dtypes-0.4.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74c6cfb5cf78535b103fde9ea3ded8e9f16f75bc07789054edc7776abfb3d752"}, - {file = "ml_dtypes-0.4.1-cp311-cp311-win_amd64.whl", hash = "sha256:274cc7193dd73b35fb26bef6c5d40ae3eb258359ee71cd82f6e96a8c948bdaa6"}, - {file = "ml_dtypes-0.4.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:827d3ca2097085cf0355f8fdf092b888890bb1b1455f52801a2d7756f056f54b"}, - {file = "ml_dtypes-0.4.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:772426b08a6172a891274d581ce58ea2789cc8abc1c002a27223f314aaf894e7"}, - {file = "ml_dtypes-0.4.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:126e7d679b8676d1a958f2651949fbfa182832c3cd08020d8facd94e4114f3e9"}, - {file = "ml_dtypes-0.4.1-cp312-cp312-win_amd64.whl", hash = "sha256:df0fb650d5c582a9e72bb5bd96cfebb2cdb889d89daff621c8fbc60295eba66c"}, - {file = "ml_dtypes-0.4.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:e35e486e97aee577d0890bc3bd9e9f9eece50c08c163304008587ec8cfe7575b"}, - {file = "ml_dtypes-0.4.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:560be16dc1e3bdf7c087eb727e2cf9c0e6a3d87e9f415079d2491cc419b3ebf5"}, - {file = "ml_dtypes-0.4.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ad0b757d445a20df39035c4cdeed457ec8b60d236020d2560dbc25887533cf50"}, - {file = "ml_dtypes-0.4.1-cp39-cp39-win_amd64.whl", hash = "sha256:ef0d7e3fece227b49b544fa69e50e607ac20948f0043e9f76b44f35f229ea450"}, - {file = "ml_dtypes-0.4.1.tar.gz", hash = "sha256:fad5f2de464fd09127e49b7fd1252b9006fb43d2edc1ff112d390c324af5ca7a"}, -] - -[package.dependencies] -numpy = {version = ">=1.26.0", markers = "python_version >= \"3.12\""} - -[package.extras] -dev = ["absl-py", "pyink", "pylint (>=2.6.0)", "pytest", "pytest-xdist"] - [[package]] name = "ml-dtypes" version = "0.5.3" description = "ml_dtypes is a stand-alone implementation of several NumPy dtype extensions used in machine learning." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "ml_dtypes-0.5.3-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:0a1d68a7cb53e3f640b2b6a34d12c0542da3dd935e560fdf463c0c77f339fc20"}, {file = "ml_dtypes-0.5.3-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:0cd5a6c711b5350f3cbc2ac28def81cd1c580075ccb7955e61e9d8f4bfd40d24"}, @@ -1437,10 +2003,10 @@ files = [ [package.dependencies] numpy = [ - {version = ">=1.21.2", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, - {version = ">=1.21", markers = "python_version < \"3.10\""}, - {version = ">=1.23.3", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, - {version = ">=1.26.0", markers = "python_version >= \"3.12\" and python_version < \"3.13\""}, + {version = ">=1.21.2", markers = "python_version >= \"3.10\""}, + {version = ">=1.23.3", markers = "python_version >= \"3.11\""}, + {version = ">=2.1.0", markers = "python_version >= \"3.13\""}, + {version = ">=1.26.0", markers = "python_version == \"3.12\""}, ] [package.extras] @@ -1452,17 +2018,46 @@ version = "0.1.0" description = "A simple utility to separate the implementation of your Python package and its public API surface." optional = false python-versions = "*" +groups = ["main"] files = [ {file = "namex-0.1.0-py3-none-any.whl", hash = "sha256:e2012a474502f1e2251267062aae3114611f07df4224b6e06334c57b0f2ce87c"}, {file = "namex-0.1.0.tar.gz", hash = "sha256:117f03ccd302cc48e3f5c58a296838f6b89c83455ab8683a1e85f2a430aa4306"}, ] +[[package]] +name = "narwhals" +version = "2.25.0" +description = "Extremely lightweight compatibility layer between dataframe libraries" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "narwhals-2.25.0-py3-none-any.whl", hash = "sha256:1f0f403e8c7e4463cde9bfe78b12fdd809e3ae3dda6d9b2f802934fb9c7a6a8f"}, + {file = "narwhals-2.25.0.tar.gz", hash = "sha256:62c036c810662bf7820b7737077176313bc59350eeeefb808510f388c743e4b2"}, +] + +[package.extras] +cudf = ["cudf-cu12 (>=24.10.0) ; sys_platform == \"linux\""] +dask = ["dask[dataframe] (>=2024.8)"] +duckdb = ["duckdb (>=1.1)"] +ibis = ["ibis-framework (>=6.0.0)", "packaging (>=21.3)", "pyarrow-hotfix (>=0.7)"] +modin = ["modin (>=0.22.0)"] +pandas = ["pandas (>=1.3.4)"] +polars = ["polars (>=0.20.4)"] +pyarrow = ["pyarrow (>=13.0.0)"] +pyspark = ["pyspark (>=3.5.0)"] +pyspark-connect = ["pyspark[connect] (>=3.5.0)"] +sql = ["narwhals[duckdb]", "sqlparse (>=0.5.5)"] +sqlframe = ["sqlframe (>=3.22.0,!=3.39.3)"] + [[package]] name = "nest-asyncio" version = "1.6.0" description = "Patch asyncio to allow nested event loops" optional = false python-versions = ">=3.5" +groups = ["dev"] files = [ {file = "nest_asyncio-1.6.0-py3-none-any.whl", hash = "sha256:87af6efd6b5e897c81050477ef65c62e2b2f35d51703cae01aff2905b1852e1c"}, {file = "nest_asyncio-1.6.0.tar.gz", hash = "sha256:6f172d5449aca15afd6c646851f4e31e02c598d553a667e38cafa997cfec55fe"}, @@ -1474,6 +2069,7 @@ version = "1.9.1" description = "Node.js virtual environment builder" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +groups = ["dev"] files = [ {file = "nodeenv-1.9.1-py2.py3-none-any.whl", hash = "sha256:ba11c9782d29c27c70ffbdda2d7415098754709be8a7056d79a737cd901155c9"}, {file = "nodeenv-1.9.1.tar.gz", hash = "sha256:6ec12890a2dab7946721edbfbcd91f3319c6ccc9aec47be7c7e6b7011ee6645f"}, @@ -1481,56 +2077,67 @@ files = [ [[package]] name = "numpy" -version = "2.0.2" +version = "2.1.3" description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.9" -files = [ - {file = "numpy-2.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:51129a29dbe56f9ca83438b706e2e69a39892b5eda6cedcb6b0c9fdc9b0d3ece"}, - {file = "numpy-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f15975dfec0cf2239224d80e32c3170b1d168335eaedee69da84fbe9f1f9cd04"}, - {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:8c5713284ce4e282544c68d1c3b2c7161d38c256d2eefc93c1d683cf47683e66"}, - {file = "numpy-2.0.2-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:becfae3ddd30736fe1889a37f1f580e245ba79a5855bff5f2a29cb3ccc22dd7b"}, - {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2da5960c3cf0df7eafefd806d4e612c5e19358de82cb3c343631188991566ccd"}, - {file = "numpy-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:496f71341824ed9f3d2fd36cf3ac57ae2e0165c143b55c3a035ee219413f3318"}, - {file = "numpy-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a61ec659f68ae254e4d237816e33171497e978140353c0c2038d46e63282d0c8"}, - {file = "numpy-2.0.2-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d731a1c6116ba289c1e9ee714b08a8ff882944d4ad631fd411106a30f083c326"}, - {file = "numpy-2.0.2-cp310-cp310-win32.whl", hash = "sha256:984d96121c9f9616cd33fbd0618b7f08e0cfc9600a7ee1d6fd9b239186d19d97"}, - {file = "numpy-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:c7b0be4ef08607dd04da4092faee0b86607f111d5ae68036f16cc787e250a131"}, - {file = "numpy-2.0.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:49ca4decb342d66018b01932139c0961a8f9ddc7589611158cb3c27cbcf76448"}, - {file = "numpy-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:11a76c372d1d37437857280aa142086476136a8c0f373b2e648ab2c8f18fb195"}, - {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:807ec44583fd708a21d4a11d94aedf2f4f3c3719035c76a2bbe1fe8e217bdc57"}, - {file = "numpy-2.0.2-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:8cafab480740e22f8d833acefed5cc87ce276f4ece12fdaa2e8903db2f82897a"}, - {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a15f476a45e6e5a3a79d8a14e62161d27ad897381fecfa4a09ed5322f2085669"}, - {file = "numpy-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:13e689d772146140a252c3a28501da66dfecd77490b498b168b501835041f951"}, - {file = "numpy-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:9ea91dfb7c3d1c56a0e55657c0afb38cf1eeae4544c208dc465c3c9f3a7c09f9"}, - {file = "numpy-2.0.2-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:c1c9307701fec8f3f7a1e6711f9089c06e6284b3afbbcd259f7791282d660a15"}, - {file = "numpy-2.0.2-cp311-cp311-win32.whl", hash = "sha256:a392a68bd329eafac5817e5aefeb39038c48b671afd242710b451e76090e81f4"}, - {file = "numpy-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:286cd40ce2b7d652a6f22efdfc6d1edf879440e53e76a75955bc0c826c7e64dc"}, - {file = "numpy-2.0.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:df55d490dea7934f330006d0f81e8551ba6010a5bf035a249ef61a94f21c500b"}, - {file = "numpy-2.0.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8df823f570d9adf0978347d1f926b2a867d5608f434a7cff7f7908c6570dcf5e"}, - {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:9a92ae5c14811e390f3767053ff54eaee3bf84576d99a2456391401323f4ec2c"}, - {file = "numpy-2.0.2-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:a842d573724391493a97a62ebbb8e731f8a5dcc5d285dfc99141ca15a3302d0c"}, - {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c05e238064fc0610c840d1cf6a13bf63d7e391717d247f1bf0318172e759e692"}, - {file = "numpy-2.0.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0123ffdaa88fa4ab64835dcbde75dcdf89c453c922f18dced6e27c90d1d0ec5a"}, - {file = "numpy-2.0.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:96a55f64139912d61de9137f11bf39a55ec8faec288c75a54f93dfd39f7eb40c"}, - {file = "numpy-2.0.2-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:ec9852fb39354b5a45a80bdab5ac02dd02b15f44b3804e9f00c556bf24b4bded"}, - {file = "numpy-2.0.2-cp312-cp312-win32.whl", hash = "sha256:671bec6496f83202ed2d3c8fdc486a8fc86942f2e69ff0e986140339a63bcbe5"}, - {file = "numpy-2.0.2-cp312-cp312-win_amd64.whl", hash = "sha256:cfd41e13fdc257aa5778496b8caa5e856dc4896d4ccf01841daee1d96465467a"}, - {file = "numpy-2.0.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9059e10581ce4093f735ed23f3b9d283b9d517ff46009ddd485f1747eb22653c"}, - {file = "numpy-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:423e89b23490805d2a5a96fe40ec507407b8ee786d66f7328be214f9679df6dd"}, - {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_arm64.whl", hash = "sha256:2b2955fa6f11907cf7a70dab0d0755159bca87755e831e47932367fc8f2f2d0b"}, - {file = "numpy-2.0.2-cp39-cp39-macosx_14_0_x86_64.whl", hash = "sha256:97032a27bd9d8988b9a97a8c4d2c9f2c15a81f61e2f21404d7e8ef00cb5be729"}, - {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e795a8be3ddbac43274f18588329c72939870a16cae810c2b73461c40718ab1"}, - {file = "numpy-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f26b258c385842546006213344c50655ff1555a9338e2e5e02a0756dc3e803dd"}, - {file = "numpy-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5fec9451a7789926bcf7c2b8d187292c9f93ea30284802a0ab3f5be8ab36865d"}, - {file = "numpy-2.0.2-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:9189427407d88ff25ecf8f12469d4d39d35bee1db5d39fc5c168c6f088a6956d"}, - {file = "numpy-2.0.2-cp39-cp39-win32.whl", hash = "sha256:905d16e0c60200656500c95b6b8dca5d109e23cb24abc701d41c02d74c6b3afa"}, - {file = "numpy-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:a3f4ab0caa7f053f6797fcd4e1e25caee367db3112ef2b6ef82d749530768c73"}, - {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:7f0a0c6f12e07fa94133c8a67404322845220c06a9e80e85999afe727f7438b8"}, - {file = "numpy-2.0.2-pp39-pypy39_pp73-macosx_14_0_x86_64.whl", hash = "sha256:312950fdd060354350ed123c0e25a71327d3711584beaef30cdaa93320c392d4"}, - {file = "numpy-2.0.2-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26df23238872200f63518dd2aa984cfca675d82469535dc7162dc2ee52d9dd5c"}, - {file = "numpy-2.0.2-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:a46288ec55ebbd58947d31d72be2c63cbf839f0a63b49cb755022310792a3385"}, - {file = "numpy-2.0.2.tar.gz", hash = "sha256:883c987dee1880e2a864ab0dc9892292582510604156762362d9326444636e78"}, +python-versions = ">=3.10" +groups = ["main", "dev"] +files = [ + {file = "numpy-2.1.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c894b4305373b9c5576d7a12b473702afdf48ce5369c074ba304cc5ad8730dff"}, + {file = "numpy-2.1.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b47fbb433d3260adcd51eb54f92a2ffbc90a4595f8970ee00e064c644ac788f5"}, + {file = "numpy-2.1.3-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:825656d0743699c529c5943554d223c021ff0494ff1442152ce887ef4f7561a1"}, + {file = "numpy-2.1.3-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:6a4825252fcc430a182ac4dee5a505053d262c807f8a924603d411f6718b88fd"}, + {file = "numpy-2.1.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e711e02f49e176a01d0349d82cb5f05ba4db7d5e7e0defd026328e5cfb3226d3"}, + {file = "numpy-2.1.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:78574ac2d1a4a02421f25da9559850d59457bac82f2b8d7a44fe83a64f770098"}, + {file = "numpy-2.1.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:c7662f0e3673fe4e832fe07b65c50342ea27d989f92c80355658c7f888fcc83c"}, + {file = "numpy-2.1.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:fa2d1337dc61c8dc417fbccf20f6d1e139896a30721b7f1e832b2bb6ef4eb6c4"}, + {file = "numpy-2.1.3-cp310-cp310-win32.whl", hash = "sha256:72dcc4a35a8515d83e76b58fdf8113a5c969ccd505c8a946759b24e3182d1f23"}, + {file = "numpy-2.1.3-cp310-cp310-win_amd64.whl", hash = "sha256:ecc76a9ba2911d8d37ac01de72834d8849e55473457558e12995f4cd53e778e0"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4d1167c53b93f1f5d8a139a742b3c6f4d429b54e74e6b57d0eff40045187b15d"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c80e4a09b3d95b4e1cac08643f1152fa71a0a821a2d4277334c88d54b2219a41"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:576a1c1d25e9e02ed7fa5477f30a127fe56debd53b8d2c89d5578f9857d03ca9"}, + {file = "numpy-2.1.3-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:973faafebaae4c0aaa1a1ca1ce02434554d67e628b8d805e61f874b84e136b09"}, + {file = "numpy-2.1.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:762479be47a4863e261a840e8e01608d124ee1361e48b96916f38b119cfda04a"}, + {file = "numpy-2.1.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc6f24b3d1ecc1eebfbf5d6051faa49af40b03be1aaa781ebdadcbc090b4539b"}, + {file = "numpy-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:17ee83a1f4fef3c94d16dc1802b998668b5419362c8a4f4e8a491de1b41cc3ee"}, + {file = "numpy-2.1.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:15cb89f39fa6d0bdfb600ea24b250e5f1a3df23f901f51c8debaa6a5d122b2f0"}, + {file = "numpy-2.1.3-cp311-cp311-win32.whl", hash = "sha256:d9beb777a78c331580705326d2367488d5bc473b49a9bc3036c154832520aca9"}, + {file = "numpy-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:d89dd2b6da69c4fff5e39c28a382199ddedc3a5be5390115608345dec660b9e2"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:f55ba01150f52b1027829b50d70ef1dafd9821ea82905b63936668403c3b471e"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:13138eadd4f4da03074851a698ffa7e405f41a0845a6b1ad135b81596e4e9958"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a6b46587b14b888e95e4a24d7b13ae91fa22386c199ee7b418f449032b2fa3b8"}, + {file = "numpy-2.1.3-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:0fa14563cc46422e99daef53d725d0c326e99e468a9320a240affffe87852564"}, + {file = "numpy-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8637dcd2caa676e475503d1f8fdb327bc495554e10838019651b76d17b98e512"}, + {file = "numpy-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2312b2aa89e1f43ecea6da6ea9a810d06aae08321609d8dc0d0eda6d946a541b"}, + {file = "numpy-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:a38c19106902bb19351b83802531fea19dee18e5b37b36454f27f11ff956f7fc"}, + {file = "numpy-2.1.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:02135ade8b8a84011cbb67dc44e07c58f28575cf9ecf8ab304e51c05528c19f0"}, + {file = "numpy-2.1.3-cp312-cp312-win32.whl", hash = "sha256:e6988e90fcf617da2b5c78902fe8e668361b43b4fe26dbf2d7b0f8034d4cafb9"}, + {file = "numpy-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:0d30c543f02e84e92c4b1f415b7c6b5326cbe45ee7882b6b77db7195fb971e3a"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:96fe52fcdb9345b7cd82ecd34547fca4321f7656d500eca497eb7ea5a926692f"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:f653490b33e9c3a4c1c01d41bc2aef08f9475af51146e4a7710c450cf9761598"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:dc258a761a16daa791081d026f0ed4399b582712e6fc887a95af09df10c5ca57"}, + {file = "numpy-2.1.3-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:016d0f6f5e77b0f0d45d77387ffa4bb89816b57c835580c3ce8e099ef830befe"}, + {file = "numpy-2.1.3-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c181ba05ce8299c7aa3125c27b9c2167bca4a4445b7ce73d5febc411ca692e43"}, + {file = "numpy-2.1.3-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5641516794ca9e5f8a4d17bb45446998c6554704d888f86df9b200e66bdcce56"}, + {file = "numpy-2.1.3-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ea4dedd6e394a9c180b33c2c872b92f7ce0f8e7ad93e9585312b0c5a04777a4a"}, + {file = "numpy-2.1.3-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:b0df3635b9c8ef48bd3be5f862cf71b0a4716fa0e702155c45067c6b711ddcef"}, + {file = "numpy-2.1.3-cp313-cp313-win32.whl", hash = "sha256:50ca6aba6e163363f132b5c101ba078b8cbd3fa92c7865fd7d4d62d9779ac29f"}, + {file = "numpy-2.1.3-cp313-cp313-win_amd64.whl", hash = "sha256:747641635d3d44bcb380d950679462fae44f54b131be347d5ec2bce47d3df9ed"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:996bb9399059c5b82f76b53ff8bb686069c05acc94656bb259b1d63d04a9506f"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:45966d859916ad02b779706bb43b954281db43e185015df6eb3323120188f9e4"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:baed7e8d7481bfe0874b566850cb0b85243e982388b7b23348c6db2ee2b2ae8e"}, + {file = "numpy-2.1.3-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:a9f7f672a3388133335589cfca93ed468509cb7b93ba3105fce780d04a6576a0"}, + {file = "numpy-2.1.3-cp313-cp313t-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d7aac50327da5d208db2eec22eb11e491e3fe13d22653dce51b0f4109101b408"}, + {file = "numpy-2.1.3-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4394bc0dbd074b7f9b52024832d16e019decebf86caf909d94f6b3f77a8ee3b6"}, + {file = "numpy-2.1.3-cp313-cp313t-musllinux_1_1_x86_64.whl", hash = "sha256:50d18c4358a0a8a53f12a8ba9d772ab2d460321e6a93d6064fc22443d189853f"}, + {file = "numpy-2.1.3-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:14e253bd43fc6b37af4921b10f6add6925878a42a0c5fe83daee390bca80bc17"}, + {file = "numpy-2.1.3-cp313-cp313t-win32.whl", hash = "sha256:08788d27a5fd867a663f6fc753fd7c3ad7e92747efc73c53bca2f19f8bc06f48"}, + {file = "numpy-2.1.3-cp313-cp313t-win_amd64.whl", hash = "sha256:2564fbdf2b99b3f815f2107c1bbc93e2de8ee655a69c261363a1172a79a257d4"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:4f2015dfe437dfebbfce7c85c7b53d81ba49e71ba7eadbf1df40c915af75979f"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-macosx_14_0_x86_64.whl", hash = "sha256:3522b0dfe983a575e6a9ab3a4a4dfe156c3e428468ff08ce582b9bb6bd1d71d4"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c006b607a865b07cd981ccb218a04fc86b600411d83d6fc261357f1c0966755d"}, + {file = "numpy-2.1.3-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:e14e26956e6f1696070788252dcdff11b4aca4c3e8bd166e0df1bb8f315a67cb"}, + {file = "numpy-2.1.3.tar.gz", hash = "sha256:aa08e04e08aaf974d4458def539dece0d28146d866a39da5639596f4921fd761"}, ] [[package]] @@ -1539,6 +2146,7 @@ version = "3.4.0" description = "Path optimization of einsum functions." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "opt_einsum-3.4.0-py3-none-any.whl", hash = "sha256:69bb92469f86a1565195ece4ac0323943e83477171b91d24c35afe028a90d7cd"}, {file = "opt_einsum-3.4.0.tar.gz", hash = "sha256:96ca72f1b886d148241348783498194c577fa30a8faac108586b14f1ba4473ac"}, @@ -1550,6 +2158,7 @@ version = "0.17.0" description = "Optimized PyTree Utilities." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "optree-0.17.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:85ec183b8eec6efc9a5572c2a84c62214c949555efbc69ca2381aca6048d08df"}, {file = "optree-0.17.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:6e77b6e0b7bb3ecfeb9a92ba605ef21b39bff38829b745af993e2e2b474322e2"}, @@ -1661,6 +2270,7 @@ version = "25.0" description = "Core utilities for Python packages" optional = false python-versions = ">=3.8" +groups = ["main", "dev", "doc"] files = [ {file = "packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484"}, {file = "packaging-25.0.tar.gz", hash = "sha256:d443872c98d677bf60f6a1f2f8c1cb748e8fe762d2bf9d3148b5599295b0fc4f"}, @@ -1672,6 +2282,7 @@ version = "0.5.7" description = "Divides large result sets into pages for easier browsing" optional = false python-versions = "*" +groups = ["doc"] files = [ {file = "paginate-0.5.7-py2.py3-none-any.whl", hash = "sha256:b885e2af73abcf01d9559fd5216b57ef722f8c42affbb63942377668e35c7591"}, {file = "paginate-0.5.7.tar.gz", hash = "sha256:22bd083ab41e1a8b4f3690544afb2c60c25e5c9a63a30fa2f483f6c60c8e5945"}, @@ -1687,6 +2298,7 @@ version = "2.3.1" description = "Powerful data structures for data analysis, time series, and statistics" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "pandas-2.3.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:22c2e866f7209ebc3a8f08d75766566aae02bcc91d196935a1d9e59c7b990ac9"}, {file = "pandas-2.3.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3583d348546201aff730c8c47e49bc159833f971c2899d6097bce68b9112a4f1"}, @@ -1773,6 +2385,7 @@ version = "0.9.0" description = "Parameterized testing with any Python test framework" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "parameterized-0.9.0-py2.py3-none-any.whl", hash = "sha256:4e0758e3d41bea3bbd05ec14fc2c24736723f243b28d702081aef438c9372b1b"}, {file = "parameterized-0.9.0.tar.gz", hash = "sha256:7fc905272cefa4f364c1a3429cbbe9c0f98b793988efb5bf90aac80f08db09b1"}, @@ -1787,6 +2400,7 @@ version = "0.8.4" description = "A Python Parser" optional = false python-versions = ">=3.6" +groups = ["dev"] files = [ {file = "parso-0.8.4-py2.py3-none-any.whl", hash = "sha256:a418670a20291dacd2dddc80c377c5c3791378ee1e8d12bffc35420643d43f18"}, {file = "parso-0.8.4.tar.gz", hash = "sha256:eb3a7b58240fb99099a345571deecc0f9540ea5f4dd2fe14c2a99d6b281ab92d"}, @@ -1802,6 +2416,7 @@ version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." optional = false python-versions = ">=3.8" +groups = ["doc"] files = [ {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, @@ -1813,6 +2428,8 @@ version = "4.9.0" description = "Pexpect allows easy control of interactive console applications." optional = false python-versions = "*" +groups = ["dev"] +markers = "sys_platform != \"win32\"" files = [ {file = "pexpect-4.9.0-py2.py3-none-any.whl", hash = "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523"}, {file = "pexpect-4.9.0.tar.gz", hash = "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"}, @@ -1821,12 +2438,118 @@ files = [ [package.dependencies] ptyprocess = ">=0.5" +[[package]] +name = "pillow" +version = "12.3.0" +description = "Python Imaging Library (fork)" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +files = [ + {file = "pillow-12.3.0-cp310-cp310-macosx_10_10_x86_64.whl", hash = "sha256:6c0016e7b354317c4e9e525b937ac8596c38d2d232b419529b9cd7a1cd46e39a"}, + {file = "pillow-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bcc33feacfaefce60c12fd500a277533bdc02b10a19f7f6d348763d8140bbba7"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5594fc43d548a7ed94949d139aa1341b270f1863f11cfd37f5a6c8b778a6b67f"}, + {file = "pillow-12.3.0-cp310-cp310-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f0606c8bf2cdefea14a43530f7657cbbb7ecf1c4222512492ef4a4434a9501ec"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:85f998ea1848bc6757289e739cfbdda3a04adfd58b02fc018ce54d754a5ce468"}, + {file = "pillow-12.3.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:25b9b82bb22e6e2b3cd07b39c68b7b862001226cb3dff7130d1cb914121b39ed"}, + {file = "pillow-12.3.0-cp310-cp310-win32.whl", hash = "sha256:37dc8f7bbb66efe481bb60defacef820c950c24713fb44962ed6aa2a50966de1"}, + {file = "pillow-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:300557495eb45ebb8aec96c2da9c4be642fbf7cd937278b4013ba894ea8eb0eb"}, + {file = "pillow-12.3.0-cp310-cp310-win_arm64.whl", hash = "sha256:514435a37670e3e5e08f3945b68718b6ed329bb84367777e16f9f4dfe1e61a0f"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_10_10_x86_64.whl", hash = "sha256:00808c5e14ef63ac5161091d242999076604ff74b883423a11e5d7bbb38bf756"}, + {file = "pillow-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37d6d0a00072fd2948eb22bce7e1475f34569d90c87c59f7a2ec59541b77f7a6"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:bcb46e2f9feff8d06323983bd83ed00c201fdcab3d74973e7072a889b3979fcd"}, + {file = "pillow-12.3.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:23d27a3e0307ec2244cc51e7287b919aa68d097504ebe19df4e76a98a3eea5bd"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:4f883547d4b7f0495ebe7056b0cc2aea76094e7a4abc8e933540f3271df27d9c"}, + {file = "pillow-12.3.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:236ff70b9312fb68943c703aa842ca6a758abfa45ac187a5e7c1452e96ef72b5"}, + {file = "pillow-12.3.0-cp311-cp311-win32.whl", hash = "sha256:10e41f0fbf1eec8cfd234b8fe17a4caac7c9d0db4c204d3c173a8f9f6ef3232b"}, + {file = "pillow-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:8e95e1385e4998ae9694eeaa4730ba5457ff61185b3a55e2e7bea0880aef452a"}, + {file = "pillow-12.3.0-cp311-cp311-win_arm64.whl", hash = "sha256:ebaea975e03d3141d9d3a507df75c9b3ec90fa9d2ffd07567b3a978d9d790b26"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:ba09209fbe443b4acccebe845d8a138b89a8f4fbaeedd44953490b5315d5e965"}, + {file = "pillow-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:ffd0c5368496f41b0944be820fcb7a838aa6e623d250b01acf2643939c3f99d7"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d9c7f76c0673154f044e9d78c8655fb4213f6ca31a836df48b40fe5d187717b9"}, + {file = "pillow-12.3.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:78cb2c6865a35ab8ff8b75fd122f6033b92a62c82801110e48ddd6c936a45d91"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:e491916b378fba47242221bb9ead245211b70d504f495d105d17b14a24b4907c"}, + {file = "pillow-12.3.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:0dd2064cbc55aaec028ef5fbb60fa47bb6c3e7918e07ff17935284b227a9d2df"}, + {file = "pillow-12.3.0-cp312-cp312-win32.whl", hash = "sha256:dbce0b29841537a2fa4a214c2bbf14de3587c9680caa9b4e217568472490b28f"}, + {file = "pillow-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:a2b55dd6b2a4c4b7d87ffa56bdb33fdc5fdb9a462173861a7bc097f17d91cb09"}, + {file = "pillow-12.3.0-cp312-cp312-win_arm64.whl", hash = "sha256:331b624368d4f1d069149002f25f44bc61c8919ce8ddb3c45bdad8f6e2d89510"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:21900ce7ba264168cd50defae43cd75d25c833ad4ad6e73ffc5596d12e25ac89"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:4e8c2a84d977f50b9daed6eeaf3baef67d00d5d74d932288f02cb94518ee3ace"}, + {file = "pillow-12.3.0-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:ae26d61dfa7a47befdc7572b521024e8745f3d809bd95ca9505a7bba9ef849ec"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7a743ff716f746fc19a9557f60dab1600d4613255f8a7aeb3cdde4db7eb15a66"}, + {file = "pillow-12.3.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d69141514cc30b774ceea5e3ed3a6635c8d8a96edf664689b890f4089111fb35"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:f7401aebd7f581d7f83a439d87d474999317ee099218e5ad25d125290990ba65"}, + {file = "pillow-12.3.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0847a763afefb695bc912d7c131e7e0632d4edc1d8698f58ddabec8e46b8b6d3"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:571b9fcb07b97ef3a492028fb3d2dc0993ca23a06138b0315286566d29ef718a"}, + {file = "pillow-12.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:756c768d0c9c2955feb7a56c37ea24aea2e369f8d36a88da270b6a9f19e62b5e"}, + {file = "pillow-12.3.0-cp313-cp313-win32.whl", hash = "sha256:a876864214e136f0eb367788dbd7df045f4806801518e2cfe9e13229cfe06d8f"}, + {file = "pillow-12.3.0-cp313-cp313-win_amd64.whl", hash = "sha256:1cca606cd25738df4ed873d5ad46bbdb3d83b5cbca291f6b4ff13a4df6b0bbe8"}, + {file = "pillow-12.3.0-cp313-cp313-win_arm64.whl", hash = "sha256:b629de27fda84b42cde7edef0d85f13b958b47f6e9bbcbba9b673c562a89bd8b"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:9cf95fe4d0f84c82d282745d9bb08ad9f926efa00be4697e767b814ce40d4330"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:8728f216dcdb6e6d555cf971cb34076139ad74b31fc2c14da4fafc741c5f6217"}, + {file = "pillow-12.3.0-cp314-cp314-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:a45650e8ce7fafffd731db8550230db6b0d306d181a90b67d3e6bca2f1990930"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:ba54cfebe86920a559a7c4d6b9050791c20513650a1952ebe3368c7dc70306f8"}, + {file = "pillow-12.3.0-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:e158cb00350dc278f3b91551101aa7d12415a66ebf2c91d8d5ac14e56ddd3ad0"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:e9aeb04d6aef139de265b29683e119b638208f88cf73cdd1658aa07221165321"}, + {file = "pillow-12.3.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:251bf95b67017e27b13d82f5b326234ca62d70f9cf4c2b9032de2358a3b12c7b"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:fe3cca2e4e8a592be0f269a1ca4835c25199d9f3ce815c8491048f785b0a0198"}, + {file = "pillow-12.3.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:23aceaa007d6172b02c277f0cd359c79492bbb14f7072b4ede9fbcaf20648130"}, + {file = "pillow-12.3.0-cp314-cp314-win32.whl", hash = "sha256:af8d94b0db561cf68b88a267c5c44b49e134f525d0dc2cb7ed413a66bc23559a"}, + {file = "pillow-12.3.0-cp314-cp314-win_amd64.whl", hash = "sha256:fdafc9cce40277e0f7a0feabce0ee50dd2fa1800f3b38015e51296b5e814048d"}, + {file = "pillow-12.3.0-cp314-cp314-win_arm64.whl", hash = "sha256:e91206ee562682b51b98ef4b26a6ef48fd84e15fd4c4bc5ec768eb641d206838"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:164b31cd1a0490ab6efae01aa5df49da7061be0af1b30e035b6e9a1bfe34ee6e"}, + {file = "pillow-12.3.0-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:5afb51d599ea772b8365ae807ae557f18bccfe46ab261fd1c2a9ed700fc6eb17"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3edce1d53195db527e0191f84b71d02022de0540bf43a16ed734ed7537b07385"}, + {file = "pillow-12.3.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:bf16ba1b4d0b6b7c8e534936632270cf70eb00dbe09005bc345b2677b726855c"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:24870b09b224f7ae3c39ed07d10e819d06f8720bc551847b1d623832b5b0e28d"}, + {file = "pillow-12.3.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:30f2aa603c41533cc25c05acd0da21636e84a315768feb631c937177db558931"}, + {file = "pillow-12.3.0-cp314-cp314t-win32.whl", hash = "sha256:4b0a7fe987b14c31ebda6083f74f22b561fd3739bc0ac51e019622e3d72668c7"}, + {file = "pillow-12.3.0-cp314-cp314t-win_amd64.whl", hash = "sha256:962864dc93511324d51ddbb5b9f8731bf71675b93ca612a07441896f4688fb8c"}, + {file = "pillow-12.3.0-cp314-cp314t-win_arm64.whl", hash = "sha256:0740a512dc522224c77d9aa5a8d70d8b7d73fb91f2c21125d8d025d3b8990e45"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:0feb2e9d6ad6c9e3c06effe9d00f3f1e618a6643273576b016f591e9315a7139"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:9e881fca225083806662a5c43d627d215f258ff43c890f831966c7d7ba9c7402"}, + {file = "pillow-12.3.0-cp315-cp315-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:4998562bf62a445225f22e07c896bb04b35b1b1f2eb6d760584c9c51d7a5f78c"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:dc624f6bc473dacdf7ef7eb8678d0d08edf15cd94fad6ae5c7d6cc67a4e4902f"}, + {file = "pillow-12.3.0-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:71d6097b330eea8fd15097780c8e89cb1a8ce7838669f48c5bacd6f663dd4701"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:28ce87c5ab450a9dd970b52e5aca5fe63ed432d18a2eaddd1979a00a1ba24ace"}, + {file = "pillow-12.3.0-cp315-cp315-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:6b02afb9b97f65fbca5f31db6a2a3ba21aa93030225f150fa3f249717e938fb4"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:1182d52bc2d5e5d7d0949503aa7e36d12f42205dc287e4883f407b1988820d39"}, + {file = "pillow-12.3.0-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:e795b7eb908249c4e43c7c99fac7c2c75dab0c43566e37db472a355f63693d71"}, + {file = "pillow-12.3.0-cp315-cp315-win32.whl", hash = "sha256:57b3d78c95ba9059768b10e28b813002261d3f3dfc55cc48b0c988f625175827"}, + {file = "pillow-12.3.0-cp315-cp315-win_amd64.whl", hash = "sha256:fa4ecea169a355be7a3ade2c783e2ed12f0e40d2c5621cda8b3297faf7fbb9f5"}, + {file = "pillow-12.3.0-cp315-cp315-win_arm64.whl", hash = "sha256:877c3f311ff35410f690861c4409e7ccbf0cd2f878e50628a28e5a0bb689e658"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:e9871b1ffbfa9656b60aeee92ed5136a5742696006fa322b29ea3d8da0ecc9cf"}, + {file = "pillow-12.3.0-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:53aa02d20d10c3d814d536aa4e5ac9b84ca0ff5a88377963b085ad6822f93e64"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:446c34dcc4324b084a53b705127dc15717b22c5e140ae0a3c38349d4efec071e"}, + {file = "pillow-12.3.0-cp315-cp315t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:cf1845d02ad822a369a49f2bb9345b1614744267682e7a03527dc3bf6eea1777"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:186941b6aef820ad110fb01fb06eb925374dc3a21b17e37ec9a53b250c6fe2d1"}, + {file = "pillow-12.3.0-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:f13c32a3abd6079a66d9526e18dad9b6d280384d49d7c54040cd57b6424041d9"}, + {file = "pillow-12.3.0-cp315-cp315t-win32.whl", hash = "sha256:1657923d2d45afb66526e5b933e5b3052e6bdea196c90d3abb2424e18c77dae8"}, + {file = "pillow-12.3.0-cp315-cp315t-win_amd64.whl", hash = "sha256:8cd2f7bdda092d99c9fc2fb7391354f306d01443d22785d0cbfafa2e2c8bb418"}, + {file = "pillow-12.3.0-cp315-cp315t-win_arm64.whl", hash = "sha256:06ff022112bc9cbf83b60f8e028d94ad87b60621706487e65f673de61610ab59"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_10_15_x86_64.whl", hash = "sha256:b3c777e849237620b022f7f297dd67705f9f5cf1685f09f02e46f93e92725468"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-macosx_11_0_arm64.whl", hash = "sha256:b343699e8308bdc51978310e1c959c584e7869cc8c40780058c87da7781a1e94"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fbd139c8447d25dd750ab79ee274cc5e1fe80fc56340ab10b18a195e1b6eca3e"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e7e480451b9fa137494bccd3a7d69adbe8ac65a87d97be61e11f1b1050a5bac3"}, + {file = "pillow-12.3.0-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:04f01d28a6aaff387bf842a13be313df23ba0597a44f1a976c9feb3c6ff4711a"}, + {file = "pillow-12.3.0.tar.gz", hash = "sha256:3b8182a766685eaa002637e28b4ec8d6b18819a0c71f579bf0dbaa5830297cce"}, +] + +[package.extras] +docs = ["furo", "olefile", "sphinx (>=8.2)", "sphinx-autobuild", "sphinx-copybutton", "sphinx-inline-tabs", "sphinxext-opengraph"] +fpx = ["olefile"] +mic = ["olefile"] +test-arrow = ["arro3-compute", "arro3-core", "nanoarrow", "pyarrow"] +tests = ["coverage (>=7.4.2)", "defusedxml", "markdown2", "olefile", "packaging", "pytest", "pytest-cov", "pytest-timeout", "pytest-xdist", "setuptools", "trove-classifiers (>=2024.10.12)"] +xmp = ["defusedxml"] + [[package]] name = "platformdirs" version = "4.3.8" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.9" +groups = ["dev", "doc"] files = [ {file = "platformdirs-4.3.8-py3-none-any.whl", hash = "sha256:ff7059bb7eb1179e2685604f4aaf157cfd9535242bd23742eadc3c13542139b4"}, {file = "platformdirs-4.3.8.tar.gz", hash = "sha256:3d512d96e16bcb959a814c9f348431070822a6496326a4be0911c40b5a74c2bc"}, @@ -1843,6 +2566,7 @@ version = "1.6.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746"}, {file = "pluggy-1.6.0.tar.gz", hash = "sha256:7dcc130b76258d33b90f61b658791dede3486c3e6bfb003ee5c9bfb396dd22f3"}, @@ -1858,6 +2582,7 @@ version = "3.8.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pre_commit-3.8.0-py2.py3-none-any.whl", hash = "sha256:9a90a53bf82fdd8778d58085faf8d83df56e40dfe18f45b19446e26bf1b3a63f"}, {file = "pre_commit-3.8.0.tar.gz", hash = "sha256:8bb6494d4a20423842e198980c9ecf9f96607a07ea29549e180eef9ae80fe7af"}, @@ -1876,6 +2601,7 @@ version = "3.0.51" description = "Library for building powerful interactive command lines in Python" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "prompt_toolkit-3.0.51-py3-none-any.whl", hash = "sha256:52742911fde84e2d423e2f9a4cf1de7d7ac4e51958f648d9540e0fb8db077b07"}, {file = "prompt_toolkit-3.0.51.tar.gz", hash = "sha256:931a162e3b27fc90c86f1b48bb1fb2c528c2761475e57c9c06de13311c7b54ed"}, @@ -1890,6 +2616,7 @@ version = "5.29.5" description = "" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "protobuf-5.29.5-cp310-abi3-win32.whl", hash = "sha256:3f1c6468a2cfd102ff4703976138844f78ebd1fb45f49011afc5139e9e283079"}, {file = "protobuf-5.29.5-cp310-abi3-win_amd64.whl", hash = "sha256:3f76e3a3675b4a4d867b52e4a5f5b78a2ef9565549d4037e06cf7b0942b1d3fc"}, @@ -1910,6 +2637,7 @@ version = "5.9.8" description = "Cross-platform lib for process and system monitoring in Python." optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" +groups = ["dev"] files = [ {file = "psutil-5.9.8-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8"}, {file = "psutil-5.9.8-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73"}, @@ -1930,7 +2658,7 @@ files = [ ] [package.extras] -test = ["enum34", "ipaddress", "mock", "pywin32", "wmi"] +test = ["enum34 ; python_version <= \"3.4\"", "ipaddress ; python_version < \"3.0\"", "mock ; python_version < \"3.0\"", "pywin32 ; sys_platform == \"win32\"", "wmi ; sys_platform == \"win32\""] [[package]] name = "ptyprocess" @@ -1938,6 +2666,8 @@ version = "0.7.0" description = "Run a subprocess in a pseudo terminal" optional = false python-versions = "*" +groups = ["dev"] +markers = "sys_platform != \"win32\"" files = [ {file = "ptyprocess-0.7.0-py2.py3-none-any.whl", hash = "sha256:4b41f3967fce3af57cc7e94b888626c18bf37a083e3651ca8feeb66d492fef35"}, {file = "ptyprocess-0.7.0.tar.gz", hash = "sha256:5c5d0a3b48ceee0b48485e0c26037c0acd7d29765ca3fbb5cb3831d347423220"}, @@ -1949,6 +2679,7 @@ version = "0.2.3" description = "Safely evaluate AST nodes without side effects" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "pure_eval-0.2.3-py3-none-any.whl", hash = "sha256:1db8e35b67b3d218d818ae653e27f06c3aa420901fa7b081ca98cbedc874e0d0"}, {file = "pure_eval-0.2.3.tar.gz", hash = "sha256:5f4e983f40564c576c7c8635ae88db5956bb2229d7e9237d03b3c0b0190eaf42"}, @@ -1963,6 +2694,7 @@ version = "9.0.0" description = "Get CPU info with pure Python" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "py-cpuinfo-9.0.0.tar.gz", hash = "sha256:3cdbbf3fac90dc6f118bfd64384f309edeadd902d7c8fb17f02ffa1fc3f49690"}, {file = "py_cpuinfo-9.0.0-py3-none-any.whl", hash = "sha256:859625bc251f64e21f077d099d4162689c762b5d6a4c3c97553d56241c9674d5"}, @@ -1974,6 +2706,8 @@ version = "2.22" description = "C parser in Python" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "implementation_name == \"pypy\"" files = [ {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, @@ -1985,6 +2719,7 @@ version = "2.0.0" description = "Python interface to Graphviz's Dot" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pydot-2.0.0-py3-none-any.whl", hash = "sha256:408a47913ea7bd5d2d34b274144880c1310c4aee901f353cf21fe2e526a4ea28"}, {file = "pydot-2.0.0.tar.gz", hash = "sha256:60246af215123fa062f21cd791be67dda23a6f280df09f68919e637a1e4f3235"}, @@ -2004,6 +2739,7 @@ version = "2.19.2" description = "Pygments is a syntax highlighting package written in Python." optional = false python-versions = ">=3.8" +groups = ["main", "dev", "doc"] files = [ {file = "pygments-2.19.2-py3-none-any.whl", hash = "sha256:86540386c03d588bb81d44bc3928634ff26449851e99741617ecb9037ee5ec0b"}, {file = "pygments-2.19.2.tar.gz", hash = "sha256:636cb2477cec7f8952536970bc533bc43743542f70392ae026374600add5b887"}, @@ -2018,6 +2754,7 @@ version = "10.16.1" description = "Extension pack for Python Markdown." optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "pymdown_extensions-10.16.1-py3-none-any.whl", hash = "sha256:d6ba157a6c03146a7fb122b2b9a121300056384eafeec9c9f9e584adfdb2a32d"}, {file = "pymdown_extensions-10.16.1.tar.gz", hash = "sha256:aace82bcccba3efc03e25d584e6a22d27a8e17caa3f4dd9f207e49b787aa9a91"}, @@ -2036,6 +2773,7 @@ version = "3.2.3" description = "pyparsing module - Classes and methods to define and execute parsing grammars" optional = false python-versions = ">=3.9" +groups = ["dev", "doc"] files = [ {file = "pyparsing-3.2.3-py3-none-any.whl", hash = "sha256:a749938e02d6fd0b59b356ca504a24982314bb090c383e3cf201c95ef7e2bfcf"}, {file = "pyparsing-3.2.3.tar.gz", hash = "sha256:b9c13f1ab8b3b542f72e28f634bad4de758ab3ce4546e4301970ad6fa77c38be"}, @@ -2050,6 +2788,7 @@ version = "8.4.1" description = "pytest: simple powerful testing with Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pytest-8.4.1-py3-none-any.whl", hash = "sha256:539c70ba6fcead8e78eebbf1115e8b589e7565830d7d006a8723f19ac8a0afb7"}, {file = "pytest-8.4.1.tar.gz", hash = "sha256:7c67fd69174877359ed9371ec3af8a3d2b04741818c51e5e99cc1742251fa93c"}, @@ -2073,6 +2812,7 @@ version = "4.0.0" description = "A ``pytest`` fixture for benchmarking code. It will group the tests into rounds that are calibrated to the chosen timer." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pytest-benchmark-4.0.0.tar.gz", hash = "sha256:fb0785b83efe599a6a956361c0691ae1dbb5318018561af10f3e915caa0048d1"}, {file = "pytest_benchmark-4.0.0-py3-none-any.whl", hash = "sha256:fdb7db64e31c8b277dff9850d2a2556d8b60bcb0ea6524e36e28ffd7c87f71d6"}, @@ -2093,6 +2833,7 @@ version = "4.1.0" description = "Pytest plugin for measuring coverage." optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pytest-cov-4.1.0.tar.gz", hash = "sha256:3904b13dfbfec47f003b8e77fd5b589cd11904a21ddf1ab38a64f204d6a10ef6"}, {file = "pytest_cov-4.1.0-py3-none-any.whl", hash = "sha256:6ba70b9e97e69fcc3fb45bfeab2d0a138fb65c4d0d6a41ef33983ad114be8c3a"}, @@ -2111,6 +2852,7 @@ version = "4.1.1" description = "pytest plugin for generating HTML reports" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest_html-4.1.1-py3-none-any.whl", hash = "sha256:c8152cea03bd4e9bee6d525573b67bbc6622967b72b9628dda0ea3e2a0b5dd71"}, {file = "pytest_html-4.1.1.tar.gz", hash = "sha256:70a01e8ae5800f4a074b56a4cb1025c8f4f9b038bba5fe31e3c98eb996686f07"}, @@ -2131,6 +2873,7 @@ version = "3.1.1" description = "pytest plugin for test session metadata" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest_metadata-3.1.1-py3-none-any.whl", hash = "sha256:c8e0844db684ee1c798cfa38908d20d67d0463ecb6137c72e91f418558dd5f4b"}, {file = "pytest_metadata-3.1.1.tar.gz", hash = "sha256:d2a29b0355fbc03f168aa96d41ff88b1a3b44a3b02acbe491801c98a048017c8"}, @@ -2148,6 +2891,7 @@ version = "3.14.1" description = "Thin-wrapper around the mock package for easier use with pytest" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pytest_mock-3.14.1-py3-none-any.whl", hash = "sha256:178aefcd11307d874b4cd3100344e7e2d888d9791a6a1d9bfe90fbc1b74fd1d0"}, {file = "pytest_mock-3.14.1.tar.gz", hash = "sha256:159e9edac4c451ce77a5cdb9fc5d1100708d2dd4ba3c3df572f14097351af80e"}, @@ -2165,6 +2909,7 @@ version = "0.1.1" description = "a pytest plugin for parallel and concurrent testing" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "pytest-parallel-0.1.1.tar.gz", hash = "sha256:9aac3fc199a168c0a8559b60249d9eb254de7af58c12cee0310b54d4affdbfab"}, {file = "pytest_parallel-0.1.1-py3-none-any.whl", hash = "sha256:9e3703015b0eda52be9e07d2ba3498f09340a56d5c79a39b50f22fc5c38212fe"}, @@ -2180,6 +2925,7 @@ version = "0.9.7" description = "pytest-sugar is a plugin for pytest that changes the default look and feel of pytest (e.g. progressbar, show tests that fail instantly)." optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "pytest-sugar-0.9.7.tar.gz", hash = "sha256:f1e74c1abfa55f7241cf7088032b6e378566f16b938f3f08905e2cf4494edd46"}, {file = "pytest_sugar-0.9.7-py2.py3-none-any.whl", hash = "sha256:8cb5a4e5f8bbcd834622b0235db9e50432f4cbd71fef55b467fe44e43701e062"}, @@ -2199,6 +2945,7 @@ version = "2.4.0" description = "pytest plugin to abort hanging tests" optional = false python-versions = ">=3.7" +groups = ["dev"] files = [ {file = "pytest_timeout-2.4.0-py3-none-any.whl", hash = "sha256:c42667e5cdadb151aeb5b26d114aff6bdf5a907f176a007a30b940d3d865b5c2"}, {file = "pytest_timeout-2.4.0.tar.gz", hash = "sha256:7e68e90b01f9eff71332b25001f85c75495fc4e3a836701876183c4bcfd0540a"}, @@ -2213,6 +2960,7 @@ version = "3.8.0" description = "pytest xdist plugin for distributed testing, most importantly across multiple CPUs" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "pytest_xdist-3.8.0-py3-none-any.whl", hash = "sha256:202ca578cfeb7370784a8c33d6d05bc6e13b4f25b5053c30a152269fd10f0b88"}, {file = "pytest_xdist-3.8.0.tar.gz", hash = "sha256:7e578125ec9bc6050861aa93f2d59f1d8d085595d6551c2c90b6f4fad8d3a9f1"}, @@ -2233,6 +2981,7 @@ version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "dev", "doc"] files = [ {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, @@ -2247,6 +2996,7 @@ version = "2025.2" description = "World timezone definitions, modern and historical" optional = false python-versions = "*" +groups = ["main"] files = [ {file = "pytz-2025.2-py2.py3-none-any.whl", hash = "sha256:5ddf76296dd8c44c26eb8f4b6f35488f3ccbf6fbbd7adee0b7262d43f0ec2f00"}, {file = "pytz-2025.2.tar.gz", hash = "sha256:360b9e3dbb49a209c21ad61809c7fb453643e048b38924c765813546746e81c3"}, @@ -2258,6 +3008,8 @@ version = "311" description = "Python for Window Extensions" optional = false python-versions = "*" +groups = ["dev"] +markers = "sys_platform == \"win32\" and platform_python_implementation != \"PyPy\"" files = [ {file = "pywin32-311-cp310-cp310-win32.whl", hash = "sha256:d03ff496d2a0cd4a5893504789d4a15399133fe82517455e78bad62efbb7f0a3"}, {file = "pywin32-311-cp310-cp310-win_amd64.whl", hash = "sha256:797c2772017851984b97180b0bebe4b620bb86328e8a884bb626156295a63b3b"}, @@ -2287,6 +3039,7 @@ version = "6.0.2" description = "YAML parser and emitter for Python" optional = false python-versions = ">=3.8" +groups = ["dev", "doc"] files = [ {file = "PyYAML-6.0.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:0a9a2848a5b7feac301353437eb7d5957887edbf81d56e903999a75a3d743086"}, {file = "PyYAML-6.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:29717114e51c84ddfba879543fb232a6ed60086602313ca38cce623c1d62cfbf"}, @@ -2349,6 +3102,7 @@ version = "1.1" description = "A custom YAML tag for referencing environment variables in YAML files." optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "pyyaml_env_tag-1.1-py3-none-any.whl", hash = "sha256:17109e1a528561e32f026364712fee1264bc2ea6715120891174ed1b980d2e04"}, {file = "pyyaml_env_tag-1.1.tar.gz", hash = "sha256:2eb38b75a2d21ee0475d6d97ec19c63287a7e140231e4214969d0eac923cd7ff"}, @@ -2363,6 +3117,7 @@ version = "27.0.0" description = "Python bindings for 0MQ" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "pyzmq-27.0.0-cp310-cp310-macosx_10_15_universal2.whl", hash = "sha256:b973ee650e8f442ce482c1d99ca7ab537c69098d53a3d046676a484fd710c87a"}, {file = "pyzmq-27.0.0-cp310-cp310-manylinux2014_i686.manylinux_2_17_i686.whl", hash = "sha256:661942bc7cd0223d569d808f2e5696d9cc120acc73bf3e88a1f1be7ab648a7e4"}, @@ -2454,6 +3209,7 @@ version = "2.32.4" description = "Python HTTP for Humans." optional = false python-versions = ">=3.8" +groups = ["main", "doc"] files = [ {file = "requests-2.32.4-py3-none-any.whl", hash = "sha256:27babd3cda2a6d50b30443204ee89830707d396671944c998b5975b031ac2b2c"}, {file = "requests-2.32.4.tar.gz", hash = "sha256:27d0316682c8a29834d3264820024b62a36942083d52caf2f14c0591336d3422"}, @@ -2475,6 +3231,7 @@ version = "14.1.0" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" optional = false python-versions = ">=3.8.0" +groups = ["main"] files = [ {file = "rich-14.1.0-py3-none-any.whl", hash = "sha256:536f5f1785986d6dbdea3c75205c473f970777b4a0d6c6dd1b696aa05a3fa04f"}, {file = "rich-14.1.0.tar.gz", hash = "sha256:e497a48b844b0320d45007cdebfeaeed8db2a4f4bcf49f15e455cfc4af11eaa8"}, @@ -2487,12 +3244,128 @@ pygments = ">=2.13.0,<3.0.0" [package.extras] jupyter = ["ipywidgets (>=7.5.1,<9)"] +[[package]] +name = "scikit-learn" +version = "1.7.2" +description = "A set of python modules for machine learning and data mining" +optional = false +python-versions = ">=3.10" +groups = ["dev"] +markers = "python_version == \"3.10\"" +files = [ + {file = "scikit_learn-1.7.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b33579c10a3081d076ab403df4a4190da4f4432d443521674637677dc91e61f"}, + {file = "scikit_learn-1.7.2-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:36749fb62b3d961b1ce4fedf08fa57a1986cd409eff2d783bca5d4b9b5fce51c"}, + {file = "scikit_learn-1.7.2-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7a58814265dfc52b3295b1900cfb5701589d30a8bb026c7540f1e9d3499d5ec8"}, + {file = "scikit_learn-1.7.2-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4a847fea807e278f821a0406ca01e387f97653e284ecbd9750e3ee7c90347f18"}, + {file = "scikit_learn-1.7.2-cp310-cp310-win_amd64.whl", hash = "sha256:ca250e6836d10e6f402436d6463d6c0e4d8e0234cfb6a9a47835bd392b852ce5"}, + {file = "scikit_learn-1.7.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:c7509693451651cd7361d30ce4e86a1347493554f172b1c72a39300fa2aea79e"}, + {file = "scikit_learn-1.7.2-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:0486c8f827c2e7b64837c731c8feff72c0bd2b998067a8a9cbc10643c31f0fe1"}, + {file = "scikit_learn-1.7.2-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:89877e19a80c7b11a2891a27c21c4894fb18e2c2e077815bcade10d34287b20d"}, + {file = "scikit_learn-1.7.2-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8da8bf89d4d79aaec192d2bda62f9b56ae4e5b4ef93b6a56b5de4977e375c1f1"}, + {file = "scikit_learn-1.7.2-cp311-cp311-win_amd64.whl", hash = "sha256:9b7ed8d58725030568523e937c43e56bc01cadb478fc43c042a9aca1dacb3ba1"}, + {file = "scikit_learn-1.7.2-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:8d91a97fa2b706943822398ab943cde71858a50245e31bc71dba62aab1d60a96"}, + {file = "scikit_learn-1.7.2-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:acbc0f5fd2edd3432a22c69bed78e837c70cf896cd7993d71d51ba6708507476"}, + {file = "scikit_learn-1.7.2-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e5bf3d930aee75a65478df91ac1225ff89cd28e9ac7bd1196853a9229b6adb0b"}, + {file = "scikit_learn-1.7.2-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:b4d6e9deed1a47aca9fe2f267ab8e8fe82ee20b4526b2c0cd9e135cea10feb44"}, + {file = "scikit_learn-1.7.2-cp312-cp312-win_amd64.whl", hash = "sha256:6088aa475f0785e01bcf8529f55280a3d7d298679f50c0bb70a2364a82d0b290"}, + {file = "scikit_learn-1.7.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0b7dacaa05e5d76759fb071558a8b5130f4845166d88654a0f9bdf3eb57851b7"}, + {file = "scikit_learn-1.7.2-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:abebbd61ad9e1deed54cca45caea8ad5f79e1b93173dece40bb8e0c658dbe6fe"}, + {file = "scikit_learn-1.7.2-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:502c18e39849c0ea1a5d681af1dbcf15f6cce601aebb657aabbfe84133c1907f"}, + {file = "scikit_learn-1.7.2-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:7a4c328a71785382fe3fe676a9ecf2c86189249beff90bf85e22bdb7efaf9ae0"}, + {file = "scikit_learn-1.7.2-cp313-cp313-win_amd64.whl", hash = "sha256:63a9afd6f7b229aad94618c01c252ce9e6fa97918c5ca19c9a17a087d819440c"}, + {file = "scikit_learn-1.7.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:9acb6c5e867447b4e1390930e3944a005e2cb115922e693c08a323421a6966e8"}, + {file = "scikit_learn-1.7.2-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:2a41e2a0ef45063e654152ec9d8bcfc39f7afce35b08902bfe290c2498a67a6a"}, + {file = "scikit_learn-1.7.2-cp313-cp313t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:98335fb98509b73385b3ab2bd0639b1f610541d3988ee675c670371d6a87aa7c"}, + {file = "scikit_learn-1.7.2-cp313-cp313t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:191e5550980d45449126e23ed1d5e9e24b2c68329ee1f691a3987476e115e09c"}, + {file = "scikit_learn-1.7.2-cp313-cp313t-win_amd64.whl", hash = "sha256:57dc4deb1d3762c75d685507fbd0bc17160144b2f2ba4ccea5dc285ab0d0e973"}, + {file = "scikit_learn-1.7.2-cp314-cp314-macosx_10_13_x86_64.whl", hash = "sha256:fa8f63940e29c82d1e67a45d5297bdebbcb585f5a5a50c4914cc2e852ab77f33"}, + {file = "scikit_learn-1.7.2-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:f95dc55b7902b91331fa4e5845dd5bde0580c9cd9612b1b2791b7e80c3d32615"}, + {file = "scikit_learn-1.7.2-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9656e4a53e54578ad10a434dc1f993330568cfee176dff07112b8785fb413106"}, + {file = "scikit_learn-1.7.2-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:96dc05a854add0e50d3f47a1ef21a10a595016da5b007c7d9cd9d0bffd1fcc61"}, + {file = "scikit_learn-1.7.2-cp314-cp314-win_amd64.whl", hash = "sha256:bb24510ed3f9f61476181e4db51ce801e2ba37541def12dc9333b946fc7a9cf8"}, + {file = "scikit_learn-1.7.2.tar.gz", hash = "sha256:20e9e49ecd130598f1ca38a1d85090e1a600147b9c02fa6f15d69cb53d968fda"}, +] + +[package.dependencies] +joblib = ">=1.2.0" +numpy = ">=1.22.0" +scipy = ">=1.8.0" +threadpoolctl = ">=3.1.0" + +[package.extras] +benchmark = ["matplotlib (>=3.5.0)", "memory_profiler (>=0.57.0)", "pandas (>=1.4.0)"] +build = ["cython (>=3.0.10)", "meson-python (>=0.17.1)", "numpy (>=1.22.0)", "scipy (>=1.8.0)"] +docs = ["Pillow (>=8.4.0)", "matplotlib (>=3.5.0)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.4.0)", "plotly (>=5.14.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pydata-sphinx-theme (>=0.15.3)", "scikit-image (>=0.19.0)", "seaborn (>=0.9.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.5.0)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.17.1)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)", "towncrier (>=24.8.0)"] +examples = ["matplotlib (>=3.5.0)", "pandas (>=1.4.0)", "plotly (>=5.14.0)", "pooch (>=1.6.0)", "scikit-image (>=0.19.0)", "seaborn (>=0.9.0)"] +install = ["joblib (>=1.2.0)", "numpy (>=1.22.0)", "scipy (>=1.8.0)", "threadpoolctl (>=3.1.0)"] +maintenance = ["conda-lock (==3.0.1)"] +tests = ["matplotlib (>=3.5.0)", "mypy (>=1.15)", "numpydoc (>=1.2.0)", "pandas (>=1.4.0)", "polars (>=0.20.30)", "pooch (>=1.6.0)", "pyamg (>=4.2.1)", "pyarrow (>=12.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "ruff (>=0.11.7)", "scikit-image (>=0.19.0)"] + +[[package]] +name = "scikit-learn" +version = "1.9.0" +description = "A set of python modules for machine learning and data mining" +optional = false +python-versions = ">=3.11" +groups = ["dev"] +markers = "python_version >= \"3.11\"" +files = [ + {file = "scikit_learn-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9db6f4d34e68c8899e4cab27fdf8eafe6ed21f2ba52ceb25ea250cd237f8e47b"}, + {file = "scikit_learn-1.9.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:f401448645a3e7bc115aa3c094097865155b34bff1cba8101857d9104e99074c"}, + {file = "scikit_learn-1.9.0-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:fd3a8ef0c758555a3b23c03adaa858af32f7736785ded50ad5991f59c4ed03fa"}, + {file = "scikit_learn-1.9.0-cp311-cp311-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f7e254636164090da847715a27f8e5478feb98c40a9e0ee90cbd277de9e5ceb8"}, + {file = "scikit_learn-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:5dc1818c77575d149e25fce9ef82dd7b7263ae372f03494158668ad632a69759"}, + {file = "scikit_learn-1.9.0-cp311-cp311-win_arm64.whl", hash = "sha256:366652351f092b219c248f1e72821e841960a63d8f358f1dcfd54dc1cbdbbc28"}, + {file = "scikit_learn-1.9.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:2bd41b0d201bc81575531b96b713d3eb5e5f50fb0b82101ff0f92294fdc236ac"}, + {file = "scikit_learn-1.9.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:5be45aa4a42a68a533913a6ed736cf309de2226411c79ef8d609a5456f1939b1"}, + {file = "scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5e50ed4da51974e86e940690e9a3d82e729b62b5a49f7c9bac534d515d39d86f"}, + {file = "scikit_learn-1.9.0-cp312-cp312-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:056c92bb67ad4c28463c2f2653d9701449201e7e7a9e94e321be0f71c4fef2b8"}, + {file = "scikit_learn-1.9.0-cp312-cp312-win_amd64.whl", hash = "sha256:4306775fad04cc4b472a1b15af1ae9cede1540fbfcc17fbce3767cd8dc7ae283"}, + {file = "scikit_learn-1.9.0-cp312-cp312-win_arm64.whl", hash = "sha256:26e22435f63bcdcf396b574273f29f13dd531f5ea035801f5be10ba1540a4e60"}, + {file = "scikit_learn-1.9.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:80746d63bd4b6eaca54d36fe5feaf4d28bb38dc6f9470f81c7cad7c40155f119"}, + {file = "scikit_learn-1.9.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:5b934c45c252844a91d69fda3a34cff5e7307e1db10d77cb10a3980312c74713"}, + {file = "scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:38c3dcb9a1ffb85505ec53d54c7b4aea0cff70050425a7760c2af661ac85df05"}, + {file = "scikit_learn-1.9.0-cp313-cp313-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:da76d09304a4706db7cc1e3ebaa3b6b98a67365cc11d2996c4f1e58ba47df714"}, + {file = "scikit_learn-1.9.0-cp313-cp313-win_amd64.whl", hash = "sha256:5808d98f15c6bf6d9d96d2348c1997392a5888ce7097e664105f930c4bca1277"}, + {file = "scikit_learn-1.9.0-cp313-cp313-win_arm64.whl", hash = "sha256:d77f54c017633791bc0225a43e2f8d03745fdcfe4880268fcc4df15f505dec2e"}, + {file = "scikit_learn-1.9.0-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:9656acd4e93f74e0b66c8a36c88830a99252dfa900044d36bc2212ae89a47162"}, + {file = "scikit_learn-1.9.0-cp314-cp314-macosx_12_0_arm64.whl", hash = "sha256:24360002ae845e7866522b0a5bbf690802e7bc388cac8663502e78aa98598aa2"}, + {file = "scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5162ad10a418c8a282dde04c9aa06965de3e9a65f33c1440c0ae69bb1a09d913"}, + {file = "scikit_learn-1.9.0-cp314-cp314-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:1fea2cc5677ab49d6f5bade978c866da44957b712d92e9635e8b4f723013c3cb"}, + {file = "scikit_learn-1.9.0-cp314-cp314-win_amd64.whl", hash = "sha256:64fa347efc1c839c487433e40c5144d38c336e8a2b59c81aa8660373945c2673"}, + {file = "scikit_learn-1.9.0-cp314-cp314-win_arm64.whl", hash = "sha256:1b944b6db288f6b926e3650026ddafb988929de95d11fc2cc5fa117773c9ba42"}, + {file = "scikit_learn-1.9.0-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:4ccacf04ca5f4b492158a5f28afe0ace43f81b2571e4b9a66d34848b46128949"}, + {file = "scikit_learn-1.9.0-cp314-cp314t-macosx_12_0_arm64.whl", hash = "sha256:ee1a8db2c18c08e34c7412d4b10be1cac214cd4ea7dc9715a6a327eb49a37c96"}, + {file = "scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:147e9329ef0e39f75d4cffa02b2aa48d827832684926cd5210d9a2cb5c57246b"}, + {file = "scikit_learn-1.9.0-cp314-cp314t-manylinux_2_27_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5bad8f8b9950321b54c965fdcbac6c6c55e79e16646b49977bcf3668d3870a1a"}, + {file = "scikit_learn-1.9.0-cp314-cp314t-win_amd64.whl", hash = "sha256:78fc56eafd4edb9575d2d8950d1dd152061abb573341a1cb7e099fc40f6c6666"}, + {file = "scikit_learn-1.9.0-cp314-cp314t-win_arm64.whl", hash = "sha256:051075bda8b7aab87b1906ab3d4740a1e1224a19d7b3781a576736edc94e76aa"}, + {file = "scikit_learn-1.9.0.tar.gz", hash = "sha256:8833266989d3a5110178a9fae30783675460724d0e1efb13b14901d2c660c557"}, +] + +[package.dependencies] +joblib = ">=1.4.0" +narwhals = ">=2.0.1" +numpy = ">=1.24.1" +scipy = ">=1.10.0" +threadpoolctl = ">=3.5.0" + +[package.extras] +benchmark = ["matplotlib (>=3.6.1)", "memory_profiler (>=0.57.0)", "pandas (>=1.5.0)"] +build = ["cython (>=3.1.2)", "meson-python (>=0.17.1)", "numpy (>=1.24.1)", "scipy (>=1.10.0)"] +docs = ["Pillow (>=12.1.1)", "matplotlib (>=3.6.1)", "memory_profiler (>=0.57.0)", "numpydoc (>=1.2.0)", "pandas (>=1.5.0)", "plotly (>=5.22.0)", "polars (>=0.20.30)", "pooch (>=1.8.0)", "pydata-sphinx-theme (>=0.15.3)", "rich (>=14.1.0)", "scikit-image (>=0.22.0)", "seaborn (>=0.13.0)", "sphinx (>=7.3.7)", "sphinx-copybutton (>=0.5.2)", "sphinx-design (>=0.6.0)", "sphinx-gallery (>=0.17.1)", "sphinx-prompt (>=1.4.0)", "sphinx-remove-toctrees (>=1.0.0.post1)", "sphinxcontrib-sass (>=0.3.4)", "sphinxext-opengraph (>=0.9.1)", "towncrier (>=24.8.0)"] +examples = ["matplotlib (>=3.6.1)", "pandas (>=1.5.0)", "plotly (>=5.22.0)", "pooch (>=1.8.0)", "rich (>=14.1.0)", "scikit-image (>=0.22.0)", "seaborn (>=0.13.0)"] +install = ["joblib (>=1.4.0)", "narwhals (>=2.0.1)", "numpy (>=1.24.1)", "scipy (>=1.10.0)", "threadpoolctl (>=3.5.0)"] +maintenance = ["conda-lock (==3.0.1)"] +tests = ["matplotlib (>=3.6.1)", "mypy (>=1.15)", "numpydoc (>=1.2.0)", "pandas (>=1.5.0)", "polars (>=0.20.30)", "pooch (>=1.8.0)", "pyamg (>=5.0.0)", "pyarrow (>=13.0.0)", "pytest (>=7.1.2)", "pytest-cov (>=2.9.0)", "rich (>=14.1.0)", "ruff (>=0.12.2)"] + [[package]] name = "scipy" version = "1.13.1" description = "Fundamental algorithms for scientific computing in Python" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "scipy-1.13.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:20335853b85e9a49ff7572ab453794298bcf0354d8068c5f6775a0eabf350aca"}, {file = "scipy-1.13.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:d605e9c23906d1994f55ace80e0125c587f96c020037ea6aa98d01b4bd2e222f"}, @@ -2535,19 +3408,20 @@ version = "80.9.0" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "setuptools-80.9.0-py3-none-any.whl", hash = "sha256:062d34222ad13e0cc312a4c02d73f059e86a4acbfbdea8f8f76b28c99f306922"}, {file = "setuptools-80.9.0.tar.gz", hash = "sha256:f36b47402ecde768dbfafc46e8e4207b4360c654f1f3bb84475f0a28628fb19c"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)", "ruff (>=0.8.0)"] -core = ["importlib_metadata (>=6)", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1)", "wheel (>=0.43.0)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\"", "ruff (>=0.8.0) ; sys_platform != \"cygwin\""] +core = ["importlib_metadata (>=6) ; python_version < \"3.10\"", "jaraco.functools (>=4)", "jaraco.text (>=3.7)", "more_itertools", "more_itertools (>=8.8)", "packaging (>=24.2)", "platformdirs (>=4.2.2)", "tomli (>=2.0.1) ; python_version < \"3.11\"", "wheel (>=0.43.0)"] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "pyproject-hooks (!=1.1)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier", "towncrier (<24.7)"] enabler = ["pytest-enabler (>=2.2)"] -test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] -type = ["importlib_metadata (>=7.0.2)", "jaraco.develop (>=7.21)", "mypy (==1.14.*)", "pytest-mypy"] +test = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "ini2toml[lite] (>=0.14)", "jaraco.develop (>=7.21) ; python_version >= \"3.9\" and sys_platform != \"cygwin\"", "jaraco.envs (>=2.2)", "jaraco.path (>=3.7.2)", "jaraco.test (>=5.5)", "packaging (>=24.2)", "pip (>=19.1)", "pyproject-hooks (!=1.1)", "pytest (>=6,!=8.1.*)", "pytest-home (>=0.5)", "pytest-perf ; sys_platform != \"cygwin\"", "pytest-subprocess", "pytest-timeout", "pytest-xdist (>=3)", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel (>=0.44.0)"] +type = ["importlib_metadata (>=7.0.2) ; python_version < \"3.10\"", "jaraco.develop (>=7.21) ; sys_platform != \"cygwin\"", "mypy (==1.14.*)", "pytest-mypy"] [[package]] name = "six" @@ -2555,6 +3429,7 @@ version = "1.17.0" description = "Python 2 and 3 compatibility utilities" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +groups = ["main", "dev", "doc"] files = [ {file = "six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274"}, {file = "six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81"}, @@ -2566,6 +3441,7 @@ version = "0.6.3" description = "Extract data from python stack frames and tracebacks for informative displays" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "stack_data-0.6.3-py3-none-any.whl", hash = "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"}, {file = "stack_data-0.6.3.tar.gz", hash = "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9"}, @@ -2585,39 +3461,19 @@ version = "3.1.0" description = "Traceback serialization library." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "tblib-3.1.0-py3-none-any.whl", hash = "sha256:670bb4582578134b3d81a84afa1b016128b429f3d48e6cbbaecc9d15675e984e"}, {file = "tblib-3.1.0.tar.gz", hash = "sha256:06404c2c9f07f66fee2d7d6ad43accc46f9c3361714d9b8426e7f47e595cd652"}, ] -[[package]] -name = "tensorboard" -version = "2.18.0" -description = "TensorBoard lets you watch Tensors Flow" -optional = false -python-versions = ">=3.9" -files = [ - {file = "tensorboard-2.18.0-py3-none-any.whl", hash = "sha256:107ca4821745f73e2aefa02c50ff70a9b694f39f790b11e6f682f7d326745eab"}, -] - -[package.dependencies] -absl-py = ">=0.4" -grpcio = ">=1.48.2" -markdown = ">=2.6.8" -numpy = ">=1.12.0" -packaging = "*" -protobuf = ">=3.19.6,<4.24.0 || >4.24.0" -setuptools = ">=41.0.0" -six = ">1.9" -tensorboard-data-server = ">=0.7.0,<0.8.0" -werkzeug = ">=1.0.1" - [[package]] name = "tensorboard" version = "2.19.0" description = "TensorBoard lets you watch Tensors Flow" optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "tensorboard-2.19.0-py3-none-any.whl", hash = "sha256:5e71b98663a641a7ce8a6e70b0be8e1a4c0c45d48760b076383ac4755c35b9a0"}, ] @@ -2640,70 +3496,20 @@ version = "0.7.2" description = "Fast data loading for TensorBoard" optional = false python-versions = ">=3.7" +groups = ["main"] files = [ {file = "tensorboard_data_server-0.7.2-py3-none-any.whl", hash = "sha256:7e0610d205889588983836ec05dc098e80f97b7e7bbff7e994ebb78f578d0ddb"}, {file = "tensorboard_data_server-0.7.2-py3-none-macosx_10_9_x86_64.whl", hash = "sha256:9fe5d24221b29625dbc7328b0436ca7fc1c23de4acf4d272f1180856e32f9f60"}, {file = "tensorboard_data_server-0.7.2-py3-none-manylinux_2_31_x86_64.whl", hash = "sha256:ef687163c24185ae9754ed5650eb5bc4d84ff257aabdc33f0cc6f74d8ba54530"}, ] -[[package]] -name = "tensorflow" -version = "2.18.1" -description = "TensorFlow is an open source machine learning framework for everyone." -optional = false -python-versions = ">=3.9" -files = [ - {file = "tensorflow-2.18.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:8baba2b0f9f286f8115a0005d17c020d2febf95e434302eaf758f2020c1c4de5"}, - {file = "tensorflow-2.18.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd7284768f5a6b10e41a700e8141de70756dc62ed5d0b93360d131ccc0a6ba8"}, - {file = "tensorflow-2.18.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1f929842999d60e7da67743ae5204b477259f3b771c02e5e437d232267e49f18"}, - {file = "tensorflow-2.18.1-cp310-cp310-win_amd64.whl", hash = "sha256:db1d186c17b6a7c51813e275d0a83e964669822372aa01d074cf64b853ee76ac"}, - {file = "tensorflow-2.18.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:661029cd769b311db910b79a3a6ef50a5a61ecc947172228c777a49989722508"}, - {file = "tensorflow-2.18.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8a6485edd2148f70d011dbd1d8dc2c775e91774a5a159466e83d0d1f21580944"}, - {file = "tensorflow-2.18.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c9f87e5d2a680a4595f5dc30daf6bbaec9d4129b46d7ef1b2af63c46ac7d2828"}, - {file = "tensorflow-2.18.1-cp311-cp311-win_amd64.whl", hash = "sha256:99223d0dde08aec4ceebb3bf0f80da7802e18462dab0d5048225925c064d2af7"}, - {file = "tensorflow-2.18.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:98afa9c7f21481cdc6ccd09507a7878d533150fbb001840cc145e2132eb40942"}, - {file = "tensorflow-2.18.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1ba52b9c06ab8102b31e50acfaf56899b923171e603c8942f2bfeb181d6bb59e"}, - {file = "tensorflow-2.18.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:442d2a774811789a8ad948e7286cb950fe3d87d3754e8cc6449d53b03dbfdaa6"}, - {file = "tensorflow-2.18.1-cp312-cp312-win_amd64.whl", hash = "sha256:210baf6d421f3e044b6e09efd04494a33b75334922fe6cf11970e2885172620a"}, - {file = "tensorflow-2.18.1-cp39-cp39-macosx_12_0_arm64.whl", hash = "sha256:e0ffa318b969779baad01a11e7799dda9677ee33ccbbcdbf7b735c27f53d2a9b"}, - {file = "tensorflow-2.18.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1c0cd29c323908ed35ce72fbcce66f2ef7c8657f9c5024860ffd7ea64cf5d35d"}, - {file = "tensorflow-2.18.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ecbb9b3cd3f223ff6861faa1a4c2719c138d870dba90545826685b1c5ba5901"}, - {file = "tensorflow-2.18.1-cp39-cp39-win_amd64.whl", hash = "sha256:0f84a4c87a30cfb279c30b0077541cb5aaac7506d32adde585adb185277e49d2"}, -] - -[package.dependencies] -absl-py = ">=1.0.0" -astunparse = ">=1.6.0" -flatbuffers = ">=24.3.25" -gast = ">=0.2.1,<0.5.0 || >0.5.0,<0.5.1 || >0.5.1,<0.5.2 || >0.5.2" -google-pasta = ">=0.1.1" -grpcio = ">=1.24.3,<2.0" -h5py = ">=3.11.0" -keras = ">=3.5.0" -libclang = ">=13.0.0" -ml-dtypes = ">=0.4.0,<1.0.0" -numpy = ">=1.26.0,<2.1.0" -opt-einsum = ">=2.3.2" -packaging = "*" -protobuf = ">=3.20.3,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" -requests = ">=2.21.0,<3" -setuptools = "*" -six = ">=1.12.0" -tensorboard = ">=2.18,<2.19" -tensorflow-io-gcs-filesystem = {version = ">=0.23.1", markers = "python_version < \"3.12\""} -termcolor = ">=1.1.0" -typing-extensions = ">=3.6.6" -wrapt = ">=1.11.0" - -[package.extras] -and-cuda = ["nvidia-cublas-cu12 (==12.5.3.2)", "nvidia-cuda-cupti-cu12 (==12.5.82)", "nvidia-cuda-nvcc-cu12 (==12.5.82)", "nvidia-cuda-nvrtc-cu12 (==12.5.82)", "nvidia-cuda-runtime-cu12 (==12.5.82)", "nvidia-cudnn-cu12 (==9.3.0.75)", "nvidia-cufft-cu12 (==11.2.3.61)", "nvidia-curand-cu12 (==10.3.6.82)", "nvidia-cusolver-cu12 (==11.6.3.83)", "nvidia-cusparse-cu12 (==12.5.1.3)", "nvidia-nccl-cu12 (==2.21.5)", "nvidia-nvjitlink-cu12 (==12.5.82)"] - [[package]] name = "tensorflow" version = "2.19.0" description = "TensorFlow is an open source machine learning framework for everyone." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "tensorflow-2.19.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:c95604f25c3032e9591c7e01e457fdd442dde48e9cc1ce951078973ab1b4ca34"}, {file = "tensorflow-2.19.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b39293cae3aeee534dc4746dc6097b48c281e5e8b9a423efbd14d4495968e5c"}, @@ -2737,7 +3543,7 @@ ml-dtypes = ">=0.5.1,<1.0.0" numpy = ">=1.26.0,<2.2.0" opt-einsum = ">=2.3.2" packaging = "*" -protobuf = ">=3.20.3,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0dev" +protobuf = ">=3.20.3,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<6.0.0.dev0" requests = ">=2.21.0,<3" setuptools = "*" six = ">=1.12.0" @@ -2756,6 +3562,8 @@ version = "0.37.1" description = "TensorFlow IO" optional = false python-versions = "<3.13,>=3.7" +groups = ["main"] +markers = "python_version < \"3.12\"" files = [ {file = "tensorflow_io_gcs_filesystem-0.37.1-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:249c12b830165841411ba71e08215d0e94277a49c551e6dd5d72aab54fe5491b"}, {file = "tensorflow_io_gcs_filesystem-0.37.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:257aab23470a0796978efc9c2bcf8b0bc80f22e6298612a4c0a50d3f4e88060c"}, @@ -2788,6 +3596,7 @@ version = "3.1.0" description = "ANSI color formatting for output in terminal" optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "termcolor-3.1.0-py3-none-any.whl", hash = "sha256:591dd26b5c2ce03b9e43f391264626557873ce1d379019786f99b0c2bee140aa"}, {file = "termcolor-3.1.0.tar.gz", hash = "sha256:6a6dd7fbee581909eeec6a756cff1d7f7c376063b14e4a298dc4980309e55970"}, @@ -2797,39 +3606,25 @@ files = [ tests = ["pytest", "pytest-cov"] [[package]] -name = "tf-keras" -version = "2.18.0" -description = "Deep learning for humans." -optional = false -python-versions = ">=3.9" -files = [ - {file = "tf_keras-2.18.0-py3-none-any.whl", hash = "sha256:c431d04027eef790fcd3261cf7fdf93eb74f3cb32e05078b57b7f5a54bd53262"}, - {file = "tf_keras-2.18.0.tar.gz", hash = "sha256:ebf744519b322afead33086a2aba872245473294affd40973694f3eb7c7ad77d"}, -] - -[package.dependencies] -tensorflow = ">=2.18,<2.19" - -[[package]] -name = "tf-keras" -version = "2.19.0" -description = "Deep learning for humans." +name = "threadpoolctl" +version = "3.6.0" +description = "threadpoolctl" optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ - {file = "tf_keras-2.19.0-py3-none-any.whl", hash = "sha256:4f339e800987b39d1548a8c76a7b33b6801a97ec7fcd89c299ec29741f7890bd"}, - {file = "tf_keras-2.19.0.tar.gz", hash = "sha256:b09a407d87a4571ce1e8ca985cfc68483e3d63b2518a5d79a97ad92cb64dbe9c"}, + {file = "threadpoolctl-3.6.0-py3-none-any.whl", hash = "sha256:43a0b8fd5a2928500110039e43a5eed8480b918967083ea48dc3ab9f13c4a7fb"}, + {file = "threadpoolctl-3.6.0.tar.gz", hash = "sha256:8ab8b4aa3491d812b623328249fab5302a68d2d71745c8a4c719a2fcaba9f44e"}, ] -[package.dependencies] -tensorflow = ">=2.19,<2.20" - [[package]] name = "tomli" version = "2.2.1" description = "A lil' TOML parser" optional = false python-versions = ">=3.8" +groups = ["dev"] +markers = "python_version == \"3.10\"" files = [ {file = "tomli-2.2.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:678e4fa69e4575eb77d103de3df8a895e1591b48e740211bd1067378c69e8249"}, {file = "tomli-2.2.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:023aa114dd824ade0100497eb2318602af309e5a55595f76b626d6d9f3b7b0a6"}, @@ -2871,6 +3666,7 @@ version = "6.5.1" description = "Tornado is a Python web framework and asynchronous networking library, originally developed at FriendFeed." optional = false python-versions = ">=3.9" +groups = ["dev"] files = [ {file = "tornado-6.5.1-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:d50065ba7fd11d3bd41bcad0825227cc9a95154bad83239357094c36708001f7"}, {file = "tornado-6.5.1-cp39-abi3-macosx_10_9_x86_64.whl", hash = "sha256:9e9ca370f717997cb85606d074b0e5b247282cf5e2e1611568b8821afe0342d6"}, @@ -2892,6 +3688,7 @@ version = "5.14.3" description = "Traitlets Python configuration system" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "traitlets-5.14.3-py3-none-any.whl", hash = "sha256:b74e89e397b1ed28cc831db7aea759ba6640cb3de13090ca145426688ff1ac4f"}, {file = "traitlets-5.14.3.tar.gz", hash = "sha256:9ed0579d3502c94b4b3732ac120375cda96f923114522847de4b3bb98b96b6b7"}, @@ -2907,10 +3704,12 @@ version = "4.14.1" description = "Backported and Experimental Type Hints for Python 3.9+" optional = false python-versions = ">=3.9" +groups = ["main", "dev"] files = [ {file = "typing_extensions-4.14.1-py3-none-any.whl", hash = "sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76"}, {file = "typing_extensions-4.14.1.tar.gz", hash = "sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36"}, ] +markers = {dev = "python_version == \"3.10\""} [[package]] name = "tzdata" @@ -2918,6 +3717,7 @@ version = "2025.2" description = "Provider of IANA time zone data" optional = false python-versions = ">=2" +groups = ["main"] files = [ {file = "tzdata-2025.2-py2.py3-none-any.whl", hash = "sha256:1a403fada01ff9221ca8044d701868fa132215d84beb92242d9acd2147f667a8"}, {file = "tzdata-2025.2.tar.gz", hash = "sha256:b60a638fcc0daffadf82fe0f57e53d06bdec2f36c4df66280ae79bce6bd6f2b9"}, @@ -2929,13 +3729,14 @@ version = "2.5.0" description = "HTTP library with thread-safe connection pooling, file post, and more." optional = false python-versions = ">=3.9" +groups = ["main", "doc"] files = [ {file = "urllib3-2.5.0-py3-none-any.whl", hash = "sha256:e6b01673c0fa6a13e374b50871808eb3bf7046c4b125b216f6bf1cc604cff0dc"}, {file = "urllib3-2.5.0.tar.gz", hash = "sha256:3fc47733c7e419d4bc3f6b3dc2b4f890bb743906a30d56ba4a5bfa4bbff92760"}, ] [package.extras] -brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] +brotli = ["brotli (>=1.0.9) ; platform_python_implementation == \"CPython\"", "brotlicffi (>=0.8.0) ; platform_python_implementation != \"CPython\""] h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] @@ -2946,6 +3747,7 @@ version = "0.1.0" description = "Flexible version handling" optional = false python-versions = "*" +groups = ["doc"] files = [ {file = "verspec-0.1.0-py3-none-any.whl", hash = "sha256:741877d5633cc9464c45a469ae2a31e801e6dbbaa85b9675d481cda100f11c31"}, {file = "verspec-0.1.0.tar.gz", hash = "sha256:c4504ca697b2056cdb4bfa7121461f5a0e81809255b41c03dda4ba823637c01e"}, @@ -2960,6 +3762,7 @@ version = "20.32.0" description = "Virtual Python Environment builder" optional = false python-versions = ">=3.8" +groups = ["dev"] files = [ {file = "virtualenv-20.32.0-py3-none-any.whl", hash = "sha256:2c310aecb62e5aa1b06103ed7c2977b81e042695de2697d01017ff0f1034af56"}, {file = "virtualenv-20.32.0.tar.gz", hash = "sha256:886bf75cadfdc964674e6e33eb74d787dff31ca314ceace03ca5810620f4ecf0"}, @@ -2972,7 +3775,7 @@ platformdirs = ">=3.9.1,<5" [package.extras] docs = ["furo (>=2023.7.26)", "proselint (>=0.13)", "sphinx (>=7.1.2,!=7.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=23.6)"] -test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8)", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10)"] +test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23.1)", "pytest (>=7.4)", "pytest-env (>=0.8.2)", "pytest-freezer (>=0.4.8) ; platform_python_implementation == \"PyPy\" or platform_python_implementation == \"GraalVM\" or platform_python_implementation == \"CPython\" and sys_platform == \"win32\" and python_version >= \"3.13\"", "pytest-mock (>=3.11.1)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)", "setuptools (>=68)", "time-machine (>=2.10) ; platform_python_implementation == \"CPython\""] [[package]] name = "watchdog" @@ -2980,6 +3783,7 @@ version = "6.0.0" description = "Filesystem events monitoring" optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:d1cdb490583ebd691c012b3d6dae011000fe42edb7a82ece80965b42abd61f26"}, {file = "watchdog-6.0.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bc64ab3bdb6a04d69d4023b29422170b74681784ffb9463ed4870cf2f3e66112"}, @@ -3022,6 +3826,7 @@ version = "0.2.13" description = "Measures the displayed width of unicode strings in a terminal" optional = false python-versions = "*" +groups = ["dev"] files = [ {file = "wcwidth-0.2.13-py2.py3-none-any.whl", hash = "sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859"}, {file = "wcwidth-0.2.13.tar.gz", hash = "sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"}, @@ -3033,6 +3838,7 @@ version = "3.1.3" description = "The comprehensive WSGI web application library." optional = false python-versions = ">=3.9" +groups = ["main"] files = [ {file = "werkzeug-3.1.3-py3-none-any.whl", hash = "sha256:54b78bf3716d19a65be4fceccc0d1d7b89e608834989dfae50ea87564639213e"}, {file = "werkzeug-3.1.3.tar.gz", hash = "sha256:60723ce945c19328679790e3282cc758aa4a6040e4bb330f53d30fa546d44746"}, @@ -3050,6 +3856,7 @@ version = "0.45.1" description = "A built-package format for Python" optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "wheel-0.45.1-py3-none-any.whl", hash = "sha256:708e7481cc80179af0e556bbf0cc00b8444c7321e2700b8d8580231d13017248"}, {file = "wheel-0.45.1.tar.gz", hash = "sha256:661e1abd9198507b1409a20c02106d9670b2576e916d58f520316666abca6729"}, @@ -3064,13 +3871,15 @@ version = "1.2.0" description = "A small Python utility to set file creation time on Windows" optional = false python-versions = ">=3.5" +groups = ["main"] +markers = "sys_platform == \"win32\"" files = [ {file = "win32_setctime-1.2.0-py3-none-any.whl", hash = "sha256:95d644c4e708aba81dc3704a116d8cbc974d70b3bdb8be1d150e36be6e9d1390"}, {file = "win32_setctime-1.2.0.tar.gz", hash = "sha256:ae1fdf948f5640aae05c511ade119313fb6a30d7eabe25fef9764dca5873c4c0"}, ] [package.extras] -dev = ["black (>=19.3b0)", "pytest (>=4.6.2)"] +dev = ["black (>=19.3b0) ; python_version >= \"3.6\"", "pytest (>=4.6.2)"] [[package]] name = "wrapt" @@ -3078,6 +3887,7 @@ version = "1.17.2" description = "Module for decorators, wrappers and monkey patching." optional = false python-versions = ">=3.8" +groups = ["main"] files = [ {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d57c572081fed831ad2d26fd430d565b76aa277ed1d30ff4d40670b1c0dd984"}, {file = "wrapt-1.17.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5e251054542ae57ac7f3fba5d10bfff615b6c2fb09abeb37d2f1463f841ae22"}, @@ -3166,13 +3976,14 @@ version = "3.23.0" description = "Backport of pathlib-compatible object wrapper for zip files" optional = false python-versions = ">=3.9" +groups = ["doc"] files = [ {file = "zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e"}, {file = "zipp-3.23.0.tar.gz", hash = "sha256:a07157588a12518c9d4034df3fbbee09c814741a33ff63c05fa29d26a2404166"}, ] [package.extras] -check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1) ; sys_platform != \"cygwin\""] cover = ["pytest-cov"] doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] enabler = ["pytest-enabler (>=2.2)"] @@ -3180,6 +3991,6 @@ test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_it type = ["pytest-mypy"] [metadata] -lock-version = "2.0" -python-versions = ">=3.9,<4.0" -content-hash = "eb72db6a5fd7df7e7cd7cf1352afad181bb430643227aedc38ff50fc0e89f852" +lock-version = "2.1" +python-versions = ">=3.10,<4.0" +content-hash = "a72d64b3bdfdcec755b816a3e8cc0d0d01c7acd9e6c0b77e6cfe46762b8407d9" diff --git a/pyproject.toml b/pyproject.toml index 5ff8052..a937c20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,19 +9,32 @@ authors = ["Piotr Laczkowski "] license = "MIT" readme = "README.md" include = ["docs/getting-started/imgs/kdp_logo.png"] +keywords = ["keras", "tensorflow", "preprocessing", "tabular", "feature-engineering"] +# Poetry derives the Python-version and license classifiers from the fields +# above; only the ones it cannot infer are listed here. +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "Intended Audience :: Science/Research", + "Topic :: Scientific/Engineering :: Artificial Intelligence", +] [build-system] requires = ["poetry-core"] build-backend = "poetry.core.masonry.api" [tool.poetry.dependencies] -python = ">=3.9,<4.0" +# The package uses PEP 604 unions (`X | Y`) in annotations that are evaluated at +# import time, so 3.10 is the real floor -- `import kdp` fails on 3.9. +python = ">=3.10,<4.0" tensorflow = "^2.18.0" -tf-keras = "^2.18.0" +# Declared explicitly rather than inherited from TensorFlow: the package targets +# the Keras 3 API directly (`import keras`) instead of the `tf.keras` +# compatibility shim, so the requirement belongs here. +keras = ">=3.5,<4.0" numpy = ">=1.22.0" pandas = "^2.2.0" loguru = "^0.7.2" -parameterized = "^0.9.0" [tool.poetry.group.dev.dependencies] pre-commit = "^3.6.0" @@ -35,10 +48,15 @@ pytest-timeout = "^2.2.0" pytest-sugar = "^0.9.7" pytest-html = "^4.1.1" pytest-parallel = "^0.1.1" +parameterized = "^0.9.0" psutil = "^5.9.0" ipykernel = "^6.29.3" pydot = "^2.0.0" scipy = "^1.12.0" +# Used by the scripts in examples/, which are referenced from the docs and were +# not runnable from a plain `poetry install`. +matplotlib = "^3.8.0" +scikit-learn = "^1.4.0" [tool.poetry.group.doc.dependencies] mkdocs = "^1.5.3" @@ -51,52 +69,12 @@ mkdocs-redirects = "^1.2.2" mike = ">=2.0.0" griffe = "^0.25.0" -[tool.pytest.ini_options] -python_paths = "kdp" -testpaths = "test" -# Enable maximum parallel execution for ALL tests with optimizations -addopts = [ - "-n", "auto", - "--strict-markers", - "--strict-config", - "--tb=short", - "--cov=kdp", - "--cov-report=term-missing:skip-covered", - "--cov-report=html:htmlcov", - "--cov-report=xml", - "--cov-fail-under=75", - "--durations=10", - "--timeout=180", - "--maxfail=10", - "--disable-warnings" -] -# Test markers for categorization -markers = [ - "unit: Unit tests that run quickly", - "integration: Integration tests that may take longer", - "slow: Tests that take a long time to run", - "fast: Tests that run very quickly", - "time_series: Time series specific tests", - "layers: Layer-specific tests", - "processor: Processor-specific tests", - "inference: Inference-related tests", - "performance: Performance benchmark tests" -] -# Minimum version requirements -minversion = "6.0" -# Test discovery patterns -python_files = ["test_*.py", "*_test.py"] -python_classes = ["Test*"] -python_functions = ["test_*"] -# Timeout for individual tests (5 minutes) -timeout = 300 -# Parallel execution settings -filterwarnings = [ - "ignore::DeprecationWarning", - "ignore::PendingDeprecationWarning", - "ignore::UserWarning:tensorflow.*", - "ignore::FutureWarning:tensorflow.*" -] +# Pytest is configured by pytest.ini (plus the pytest-fast/parallel/ultrafast +# variants). A [tool.pytest.ini_options] block used to live here too, but +# pytest.ini takes precedence over pyproject.toml, so it never applied -- which +# hid the fact that it set the invalid key `python_paths` (the option is +# `pythonpath`) and omitted the `micro` and `smoke` markers that CI selects +# under --strict-markers. [tool.coverage.run] branch = true @@ -131,7 +109,7 @@ exclude_lines = [ [tool.isort] profile = "black" -line_length = 120 +line_length = 88 multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 @@ -139,9 +117,10 @@ use_parentheses = true ensure_newline_before_comments = true [tool.ruff] -line-length = 120 +# Matches the formatter default the codebase is written against. +line-length = 88 indent-width = 4 -target-version = "py311" +target-version = "py310" extend-unsafe-fixes = ["D"] [tool.ruff.lint] @@ -245,5 +224,27 @@ line-ending = "auto" [tool.ruff.lint.pydocstyle] convention = "google" +[tool.ruff.lint.flake8-unused-arguments] +# `layer_creator` callables and Keras hooks accept **kwargs they never read. +ignore-variadic-names = true + +[tool.ruff.lint.pycodestyle] +# The formatter wraps code at `line-length`; prose in docstrings and comments is +# allowed to run wider before E501 fires. +max-line-length = 120 + [tool.ruff.lint.per-file-ignores] -"**/{test,docs}/*" = ["ALL"] +# `*` does not cross directory separators, so the previous "**/{test,docs}/*" +# pattern silently missed everything under test/layers/ and friends. +"test/**" = ["ALL"] +"docs/**" = ["ALL"] +# Developer entry points: printing to stdout is the point, os.path is fine in a +# one-off script, and they are not part of the published API surface. +"scripts/**" = ["T20", "ANN", "D", "N806", "PTH", "S603", "S607"] +# `X_train` and friends are the conventional ML spelling. +"examples/**" = ["T20", "ANN", "D", "N806", "PTH"] +# Keras fixes the signatures of `call`, `build` and `__init__`: `training` and +# `**kwargs` must be accepted whether or not a given layer reads them, and the +# `tf.while_loop` condition callable must accept the full loop-variable tuple. +"kdp/layers/**" = ["ARG002"] +"kdp/moe.py" = ["ARG002"] diff --git a/scripts/analyze_dataset.py b/scripts/analyze_dataset.py index b6083bf..44b1532 100755 --- a/scripts/analyze_dataset.py +++ b/scripts/analyze_dataset.py @@ -10,7 +10,6 @@ import json import os from pathlib import Path -from typing import Optional, Dict from loguru import logger @@ -23,7 +22,7 @@ def parse_arguments(): """Parse command line arguments.""" parser = argparse.ArgumentParser( - description="Analyze a dataset and recommend optimal preprocessing configurations." + description="Analyze a dataset and recommend optimal preprocessing configurations.", ) parser.add_argument( "--data", @@ -72,10 +71,10 @@ def parse_arguments(): return parser.parse_args() -def load_feature_types(file_path: Path) -> Optional[Dict[str, FeatureType]]: +def load_feature_types(file_path: Path) -> dict[str, FeatureType] | None: """Load feature types from a JSON file.""" try: - with open(file_path, "r") as f: + with open(file_path) as f: feature_types = json.load(f) features_specs = {} @@ -96,7 +95,7 @@ def load_feature_types(file_path: Path) -> Optional[Dict[str, FeatureType]]: except KeyError: logger.warning( f"Unknown feature type '{feature_type}' for feature '{feature_name}'. " - f"Defaulting to FLOAT_NORMALIZED." + f"Defaulting to FLOAT_NORMALIZED.", ) features_specs[feature_name] = FeatureType.FLOAT_NORMALIZED diff --git a/scripts/cleanup_unused_diagrams.py b/scripts/cleanup_unused_diagrams.py new file mode 100755 index 0000000..079bc0e --- /dev/null +++ b/scripts/cleanup_unused_diagrams.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python +"""Report diagram files that no documentation page references. + +`make identify_unused_diagrams` (and therefore `make clean_old_diagrams` and +`make clean`) has always invoked this script, but the file was missing from the +repository, so those targets failed outright. + +The scan is deliberately read-only: it writes a report and leaves every file in +place, because "unreferenced" is a hint, not proof -- an image may be pulled in +by a template, a generated page or an external site. +""" + +from __future__ import annotations + +import argparse +import re +import sys +from pathlib import Path + +IMAGE_SUFFIXES = {".png", ".jpg", ".jpeg", ".gif", ".svg", ".webp"} +TEXT_SUFFIXES = {".md", ".yml", ".yaml", ".html", ".js", ".css", ".py", ".sh"} +SKIP_DIRS = {".git", ".venv", "venv", "site", "node_modules", "__pycache__", "dist"} + + +def iter_files(root: Path, suffixes: set[str]): + """Yield files under root with one of the given suffixes. + + Args: + root: Directory to walk. + suffixes: Lower-case file extensions to keep. + + Yields: + Matching paths, skipping build and vendor directories. + """ + for path in root.rglob("*"): + if any(part in SKIP_DIRS for part in path.parts): + continue + if path.is_file() and path.suffix.lower() in suffixes: + yield path + + +def find_referenced_names(root: Path) -> set[str]: + """Collect every image file name mentioned anywhere in the project. + + Matching is by base name rather than by resolved path: pages reference the + same image through relative paths, site-root paths and mkdocs redirects, and + a name match keeps the report conservative (it never claims a referenced + image is unused). + + Args: + root: Project root to scan. + + Returns: + The set of image file names that appear in any text file. + """ + referenced: set[str] = set() + name_pattern = re.compile( + r"[\w\-.]+\.(?:png|jpe?g|gif|svg|webp)", + re.IGNORECASE, + ) + for path in iter_files(root, TEXT_SUFFIXES): + try: + content = path.read_text(errors="ignore") + except OSError: + continue + referenced.update(match.lower() for match in name_pattern.findall(content)) + return referenced + + +def main() -> int: + """Scan the project and write the unused-diagram report. + + Returns: + Process exit code; always 0 so the report never breaks `make clean`. + """ + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + type=Path, + default=Path(__file__).resolve().parent.parent, + help="Project root to scan (default: the repository root).", + ) + parser.add_argument( + "--report", + type=Path, + default=Path("unused_diagrams_report.txt"), + help="Where to write the report.", + ) + args = parser.parse_args() + + root: Path = args.root + docs_dir = root / "docs" + if not docs_dir.is_dir(): + sys.stderr.write(f"No docs directory found at {docs_dir}\n") + return 0 + + referenced = find_referenced_names(root) + images = sorted(iter_files(docs_dir, IMAGE_SUFFIXES)) + unused = [img for img in images if img.name.lower() not in referenced] + + total_bytes = sum(img.stat().st_size for img in unused) + lines = [ + "Unused diagram report", + "=====================", + "", + f"Scanned {len(images)} image(s) under {docs_dir.relative_to(root)}.", + f"{len(unused)} appear unreferenced ({total_bytes / 1024:.1f} KiB).", + "", + "Nothing has been deleted. Review the list before removing anything:", + "an image can still be used by a template or an external page.", + "", + ] + lines.extend(str(img.relative_to(root)) for img in unused) + args.report.write_text("\n".join(lines) + "\n") + + sys.stdout.write( + f"Scanned {len(images)} image(s); {len(unused)} unreferenced. " + f"Report written to {args.report}\n", + ) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/generate_docstring_docs.py b/scripts/generate_docstring_docs.py index 09548f7..e0badc8 100644 --- a/scripts/generate_docstring_docs.py +++ b/scripts/generate_docstring_docs.py @@ -4,6 +4,7 @@ This allows for automatic documentation generation directly from the code. """ +import importlib import os import re import inspect @@ -15,7 +16,17 @@ if project_root not in sys.path: sys.path.insert(0, project_root) -# Import required modules +MODULES_TO_DOCUMENT = [ + "kdp.processor", + "kdp.dynamic_pipeline", + "kdp.features", + "kdp.layers.distribution_aware_encoder_layer", + "kdp.layers.distribution_transform_layer", + "kdp.layers.global_numerical_embedding_layer", + "kdp.layers.numerical_embedding_layer", + "kdp.layers.tabular_attention_layer", + "kdp.layers.multi_resolution_tabular_attention_layer", +] def docstring_to_markdown(docstring): @@ -42,7 +53,7 @@ def docstring_to_markdown(docstring): # Remove this indentation from all lines docstring = "\n".join( [lines[0]] - + [line[min_indent:] if line.strip() else line for line in lines[1:]] + + [line[min_indent:] if line.strip() else line for line in lines[1:]], ) # Replace reST/Google-style formatting with Markdown equivalents @@ -57,22 +68,30 @@ def docstring_to_markdown(docstring): # Convert Returns: section to Markdown markdown = re.sub( - r"Returns:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", r"### Returns\n\n\1", markdown + r"Returns:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", + r"### Returns\n\n\1", + markdown, ) # Convert Raises: section to Markdown markdown = re.sub( - r"Raises:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", r"### Raises\n\n\1", markdown + r"Raises:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", + r"### Raises\n\n\1", + markdown, ) # Convert Examples: section to Markdown markdown = re.sub( - r"Examples?:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", r"### Examples\n\n\1", markdown + r"Examples?:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", + r"### Examples\n\n\1", + markdown, ) # Convert Note: section to Markdown markdown = re.sub( - r"Note:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", r"### Notes\n\n\1", markdown + r"Note:\s*\n(\s+[^\n]*\n(?:\s+[^\n]*\n)*)", + r"### Notes\n\n\1", + markdown, ) # Convert parameter descriptions to bullet points @@ -89,9 +108,7 @@ def param_replace(match): ) # Remove trailing spaces - markdown = "\n".join(line.rstrip() for line in markdown.split("\n")) - - return markdown + return "\n".join(line.rstrip() for line in markdown.split("\n")) def extract_class_docs(cls): @@ -177,7 +194,7 @@ def generate_module_index(modules, output_dir): with open(output_file, "w") as f: f.write("# API Reference\n\n") f.write( - "This section provides detailed API documentation extracted directly from the codebase.\n\n" + "This section provides detailed API documentation extracted directly from the codebase.\n\n", ) for module in modules: @@ -206,18 +223,10 @@ def extract_docstrings(): # Define output directory output_dir = "docs/generated" - # List of modules to document - modules = [ - sys.modules["kdp.processor"], - sys.modules["kdp.dynamic_pipeline"], - sys.modules["kdp.features"], - sys.modules["kdp.layers.distribution_aware_encoder_layer"], - sys.modules["kdp.layers.distribution_transform_layer"], - sys.modules["kdp.layers.global_numerical_embedding_layer"], - sys.modules["kdp.layers.numerical_embedding_layer"], - sys.modules["kdp.layers.tabular_attention_layer"], - sys.modules["kdp.layers.multi_resolution_tabular_attention_layer"], - ] + # Import the modules rather than reading them out of sys.modules: nothing + # here imported them, so every lookup raised KeyError and this script -- + # which `make generate_doc_content` runs -- could never complete. + modules = [importlib.import_module(name) for name in MODULES_TO_DOCUMENT] # Process each module for module in modules: diff --git a/scripts/generate_model_diagrams.py b/scripts/generate_model_diagrams.py index b8bca23..f01ff60 100755 --- a/scripts/generate_model_diagrams.py +++ b/scripts/generate_model_diagrams.py @@ -20,6 +20,7 @@ make generate_doc_content """ +import keras import os import tempfile from pathlib import Path @@ -52,10 +53,9 @@ def generate_fake_data(features_specs, num_rows=20): """Generate fake data for testing various feature types.""" data = {} for feature_name, spec in features_specs.items(): - if isinstance(spec, FeatureType) or isinstance(spec, str): - feature_type = spec - else: - feature_type = spec.feature_type + feature_type = ( + spec if isinstance(spec, FeatureType | str) else spec.feature_type + ) if ( feature_type == FeatureType.FLOAT @@ -72,7 +72,7 @@ def generate_fake_data(features_specs, num_rows=20): elif feature_type == FeatureType.STRING_CATEGORICAL: categories = ["A", "B", "C", "D", "E"] data[feature_name] = pd.Series( - [categories[np.random.randint(0, 5)] for _ in range(num_rows)] + [categories[np.random.randint(0, 5)] for _ in range(num_rows)], ) elif feature_type == FeatureType.TEXT: texts = [ @@ -83,12 +83,12 @@ def generate_fake_data(features_specs, num_rows=20): "Natural language processing", ] data[feature_name] = pd.Series( - [texts[np.random.randint(0, 5)] for _ in range(num_rows)] + [texts[np.random.randint(0, 5)] for _ in range(num_rows)], ) elif feature_type == FeatureType.DATE: dates = pd.date_range(start="1/1/2020", periods=10) data[feature_name] = pd.Series( - [dates[np.random.randint(0, 10)] for _ in range(num_rows)] + [dates[np.random.randint(0, 10)] for _ in range(num_rows)], ) elif feature_type == FeatureType.PASSTHROUGH: # For passthrough features, use a simple array of random values @@ -103,7 +103,7 @@ def generate_fake_data(features_specs, num_rows=20): date = pd.Timestamp("2022-01-01") + pd.Timedelta(days=i) value = base_value + i * 2 + np.random.normal(0, 1) all_data.append( - {feature_name: value, "date": date, "group_id": group} + {feature_name: value, "date": date, "group_id": group}, ) # If this is a time series feature, we need to create other columns too if "date" not in data: @@ -147,7 +147,7 @@ def generate_model_diagram(name, features_specs, **kwargs): output_path = OUTPUT_DIR / filename # Use TensorFlow's plot_model to generate the image - tf.keras.utils.plot_model( + keras.utils.plot_model( model, to_file=str(output_path), show_shapes=True, @@ -297,7 +297,7 @@ def main(): feature_type=FeatureType.FLOAT_RESCALED, use_embedding=True, embedding_dim=32, - ) + ), }, ) @@ -309,7 +309,7 @@ def main(): feature_type=FeatureType.STRING_CATEGORICAL, max_tokens=1000, embedding_dim=64, - ) + ), }, ) @@ -322,7 +322,7 @@ def main(): max_tokens=5000, embedding_dim=64, sequence_length=128, - ) + ), }, ) @@ -335,7 +335,7 @@ def main(): add_day_of_week=True, add_month=True, cyclical_encoding=True, - ) + ), }, ) @@ -343,8 +343,10 @@ def main(): "custom_passthrough_feature", { "embedding": PassthroughFeature( - name="embedding", feature_type=FeatureType.PASSTHROUGH, dtype=tf.float32 - ) + name="embedding", + feature_type=FeatureType.PASSTHROUGH, + dtype=tf.float32, + ), }, ) diff --git a/scripts/generate_time_series_diagrams.py b/scripts/generate_time_series_diagrams.py index 08be924..58c3c6d 100644 --- a/scripts/generate_time_series_diagrams.py +++ b/scripts/generate_time_series_diagrams.py @@ -1,5 +1,5 @@ #!/usr/bin/env python -import tensorflow as tf +import keras from pathlib import Path from kdp.layers.time_series.lag_feature_layer import LagFeatureLayer @@ -15,22 +15,22 @@ def create_simple_model(name, layer): """Create a simple model with a single time series layer for diagram generation.""" # Create a simple input - inputs = tf.keras.Input(shape=(1,), name="sales") + inputs = keras.Input(shape=(1,), name="sales") # Apply normalization for better visualization - norm = tf.keras.layers.Normalization()(inputs) + norm = keras.layers.Normalization()(inputs) # Apply the time series layer outputs = layer(norm) # Create model - model = tf.keras.Model(inputs=inputs, outputs=outputs, name=f"time_series_{name}") + model = keras.Model(inputs=inputs, outputs=outputs, name=f"time_series_{name}") # Generate diagram filename = f"{name}.png" output_path = OUTPUT_DIR / filename - tf.keras.utils.plot_model( + keras.utils.plot_model( model, to_file=str(output_path), show_shapes=True, @@ -48,23 +48,23 @@ def create_simple_model(name, layer): def create_combined_model(name, layers): """Create a model with multiple time series layers in sequence.""" # Create a simple input - inputs = tf.keras.Input(shape=(1,), name="sales") + inputs = keras.Input(shape=(1,), name="sales") # Apply normalization for better visualization - x = tf.keras.layers.Normalization()(inputs) + x = keras.layers.Normalization()(inputs) # Apply all the time series layers sequentially for layer in layers: x = layer(x) # Create model - model = tf.keras.Model(inputs=inputs, outputs=x, name=f"time_series_{name}") + model = keras.Model(inputs=inputs, outputs=x, name=f"time_series_{name}") # Generate diagram filename = f"{name}.png" output_path = OUTPUT_DIR / filename - tf.keras.utils.plot_model( + keras.utils.plot_model( model, to_file=str(output_path), show_shapes=True, @@ -84,7 +84,8 @@ def main(): # Basic time series create_simple_model( - "basic_time_series", tf.keras.layers.LayerNormalization(name="time_series_norm") + "basic_time_series", + keras.layers.LayerNormalization(name="time_series_norm"), ) # Time series with lag features @@ -102,7 +103,10 @@ def main(): create_simple_model( "time_series_moving_average", MovingAverageLayer( - periods=[3, 7, 14], keep_original=True, name="moving_average", drop_na=False + periods=[3, 7, 14], + keep_original=True, + name="moving_average", + drop_na=False, ), ) @@ -110,7 +114,10 @@ def main(): create_simple_model( "time_series_differencing", DifferencingLayer( - order=1, keep_original=True, name="differencing", drop_na=False + order=1, + keep_original=True, + name="differencing", + drop_na=False, ), ) @@ -137,10 +144,16 @@ def main(): drop_na=False, ), MovingAverageLayer( - periods=[7], keep_original=True, name="moving_average", drop_na=False + periods=[7], + keep_original=True, + name="moving_average", + drop_na=False, ), DifferencingLayer( - order=1, keep_original=True, name="differencing", drop_na=False + order=1, + keep_original=True, + name="differencing", + drop_na=False, ), RollingStatsLayer( window_size=5, diff --git a/scripts/optimize_test_data.py b/scripts/optimize_test_data.py index 46982af..a308b48 100755 --- a/scripts/optimize_test_data.py +++ b/scripts/optimize_test_data.py @@ -8,7 +8,7 @@ import re from pathlib import Path -from typing import Dict, Any +from typing import Any class TestDataOptimizer: @@ -18,7 +18,7 @@ def __init__(self, test_dir: Path = None): self.test_dir = test_dir or Path("test") self.optimizations_applied = [] - def optimize_all_tests(self) -> Dict[str, Any]: + def optimize_all_tests(self) -> dict[str, Any]: """Optimize all test files for faster execution.""" results = { "files_processed": 0, @@ -37,7 +37,7 @@ def optimize_all_tests(self) -> Dict[str, Any]: def optimize_test_file(self, file_path: Path) -> bool: """Optimize a single test file.""" try: - with open(file_path, "r", encoding="utf-8") as f: + with open(file_path, encoding="utf-8") as f: content = f.read() original_content = content @@ -78,7 +78,9 @@ def reduce_data_sizes(self, content: str) -> str: ( r"np\.random\.choice\([^,]+,\s*(\d{3,})\)", lambda m: re.sub( - r"(\d{3,})", str(min(50, int(m.group(1)))), m.group(0) + r"(\d{3,})", + str(min(50, int(m.group(1)))), + m.group(0), ), ), ] @@ -173,7 +175,7 @@ def create_fast_test_variants(self) -> None: slow_tests = [] for test_file in self.test_dir.rglob("test_*.py"): - with open(test_file, "r") as f: + with open(test_file) as f: content = f.read() # Find slow tests @@ -189,7 +191,7 @@ def create_fast_test_variants(self) -> None: def create_fast_variant(self, original_path: Path, fast_path: Path) -> None: """Create a fast variant of a test file.""" - with open(original_path, "r") as f: + with open(original_path) as f: content = f.read() # Aggressive optimizations for fast variant diff --git a/scripts/test_optimizer.py b/scripts/test_optimizer.py index c9a329e..a229186 100755 --- a/scripts/test_optimizer.py +++ b/scripts/test_optimizer.py @@ -15,8 +15,8 @@ import sys import time from pathlib import Path -from typing import Dict, List, Tuple, Any -import xml.etree.ElementTree as ET +from typing import Any +from xml.etree import ElementTree # noqa: S405 - parses our own pytest output class TestOptimizer: @@ -27,15 +27,18 @@ def __init__(self, project_root: Path = None): self.test_dir = self.project_root / "test" def analyze_test_performance( - self, junit_xml_path: str = "pytest.xml" - ) -> Dict[str, Any]: + self, + junit_xml_path: str = "pytest.xml", + ) -> dict[str, Any]: """Analyze test performance from JUnit XML output.""" if not Path(junit_xml_path).exists(): print(f"JUnit XML file not found: {junit_xml_path}") return {} try: - tree = ET.parse(junit_xml_path) + tree = ElementTree.parse( # noqa: S314 - input is this script's own JUnit report + junit_xml_path, + ) root = tree.getroot() test_results = [] @@ -52,14 +55,14 @@ def analyze_test_performance( "classname": classname, "time": time_taken, "full_name": f"{classname}::{name}", - } + }, ) total_time += time_taken # Sort by time taken (slowest first) test_results.sort(key=lambda x: x["time"], reverse=True) - analysis = { + return { "total_tests": len(test_results), "total_time": total_time, "average_time": total_time / len(test_results) if test_results else 0, @@ -69,13 +72,11 @@ def analyze_test_performance( "slow_tests": [t for t in test_results if t["time"] > 5.0], } - return analysis - except Exception as e: print(f"Error analyzing test performance: {e}") return {} - def suggest_optimizations(self, analysis: Dict[str, Any]) -> List[str]: + def suggest_optimizations(self, analysis: dict[str, Any]) -> list[str]: """Suggest optimizations based on test analysis.""" suggestions = [] @@ -95,13 +96,13 @@ def suggest_optimizations(self, analysis: Dict[str, Any]) -> List[str]: avg_time = analysis.get("average_time", 0) if avg_time > 2.0: suggestions.append( - f"Average test time is {avg_time:.2f}s - consider optimization" + f"Average test time is {avg_time:.2f}s - consider optimization", ) total_time = analysis.get("total_time", 0) if total_time > 300: # 5 minutes suggestions.append( - f"Total test time is {total_time:.2f}s - consider parallel execution" + f"Total test time is {total_time:.2f}s - consider parallel execution", ) return suggestions @@ -112,7 +113,7 @@ def run_test_suite( parallel: bool = True, coverage: bool = False, verbose: bool = False, - ) -> Tuple[int, str]: + ) -> tuple[int, str]: """Run test suite with specified configuration.""" cmd = ["poetry", "run", "pytest"] @@ -123,7 +124,7 @@ def run_test_suite( # Add coverage if coverage: cmd.extend( - ["--cov=kdp", "--cov-report=term-missing", "--cov-report=html:htmlcov"] + ["--cov=kdp", "--cov-report=term-missing", "--cov-report=html:htmlcov"], ) # Add verbosity @@ -144,7 +145,10 @@ def run_test_suite( try: result = subprocess.run( - cmd, capture_output=True, text=True, cwd=self.project_root + cmd, + capture_output=True, + text=True, + cwd=self.project_root, ) end_time = time.time() @@ -160,7 +164,7 @@ def run_test_suite( except Exception as e: return 1, f"Error running tests: {e}" - def benchmark_test_configurations(self) -> Dict[str, Any]: + def benchmark_test_configurations(self) -> dict[str, Any]: """Benchmark different test configurations.""" configurations = [ ("sequential", {"parallel": False, "coverage": False}), @@ -194,7 +198,7 @@ def benchmark_test_configurations(self) -> Dict[str, Any]: } print( - f"Configuration {config_name}: {execution_time:.2f}s (exit code: {returncode})" + f"Configuration {config_name}: {execution_time:.2f}s (exit code: {returncode})", ) return results @@ -205,7 +209,9 @@ def generate_test_report(self, output_file: str = "test_optimization_report.json # Run a quick analysis returncode, output = self.run_test_suite( - parallel=True, coverage=False, verbose=False + parallel=True, + coverage=False, + verbose=False, ) # Analyze performance @@ -268,10 +274,14 @@ def main(): help="Enable parallel execution", ) parser.add_argument( - "--no-parallel", action="store_true", help="Disable parallel execution" + "--no-parallel", + action="store_true", + help="Disable parallel execution", ) parser.add_argument( - "--coverage", action="store_true", help="Enable coverage reporting" + "--coverage", + action="store_true", + help="Enable coverage reporting", ) parser.add_argument("--verbose", action="store_true", help="Enable verbose output") @@ -295,7 +305,7 @@ def main(): print(f" {suggestion}") else: print( - "No performance data available. Run tests with --junitxml=pytest.xml first." + "No performance data available. Run tests with --junitxml=pytest.xml first.", ) elif args.benchmark: diff --git a/test/conftest.py b/test/conftest.py index c628000..87a43c4 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -9,6 +9,7 @@ - Aggressive speed optimizations """ +import keras import gc import os import tempfile @@ -110,7 +111,7 @@ def setup_test_environment(): yield # Cleanup after all tests - tf.keras.backend.clear_session() + keras.backend.clear_session() gc.collect() @@ -118,9 +119,9 @@ def setup_test_environment(): def cleanup_after_test(): """Lightweight cleanup after each test.""" yield - # Only clear session if needed (check if any models exist) - if hasattr(tf.keras.backend, "_SESSION") and tf.keras.backend._SESSION is not None: - tf.keras.backend.clear_session() + # Keras 3 has no TF1-style global session, so there is nothing to probe: + # clearing is cheap and keeps layer-name counters from leaking across tests. + keras.backend.clear_session() @pytest.fixture(scope="session") @@ -418,14 +419,14 @@ def pytest_runtest_setup(item): # Only clear session if it's a slow test if item.get_closest_marker("slow") or item.get_closest_marker("integration"): - tf.keras.backend.clear_session() + keras.backend.clear_session() def pytest_runtest_teardown(item, nextitem): """Teardown after each test run (minimal overhead).""" # Only do expensive cleanup for slow tests if item.get_closest_marker("slow") or item.get_closest_marker("integration"): - tf.keras.backend.clear_session() + keras.backend.clear_session() gc.collect() @@ -446,5 +447,5 @@ def pytest_sessionfinish(session, exitstatus): _TEST_DATA_CACHE.clear() # Final cleanup - tf.keras.backend.clear_session() + keras.backend.clear_session() gc.collect() diff --git a/test/integration/test_factory_integration.py b/test/integration/test_factory_integration.py index 67020b6..a414ab4 100644 --- a/test/integration/test_factory_integration.py +++ b/test/integration/test_factory_integration.py @@ -1,3 +1,4 @@ +import keras import unittest import os import numpy as np @@ -103,7 +104,7 @@ def test_basic_factory_layers(self): # Test combining multiple layers in sequence # First normalize, then apply distribution transform - normalize_layer = tf.keras.layers.Normalization(name="normalize") + normalize_layer = keras.layers.Normalization(name="normalize") # Adapt the normalization layer to the data before using it normalize_layer.adapt(age_data) diff --git a/test/layers/test_distribution_aware.py b/test/layers/test_distribution_aware.py index 6111a2f..1373770 100644 --- a/test/layers/test_distribution_aware.py +++ b/test/layers/test_distribution_aware.py @@ -1,5 +1,6 @@ """Tests for the DistributionAwareEncoder layer.""" +import keras import numpy as np import tensorflow as tf import os @@ -113,9 +114,9 @@ def test_automatic_detection(self): tensor_data = tf.convert_to_tensor(dist_data, dtype=tf.float32) # Create a small model with the encoder - inputs = tf.keras.Input(shape=(1,)) + inputs = keras.Input(shape=(1,)) encoded = encoder(inputs) - model = tf.keras.Model(inputs, encoded) + model = keras.Model(inputs, encoded) # Feed data through the model to trigger detection _ = model(tensor_data, training=True) @@ -167,9 +168,9 @@ def test_various_configurations(self): tensor_data = tf.convert_to_tensor(data["normal"], dtype=tf.float32) # Create a small model - inputs = tf.keras.Input(shape=(1,)) + inputs = keras.Input(shape=(1,)) encoded = encoder(inputs) - model = tf.keras.Model(inputs, encoded) + model = keras.Model(inputs, encoded) # Feed data through the model output = model(tensor_data) @@ -340,16 +341,16 @@ def test_in_model(self): y_train = (x_train[:, 0] > 0).astype(np.float32) # Simple binary task # Create a model with the encoder - disable periodic features to avoid shape issues - inputs = tf.keras.Input(shape=(5,)) + inputs = keras.Input(shape=(5,)) encoded = DistributionAwareEncoder( embedding_dim=16, detect_periodicity=False, # Disable periodic features to avoid gradient issues auto_detect=True, )(inputs) - hidden = tf.keras.layers.Dense(8, activation="relu")(encoded) - outputs = tf.keras.layers.Dense(1, activation="sigmoid")(hidden) + hidden = keras.layers.Dense(8, activation="relu")(encoded) + outputs = keras.layers.Dense(1, activation="sigmoid")(hidden) - model = tf.keras.Model(inputs, outputs) + model = keras.Model(inputs, outputs) model.compile( optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"] ) @@ -368,7 +369,7 @@ def test_in_model(self): model.save(model_path) # Use get_custom_objects to load the model custom_objects = get_custom_objects() - loaded_model = tf.keras.models.load_model( + loaded_model = keras.saving.load_model( model_path, custom_objects=custom_objects ) diff --git a/test/layers/test_distribution_aware_encoder.py b/test/layers/test_distribution_aware_encoder.py index df34b2f..7935a31 100644 --- a/test/layers/test_distribution_aware_encoder.py +++ b/test/layers/test_distribution_aware_encoder.py @@ -1,3 +1,4 @@ +import keras import tensorflow as tf import numpy as np from kdp.layers.distribution_aware_encoder_layer import DistributionAwareEncoder @@ -79,9 +80,9 @@ def test_automatic_detection(): tensor_data = tf.convert_to_tensor(dist_data, dtype=tf.float32) # Create a small model with the encoder - inputs = tf.keras.Input(shape=(tensor_data.shape[-1],)) + inputs = keras.Input(shape=(tensor_data.shape[-1],)) encoded = encoder(inputs) - model = tf.keras.Model(inputs, encoded) + model = keras.Model(inputs, encoded) # Feed data through the model to trigger detection _ = model(tensor_data, training=True) @@ -126,9 +127,9 @@ def test_various_configurations(): tensor_data = tf.convert_to_tensor(data["normal"], dtype=tf.float32) # Create a small model with matching input shape - inputs = tf.keras.Input(shape=(tensor_data.shape[-1],)) + inputs = keras.Input(shape=(tensor_data.shape[-1],)) encoded = encoder(inputs) - model = tf.keras.Model(inputs, encoded) + model = keras.Model(inputs, encoded) # Feed data through the model output = model(tensor_data) @@ -191,14 +192,14 @@ def test_in_model(): y_train = (x_train[:, 0] > 0).astype(np.float32) # Simple binary task # Create a model with the encoder - disable periodicity detection to avoid shape mismatches - inputs = tf.keras.Input(shape=(5,)) + inputs = keras.Input(shape=(5,)) encoded = DistributionAwareEncoder(embedding_dim=16, detect_periodicity=False)( inputs ) - hidden = tf.keras.layers.Dense(8, activation="relu")(encoded) - outputs = tf.keras.layers.Dense(1, activation="sigmoid")(hidden) + hidden = keras.layers.Dense(8, activation="relu")(encoded) + outputs = keras.layers.Dense(1, activation="sigmoid")(hidden) - model = tf.keras.Model(inputs, outputs) + model = keras.Model(inputs, outputs) model.compile(optimizer="adam", loss="binary_crossentropy", metrics=["accuracy"]) # Train for just a few epochs to verify it works @@ -210,7 +211,7 @@ def test_in_model(): print(" Testing model saving and loading...") try: model.save(model_path) - loaded_model = tf.keras.models.load_model(model_path) + loaded_model = keras.saving.load_model(model_path) print(" Model saved and loaded successfully") # Test for prediction consistency diff --git a/test/layers/test_distribution_encoder.py b/test/layers/test_distribution_encoder.py index d82dd20..1af04d8 100644 --- a/test/layers/test_distribution_encoder.py +++ b/test/layers/test_distribution_encoder.py @@ -1,3 +1,4 @@ +import keras # import numpy as np # import pytest # import tensorflow as tf @@ -148,10 +149,10 @@ # def test_graph_mode_compatibility(encoder): # """Test that the encoder works in graph mode.""" # # Create a simple model with the encoder -# inputs = tf.keras.layers.Input(shape=(1,)) +# inputs = keras.layers.Input(shape=(1,)) # encoded = encoder(inputs) -# outputs = tf.keras.layers.Dense(1)(encoded) -# model = tf.keras.Model(inputs=inputs, outputs=outputs) +# outputs = keras.layers.Dense(1)(encoded) +# model = keras.Model(inputs=inputs, outputs=outputs) # # Compile the model # model.compile(optimizer="adam", loss="mse") diff --git a/test/layers/test_distribution_transform_math.py b/test/layers/test_distribution_transform_math.py new file mode 100644 index 0000000..f901429 --- /dev/null +++ b/test/layers/test_distribution_transform_math.py @@ -0,0 +1,194 @@ +"""Numerical correctness tests for DistributionTransformLayer. + +The scale-oriented transforms (robust-scale, min-max, quantile) must agree with +their textbook definitions and must operate per feature, not across the whole +tensor. +""" + +import keras +import unittest + +import numpy as np +import tensorflow as tf + +from kdp.layers.distribution_transform_layer import DistributionTransformLayer + +ALL_TRANSFORMS = [ + "none", + "log", + "sqrt", + "box-cox", + "yeo-johnson", + "arcsinh", + "cube-root", + "logit", + "min-max", + "robust-scale", + "quantile", +] + + +def _multi_scale_data(seed: int = 0) -> np.ndarray: + """Three columns with deliberately different centres and spreads.""" + rng = np.random.default_rng(seed) + return np.stack( + [ + rng.normal(loc=50, scale=10, size=200), + rng.normal(loc=-3, scale=0.5, size=200), + rng.normal(loc=1000, scale=250, size=200), + ], + axis=1, + ).astype("float32") + + +class TestRobustScale(unittest.TestCase): + """robust-scale centres on the median and divides by the IQR.""" + + def test_matches_median_and_iqr_definition(self): + """Output equals (x - median) / IQR computed per column.""" + data = _multi_scale_data() + actual = np.asarray( + DistributionTransformLayer(transform_type="robust-scale")(tf.constant(data)) + ) + + median = np.median(data, axis=0) + iqr = np.percentile(data, 75, axis=0) - np.percentile(data, 25, axis=0) + np.testing.assert_allclose(actual, (data - median) / iqr, rtol=1e-4, atol=1e-4) + + def test_output_is_centred_and_unit_iqr_per_feature(self): + """Each transformed column has median ~0 and IQR ~1.""" + data = _multi_scale_data(seed=7) + actual = np.asarray( + DistributionTransformLayer(transform_type="robust-scale")(tf.constant(data)) + ) + + np.testing.assert_allclose(np.median(actual, axis=0), 0.0, atol=1e-4) + observed_iqr = np.percentile(actual, 75, axis=0) - np.percentile( + actual, 25, axis=0 + ) + np.testing.assert_allclose(observed_iqr, 1.0, rtol=1e-4) + + def test_constant_feature_does_not_divide_by_zero(self): + """A column with zero IQR yields finite output.""" + data = np.stack([np.full(50, 3.0), np.linspace(0, 1, 50)], axis=1).astype( + "float32" + ) + actual = np.asarray( + DistributionTransformLayer(transform_type="robust-scale")(tf.constant(data)) + ) + self.assertTrue(np.all(np.isfinite(actual))) + np.testing.assert_allclose(actual[:, 0], 0.0, atol=1e-6) + + +class TestQuantileTransform(unittest.TestCase): + """quantile ranks values within each feature.""" + + def test_each_feature_is_ranked_independently(self): + """Columns on very different scales both span the full output range.""" + data = np.stack( + [np.linspace(0, 1, 40), np.linspace(1000, 2000, 40)], axis=1 + ).astype("float32") + np.random.default_rng(0).shuffle(data) + + actual = np.asarray( + DistributionTransformLayer(transform_type="quantile")(tf.constant(data)) + ) + + for column in range(data.shape[1]): + self.assertLess(actual[:, column].min(), -0.9) + self.assertGreater(actual[:, column].max(), 0.9) + + def test_preserves_rank_order_within_each_feature(self): + """The transform is monotonic per column.""" + data = _multi_scale_data(seed=3) + actual = np.asarray( + DistributionTransformLayer(transform_type="quantile")(tf.constant(data)) + ) + for column in range(data.shape[1]): + np.testing.assert_array_equal( + np.argsort(data[:, column]), np.argsort(actual[:, column]) + ) + + def test_output_is_bounded(self): + """Values stay inside [-1, 1].""" + actual = np.asarray( + DistributionTransformLayer(transform_type="quantile")( + tf.constant(_multi_scale_data(seed=11)) + ) + ) + self.assertGreaterEqual(actual.min(), -1.0) + self.assertLessEqual(actual.max(), 1.0) + + +class TestMinMax(unittest.TestCase): + """min-max maps data into [0, 1].""" + + def test_scales_into_unit_interval(self): + """With clipping disabled, the observed range maps onto [0, 1].""" + data = _multi_scale_data(seed=5) + actual = np.asarray( + DistributionTransformLayer(transform_type="min-max", clip_values=False)( + tf.constant(data) + ) + ) + self.assertAlmostEqual(float(actual.min()), 0.0, places=5) + self.assertAlmostEqual(float(actual.max()), 1.0, places=5) + + +class TestGraphModeCompatibility(unittest.TestCase): + """Every transform must build inside a functional model.""" + + def test_all_transforms_work_with_a_dynamic_batch_dimension(self): + """Transforms run when the batch size is only known at call time.""" + data = np.abs(_multi_scale_data(seed=2)[:16]) / 5000.0 + for transform_type in ALL_TRANSFORMS: + with self.subTest(transform_type=transform_type): + keras.backend.clear_session() + inputs = keras.Input(shape=(3,)) + model = keras.Model( + inputs, + DistributionTransformLayer(transform_type=transform_type)(inputs), + ) + output = model(tf.constant(data, dtype=tf.float32)) + self.assertEqual(tuple(output.shape), (16, 3)) + + +class TestStatisticsHelpers(unittest.TestCase): + """The order-statistics helper backing robust-scale.""" + + def test_percentile_matches_numpy(self): + """_percentile reproduces numpy's linear interpolation.""" + layer = DistributionTransformLayer(transform_type="robust-scale") + data = _multi_scale_data(seed=13) + sorted_data = tf.sort(tf.constant(data), axis=0) + + for q in (0.0, 0.25, 0.5, 0.75, 1.0): + with self.subTest(q=q): + np.testing.assert_allclose( + np.asarray(layer._percentile(sorted_data, q)), + np.percentile(data, q * 100, axis=0), + rtol=1e-4, + atol=1e-4, + ) + + def test_compute_robust_statistics_returns_per_feature_values(self): + """min/max/median/IQR are computed along the batch axis.""" + layer = DistributionTransformLayer(transform_type="robust-scale") + data = _multi_scale_data(seed=17) + x_min, x_max, median, iqr = layer._compute_robust_statistics(tf.constant(data)) + + np.testing.assert_allclose( + np.asarray(x_min).ravel(), data.min(axis=0), rtol=1e-5 + ) + np.testing.assert_allclose( + np.asarray(x_max).ravel(), data.max(axis=0), rtol=1e-5 + ) + np.testing.assert_allclose( + np.asarray(median), np.median(data, axis=0), rtol=1e-4 + ) + expected_iqr = np.percentile(data, 75, axis=0) - np.percentile(data, 25, axis=0) + np.testing.assert_allclose(np.asarray(iqr), expected_iqr, rtol=1e-4) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/layers/test_layer_factory.py b/test/layers/test_layer_factory.py index de02493..6cf4645 100644 --- a/test/layers/test_layer_factory.py +++ b/test/layers/test_layer_factory.py @@ -1,3 +1,4 @@ +import keras import unittest import numpy as np import tensorflow as tf @@ -63,15 +64,15 @@ def test_create_layer(self): dense_layer = PreprocessorLayerFactory.create_layer( layer_class="Dense", units=10, activation="relu" ) - self.assertIsInstance(dense_layer, tf.keras.layers.Dense) + self.assertIsInstance(dense_layer, keras.layers.Dense) self.assertEqual(dense_layer.units, 10) self.assertEqual(dense_layer.activation.__name__, "relu") # Test with class object dropout_layer = PreprocessorLayerFactory.create_layer( - layer_class=tf.keras.layers.Dropout, rate=0.5 + layer_class=keras.layers.Dropout, rate=0.5 ) - self.assertIsInstance(dropout_layer, tf.keras.layers.Dropout) + self.assertIsInstance(dropout_layer, keras.layers.Dropout) self.assertEqual(dropout_layer.rate, 0.5) def test_distribution_aware_encoder(self): diff --git a/test/layers/test_moe.py b/test/layers/test_moe.py index 65c9bcf..ecf382d 100644 --- a/test/layers/test_moe.py +++ b/test/layers/test_moe.py @@ -8,7 +8,7 @@ import unittest import numpy as np import tensorflow as tf -from tensorflow import keras +import keras from pathlib import Path import tempfile diff --git a/test/layers/test_moving_average_layer.py b/test/layers/test_moving_average_layer.py index 009b497..b049268 100644 --- a/test/layers/test_moving_average_layer.py +++ b/test/layers/test_moving_average_layer.py @@ -217,30 +217,66 @@ def test_keep_original_true(self): self.assertAllClose(output_np, expected_output, rtol=1e-5) def test_2d_input(self): - """Test MovingAverageLayer with a 2D input tensor.""" - # Create a 2D input tensor (2 samples, 5 time steps) - input_data = [[1.0, 2.0, 3.0, 4.0, 5.0], [6.0, 7.0, 8.0, 9.0, 10.0]] + """Test MovingAverageLayer with a 2D input tensor. + + Time runs along axis 0 -- the layer sits behind a chronologically sorted + input in the preprocessing pipeline -- so each column is an independent + series. + """ + # 5 time steps, 2 series + input_data = [ + [1.0, 6.0], + [2.0, 7.0], + [3.0, 8.0], + [4.0, 9.0], + [5.0, 10.0], + ] input_tensor = tf.convert_to_tensor(input_data, dtype=tf.float32) - # Create a layer with period=3 and drop_na=True layer = MovingAverageLayer(periods=[3], drop_na=True, keep_original=False) + output_np = layer(input_tensor).numpy() - # Apply the layer - output = layer(input_tensor) - - # Convert to numpy for easier assertions - output_np = output.numpy() + # MA(3) per column: [1,2,3],[2,3,4],[3,4,5] and [6,7,8],[7,8,9],[8,9,10] + expected_output = np.array([[2.0, 7.0], [3.0, 8.0], [4.0, 9.0]]) - # Expected output for each sample: - # Sample 1: MA(3) of [1,2,3], [2,3,4], [3,4,5] - # Sample 2: MA(3) of [6,7,8], [7,8,9], [8,9,10] - expected_output = np.array([[2.0, 3.0, 4.0], [7.0, 8.0, 9.0]]) + self.assertEqual(output_np.shape, (3, 2)) + self.assertAllClose(output_np, expected_output, rtol=1e-5) - # Check that the output shape is as expected - self.assertEqual(output_np.shape, (2, 3)) + def test_output_depends_on_the_input_values(self): + """Guard against canned outputs: different data must give different results.""" + layer = MovingAverageLayer(periods=[3], drop_na=True, keep_original=False) - # Check that the output contains the expected values - self.assertAllClose(output_np, expected_output, rtol=1e-5) + trend = layer(tf.constant([1.0, 3.0, 5.0, 7.0, 9.0])).numpy() + noise = layer(tf.constant([10.0, 20.0, 45.0, 5.0, 0.0])).numpy() + + self.assertAllClose(trend, np.array([3.0, 5.0, 7.0]), rtol=1e-5) + self.assertAllClose(noise, np.array([25.0, 70.0 / 3.0, 50.0 / 3.0]), rtol=1e-5) + + def test_shorter_series_than_window_yields_no_rows(self): + """With drop_na=True and no full window available the output is empty.""" + layer = MovingAverageLayer(periods=[5], drop_na=True, keep_original=False) + output_np = layer(tf.constant([1.0, 2.0, 3.0])).numpy() + self.assertEqual(output_np.shape, (0,)) + + def test_compute_output_shape_matches_call(self): + """The declared output shape agrees with what call() produces.""" + for periods, drop_na, keep_original in [ + ([3], True, False), + ([3], False, False), + ([2, 3], True, False), + ([3], True, True), + ]: + with self.subTest( + periods=periods, drop_na=drop_na, keep_original=keep_original + ): + layer = MovingAverageLayer( + periods=periods, drop_na=drop_na, keep_original=keep_original + ) + data = tf.constant([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0]) + self.assertEqual( + tuple(layer(data).shape), + tuple(layer.compute_output_shape((8,))), + ) def test_config(self): """Test that the layer can be serialized and deserialized.""" diff --git a/test/layers/test_multi_resolution_tabular_attention_layer.py b/test/layers/test_multi_resolution_tabular_attention_layer.py index 29ad012..9676a45 100644 --- a/test/layers/test_multi_resolution_tabular_attention_layer.py +++ b/test/layers/test_multi_resolution_tabular_attention_layer.py @@ -1,3 +1,4 @@ +import keras import numpy as np import tensorflow as tf @@ -137,8 +138,8 @@ def test_multi_resolution_attention_end_to_end_complex(): output_dim = 1 # Create inputs - numerical_inputs = tf.keras.Input(shape=(num_numerical, numerical_dim)) - categorical_inputs = tf.keras.Input(shape=(num_categorical, categorical_dim)) + numerical_inputs = keras.Input(shape=(num_numerical, numerical_dim)) + categorical_inputs = keras.Input(shape=(num_categorical, categorical_dim)) # Apply multi-resolution attention num_attended, cat_attended = MultiResolutionTabularAttention( @@ -146,13 +147,11 @@ def test_multi_resolution_attention_end_to_end_complex(): )(numerical_inputs, categorical_inputs) # Combine outputs - combined = tf.keras.layers.Concatenate(axis=1)([num_attended, cat_attended]) - outputs = tf.keras.layers.Dense(output_dim)(combined) + combined = keras.layers.Concatenate(axis=1)([num_attended, cat_attended]) + outputs = keras.layers.Dense(output_dim)(combined) # Create model - model = tf.keras.Model( - inputs=[numerical_inputs, categorical_inputs], outputs=outputs - ) + model = keras.Model(inputs=[numerical_inputs, categorical_inputs], outputs=outputs) # Compile the model model.compile(optimizer="adam", loss="mse") @@ -347,20 +346,18 @@ def test_multi_resolution_attention_end_to_end_simple(): num_heads = 2 # Create a simple model - numerical_inputs = tf.keras.Input(shape=(num_numerical, numerical_dim)) - categorical_inputs = tf.keras.Input(shape=(num_categorical, categorical_dim)) + numerical_inputs = keras.Input(shape=(num_numerical, numerical_dim)) + categorical_inputs = keras.Input(shape=(num_categorical, categorical_dim)) attention_layer = MultiResolutionTabularAttention( num_heads=num_heads, d_model=d_model, embedding_dim=embedding_dim ) num_output, cat_output = attention_layer(numerical_inputs, categorical_inputs) - combined = tf.keras.layers.Concatenate(axis=1)([num_output, cat_output]) - outputs = tf.keras.layers.Dense(1)(combined) + combined = keras.layers.Concatenate(axis=1)([num_output, cat_output]) + outputs = keras.layers.Dense(1)(combined) - model = tf.keras.Model( - inputs=[numerical_inputs, categorical_inputs], outputs=outputs - ) + model = keras.Model(inputs=[numerical_inputs, categorical_inputs], outputs=outputs) # Compile model model.compile(optimizer="adam", loss="mse") diff --git a/test/layers/test_preserve_dtype_layer.py b/test/layers/test_preserve_dtype_layer.py index 8df2939..4a5b03c 100644 --- a/test/layers/test_preserve_dtype_layer.py +++ b/test/layers/test_preserve_dtype_layer.py @@ -1,3 +1,4 @@ +import keras import unittest import numpy as np import tensorflow as tf @@ -101,9 +102,9 @@ def test_model_integration(self): layer = PreserveDtypeLayer(target_dtype=tf.float32) # Create a simple model - inputs = tf.keras.Input(shape=(3,), dtype=tf.int32) + inputs = keras.Input(shape=(3,), dtype=tf.int32) outputs = layer(inputs) - model = tf.keras.Model(inputs=inputs, outputs=outputs) + model = keras.Model(inputs=inputs, outputs=outputs) # Test the model test_input = tf.constant([[1, 2, 3], [4, 5, 6]]) diff --git a/test/layers/test_tabular_attention_layer.py b/test/layers/test_tabular_attention_layer.py index 2d76d49..92e2876 100644 --- a/test/layers/test_tabular_attention_layer.py +++ b/test/layers/test_tabular_attention_layer.py @@ -1,3 +1,4 @@ +import keras import tensorflow as tf import numpy as np import pytest @@ -90,10 +91,10 @@ def test_tabular_attention_end_to_end_simple(): num_features = 4 # Create a simple model with TabularAttention - inputs = tf.keras.Input(shape=(num_samples, num_features)) + inputs = keras.Input(shape=(num_samples, num_features)) x = TabularAttention(num_heads=2, d_model=8)(inputs) - outputs = tf.keras.layers.Dense(1)(x) - model = tf.keras.Model(inputs=inputs, outputs=outputs) + outputs = keras.layers.Dense(1)(x) + model = keras.Model(inputs=inputs, outputs=outputs) # Compile the model model.compile(optimizer="adam", loss="mse") @@ -218,14 +219,14 @@ def test_tabular_attention_end_to_end(): num_heads = 2 # Create a simple model - inputs = tf.keras.Input(shape=(num_samples, num_features)) + inputs = keras.Input(shape=(num_samples, num_features)) attention_layer = TabularAttention(num_heads=num_heads, d_model=d_model) x = attention_layer(inputs) - x = tf.keras.layers.GlobalAveragePooling1D()(x) - outputs = tf.keras.layers.Dense(1)(x) + x = keras.layers.GlobalAveragePooling1D()(x) + outputs = keras.layers.Dense(1)(x) - model = tf.keras.Model(inputs=inputs, outputs=outputs) + model = keras.Model(inputs=inputs, outputs=outputs) # Compile model model.compile(optimizer="adam", loss="mse") diff --git a/test/layers/test_variable_selection_layer.py b/test/layers/test_variable_selection_layer.py index 698822e..cbd0458 100644 --- a/test/layers/test_variable_selection_layer.py +++ b/test/layers/test_variable_selection_layer.py @@ -1,3 +1,4 @@ +import keras import os import tempfile @@ -144,9 +145,9 @@ def test_serialization_and_output_consistency(self): def test_model_serialization(self): """Test serialization of a model containing these layers.""" inputs = [ - tf.keras.Input(shape=(10,)), - tf.keras.Input(shape=(20,)), - tf.keras.Input(shape=(30,)), + keras.Input(shape=(10,)), + keras.Input(shape=(20,)), + keras.Input(shape=(30,)), ] vs = VariableSelection(nr_features=3, units=64) @@ -157,7 +158,7 @@ def test_model_serialization(self): x = grn(x) outputs = gl(x) - model = tf.keras.Model(inputs=inputs, outputs=outputs) + model = keras.Model(inputs=inputs, outputs=outputs) # Save and reload the model with tempfile.TemporaryDirectory() as tmp_dir: @@ -165,7 +166,7 @@ def test_model_serialization(self): tmp_dir, "test_model.keras" ) # Added .keras extension model.save(model_path) - loaded_model = tf.keras.models.load_model( + loaded_model = keras.saving.load_model( model_path, custom_objects={ "GatedLinearUnit": GatedLinearUnit, diff --git a/test/test_categorical_hashing.py b/test/test_categorical_hashing.py index b82b686..e17c5f7 100644 --- a/test/test_categorical_hashing.py +++ b/test/test_categorical_hashing.py @@ -1,7 +1,8 @@ +import keras import unittest import numpy as np import tensorflow as tf -from tensorflow.keras.layers import Input +from keras.layers import Input from kdp import ( CategoricalFeature, @@ -166,17 +167,17 @@ def test_functional_pipeline_with_hashing(self): hash_bucket_size = 64 salt = 42 - hashing_layer = tf.keras.layers.Hashing( + hashing_layer = keras.layers.Hashing( num_bins=hash_bucket_size, salt=salt, name="hash_layer" )(input_layer) # Create a category encoding layer - encoding_layer = tf.keras.layers.CategoryEncoding( + encoding_layer = keras.layers.CategoryEncoding( num_tokens=hash_bucket_size, output_mode="multi_hot", name="encoding_layer" )(hashing_layer) # Create a model - model = tf.keras.Model(inputs=input_layer, outputs=encoding_layer) + model = keras.Model(inputs=input_layer, outputs=encoding_layer) # Test with some sample data - use longer, more distinctive strings to avoid collisions user1 = tf.constant("user_with_very_distinctive_name_1", dtype=tf.string) @@ -207,20 +208,20 @@ def test_integration_with_embedding(self): hash_bucket_size = 64 embedding_dim = 8 - hashing_layer = tf.keras.layers.Hashing( + hashing_layer = keras.layers.Hashing( num_bins=hash_bucket_size, name="hash_layer" )(input_layer) # Add embedding layer after hashing - embedding_layer = tf.keras.layers.Embedding( + embedding_layer = keras.layers.Embedding( input_dim=hash_bucket_size, output_dim=embedding_dim, name="embedding_layer" )(hashing_layer) # Flatten the output - flattened_layer = tf.keras.layers.Flatten()(embedding_layer) + flattened_layer = keras.layers.Flatten()(embedding_layer) # Create a model - model = tf.keras.Model(inputs=input_layer, outputs=flattened_layer) + model = keras.Model(inputs=input_layer, outputs=flattened_layer) # Test with some sample data user1 = tf.constant("user_1", dtype=tf.string) @@ -248,17 +249,17 @@ def test_different_salt_values(self): hash_bucket_size = 64 # Model 1 with salt=1 - hashing_layer1 = tf.keras.layers.Hashing( + hashing_layer1 = keras.layers.Hashing( num_bins=hash_bucket_size, salt=1, name="hash_layer1" )(input_layer) # Model 2 with salt=2 - hashing_layer2 = tf.keras.layers.Hashing( + hashing_layer2 = keras.layers.Hashing( num_bins=hash_bucket_size, salt=2, name="hash_layer2" )(input_layer) # Create a model with both outputs - model = tf.keras.Model( + model = keras.Model( inputs=input_layer, outputs=[hashing_layer1, hashing_layer2] ) diff --git a/test/test_custom_pipelines_and_inference.py b/test/test_custom_pipelines_and_inference.py new file mode 100644 index 0000000..e6d96e8 --- /dev/null +++ b/test/test_custom_pipelines_and_inference.py @@ -0,0 +1,209 @@ +"""Tests for custom preprocessing pipelines and inference data conversion. + +Both are documented entry points whose defects only appear on the spellings the +documentation actually recommends. +""" + +import unittest + +import keras +import numpy as np +import pandas as pd +import tensorflow as tf + +from kdp import PreprocessingModel +from kdp.features import Feature, FeatureType, NumericalFeature +from kdp.inference.base import InferenceFormatter + + +def _write_dataset(directory, rows: int = 100): + """Write a small CSV with one numeric and one categorical column.""" + rng = np.random.default_rng(5) + csv_path = directory / "data.csv" + pd.DataFrame( + { + "value": rng.normal(10, 2, rows), + "label": rng.choice(["x", "y"], rows), + } + ).to_csv(csv_path, index=False) + return csv_path + + +class TestCustomPreprocessorSpellings(unittest.TestCase): + """A custom pipeline step may be a name, a class, or a built layer.""" + + def _build(self, tmp_path, preprocessors, **kwargs): + preprocessor = PreprocessingModel( + path_data=str(_write_dataset(tmp_path)), + features_specs={ + "value": Feature( + name="value", + feature_type=FeatureType.FLOAT_NORMALIZED, + preprocessors=preprocessors, + **kwargs, + ), + }, + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + preprocessor.build_preprocessor() + return preprocessor + + def test_layer_names_as_strings(self): + """The docs lead with string layer names, which used to raise.""" + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + preprocessor = self._build( + Path(tmp), ["Rescaling", "Dense"], scale=2.0, units=4 + ) + output = preprocessor.model({"value": tf.constant([[3.0]])}) + self.assertEqual(int(output.shape[-1]), 4) + + def test_layer_classes(self): + """Class objects keep working.""" + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + preprocessor = self._build( + Path(tmp), + [keras.layers.Rescaling, keras.layers.Dense], + scale=2.0, + units=3, + ) + output = preprocessor.model({"value": tf.constant([[3.0]])}) + self.assertEqual(int(output.shape[-1]), 3) + + def test_prebuilt_layer_instances(self): + """An already-built layer is used as-is rather than re-instantiated.""" + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + preprocessor = self._build( + Path(tmp), [keras.layers.Rescaling(scale=3.0, name="triple")] + ) + output = preprocessor.model({"value": tf.constant([[2.0]])}) + np.testing.assert_allclose(np.asarray(output), [[6.0]], rtol=1e-5) + + def test_repeated_layer_type_gets_distinct_names(self): + """The same layer type twice in one pipeline must not collide.""" + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + preprocessor = self._build(Path(tmp), ["Rescaling", "Rescaling"], scale=2.0) + output = preprocessor.model({"value": tf.constant([[1.0]])}) + np.testing.assert_allclose(np.asarray(output), [[4.0]], rtol=1e-5) + + +class TestInferenceTensorConversion(unittest.TestCase): + """Column types are decided from the values that are present.""" + + def setUp(self): + self.formatter = InferenceFormatter.__new__(InferenceFormatter) + + def _convert(self, values): + return self.formatter._convert_to_tensors({"column": values})["column"] + + def test_numeric_column_with_missing_values(self): + """Gaps become NaN and the column stays float.""" + result = self._convert([1.0, 2.0, np.nan]) + self.assertEqual(result.dtype, tf.float32) + self.assertTrue(np.isnan(np.asarray(result)[-1])) + + def test_categorical_column_with_missing_values(self): + """A string column with a gap used to raise "mixed types".""" + result = self._convert(["a", "b", np.nan]) + self.assertEqual(result.dtype, tf.string) + self.assertEqual(np.asarray(result)[-1], b"") + + def test_string_column_stays_string(self): + """Dates and categories are not coerced to float.""" + self.assertEqual(self._convert(["2021-01-01", "2021-01-02"]).dtype, tf.string) + + def test_integer_column_becomes_float(self): + """Numeric columns are normalised to float32.""" + self.assertEqual(self._convert([1, 2, 3]).dtype, tf.float32) + + def test_all_missing_column_is_treated_as_string(self): + """With nothing present there is no numeric evidence.""" + result = self._convert([np.nan, None]) + self.assertEqual(result.dtype, tf.string) + + +class TestTimeSeriesValidationAcceptsTensors(unittest.TestCase): + """The validator must accept what the companion formatter produces.""" + + def test_tensor_values_pass_validation(self): + """Tensors were rejected, so predict() refused the formatter's output.""" + import tempfile + from pathlib import Path + + from kdp import TimeSeriesFeature + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + rows = 60 + csv_path = tmp_path / "series.csv" + pd.DataFrame( + { + "date": pd.date_range("2022-01-01", periods=rows).strftime( + "%Y-%m-%d" + ), + "sales": np.linspace(1, 100, rows), + } + ).to_csv(csv_path, index=False) + + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs={ + "sales": TimeSeriesFeature( + name="sales", sort_by="date", lag_config={"lags": [1]} + ), + }, + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + preprocessor.build_preprocessor() + + # A tensor of several time steps is valid history. The sort column + # has to travel with it, which the validator checks separately. + self.assertTrue( + preprocessor._validate_time_series_inference_data( + { + "sales": tf.constant([[1.0], [2.0], [3.0]]), + "date": tf.constant( + [["2022-01-01"], ["2022-01-02"], ["2022-01-03"]] + ), + } + ) + ) + + # A bare scalar still is not. + with self.assertRaises(ValueError): + preprocessor._validate_time_series_inference_data( + {"sales": 1.0, "date": ["2022-01-01"]} + ) + + +class TestNumericalFeatureEmbeddingLayer(unittest.TestCase): + """get_embedding_layer keeps working without the unused input_shape.""" + + def test_input_shape_is_optional(self): + """It is accepted for compatibility but never read.""" + feature = NumericalFeature(name="value", embedding_dim=4) + with_shape = feature.get_embedding_layer(input_shape=(None, 1)) + without_shape = feature.get_embedding_layer() + self.assertEqual(with_shape.embedding_dim, without_shape.embedding_dim) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_custom_preprocessing.py b/test/test_custom_preprocessing.py index c6fccc9..e76011c 100644 --- a/test/test_custom_preprocessing.py +++ b/test/test_custom_preprocessing.py @@ -1,3 +1,4 @@ +import keras import unittest import numpy as np import tensorflow as tf @@ -16,7 +17,7 @@ # Custom scaling layer for testing -class CustomScalingLayer(tf.keras.layers.Layer): +class CustomScalingLayer(keras.layers.Layer): def __init__(self, scaling_factor=2.0, **kwargs): super().__init__(**kwargs) self.scaling_factor = scaling_factor @@ -33,7 +34,7 @@ def get_config(self): # Custom normalization layer for testing -class CustomNormalizationLayer(tf.keras.layers.Layer): +class CustomNormalizationLayer(keras.layers.Layer): def __init__(self, **kwargs): super().__init__(**kwargs) # Add __name__ attribute for compatibility with PreprocessingModel diff --git a/test/test_dynamic_pipeline.py b/test/test_dynamic_pipeline.py new file mode 100644 index 0000000..d232121 --- /dev/null +++ b/test/test_dynamic_pipeline.py @@ -0,0 +1,169 @@ +"""Tests for DynamicPreprocessingPipeline's dispatch rules. + +Layers are addressed by name: a layer reads the entry that shares its name when +the data provides one, and otherwise consumes the previous layer's output. +""" + +import keras +import unittest + +import numpy as np +import tensorflow as tf + +from kdp.dynamic_pipeline import DynamicPreprocessingPipeline +from kdp.layers_factory import PreprocessorLayerFactory +from kdp.pipeline import FeaturePreprocessor + + +class ScalingLayer(keras.layers.Layer): + """Multiplies its input by a constant factor.""" + + def __init__(self, scaling_factor: float = 2.0, **kwargs): + super().__init__(**kwargs) + self.scaling_factor = scaling_factor + + def call(self, inputs): + """Scale the inputs.""" + return inputs * self.scaling_factor + + def get_config(self): + """Return the layer configuration.""" + config = super().get_config() + config.update({"scaling_factor": self.scaling_factor}) + return config + + +def _flat(tensor) -> list: + """Return a tensor as a flat list of Python floats.""" + return np.asarray(tensor).flatten().tolist() + + +class TestDynamicPipelineDispatch(unittest.TestCase): + """The rule that decides what each layer consumes.""" + + def test_layers_chain_when_only_the_first_key_is_supplied(self): + """A layer with no entry of its own consumes the previous layer's output.""" + pipeline = DynamicPreprocessingPipeline( + [ScalingLayer(2.0, name="scaling"), ScalingLayer(10.0, name="amplified")] + ) + result = pipeline.transform({"scaling": tf.constant([[1.0], [2.0]])}) + + self.assertEqual(_flat(result["scaling"]), [2.0, 4.0]) + # 1.0 -> *2 -> *10, i.e. chained rather than re-reading the raw input. + self.assertEqual(_flat(result["amplified"]), [20.0, 40.0]) + + def test_layers_run_independently_when_each_key_is_supplied(self): + """A layer reads its own entry whenever the data provides one.""" + pipeline = DynamicPreprocessingPipeline( + [ScalingLayer(2.0, name="scaling"), ScalingLayer(10.0, name="amplified")] + ) + result = pipeline.transform( + {"scaling": tf.constant([[1.0]]), "amplified": tf.constant([[3.0]])} + ) + + self.assertEqual(_flat(result["scaling"]), [2.0]) + # 3.0 * 10 -- taken from its own entry, not from the scaling output. + self.assertEqual(_flat(result["amplified"]), [30.0]) + + def test_dispatch_is_deterministic(self): + """Repeated runs on the same inputs always route the same way.""" + pipeline = DynamicPreprocessingPipeline( + [ScalingLayer(2.0, name="scaling"), ScalingLayer(10.0, name="amplified")] + ) + observed = { + tuple( + _flat( + pipeline.transform( + { + "scaling": tf.constant([[1.0]]), + "amplified": tf.constant([[3.0]]), + } + )["scaling"] + ) + ) + for _ in range(25) + } + self.assertEqual(observed, {(2.0,)}) + + def test_input_dictionary_is_not_mutated(self): + """Processing returns a new dictionary and leaves the caller's alone.""" + pipeline = DynamicPreprocessingPipeline([ScalingLayer(2.0, name="scaling")]) + features = {"scaling": tf.constant([[1.0]])} + pipeline.transform(features) + self.assertEqual(list(features), ["scaling"]) + + def test_missing_input_raises_keyerror(self): + """A layer with nothing to read fails loudly instead of being skipped.""" + pipeline = DynamicPreprocessingPipeline([ScalingLayer(name="scaling")]) + with self.assertRaises(KeyError) as ctx: + pipeline.transform({"unrelated": tf.constant([[1.0]])}) + self.assertIn("scaling", str(ctx.exception)) + + def test_duplicate_layer_names_are_rejected(self): + """Names double as data keys, so they have to be unique.""" + with self.assertRaises(ValueError): + DynamicPreprocessingPipeline( + [ScalingLayer(name="dup"), ScalingLayer(name="dup")] + ) + + def test_initialize_and_transform_matches_transform(self): + """The legacy alias behaves exactly like transform().""" + pipeline = DynamicPreprocessingPipeline([ScalingLayer(3.0, name="scaling")]) + features = {"scaling": tf.constant([[2.0]])} + self.assertEqual( + _flat(pipeline.initialize_and_transform(features)["scaling"]), + _flat(pipeline.transform(features)["scaling"]), + ) + + +class TestDynamicPipelineOverDatasets(unittest.TestCase): + """The tf.data entry point.""" + + def test_process_applies_the_same_chaining_rules(self): + """Streaming through tf.data produces the same values as transform().""" + pipeline = DynamicPreprocessingPipeline( + [ScalingLayer(2.0, name="scaling"), ScalingLayer(10.0, name="amplified")] + ) + dataset = tf.data.Dataset.from_tensor_slices( + {"scaling": np.array([[1.0], [2.0]], dtype=np.float32)} + ).batch(2) + + batch = next(iter(pipeline.process(dataset).take(1))) + self.assertEqual(_flat(batch["scaling"]), [2.0, 4.0]) + self.assertEqual(_flat(batch["amplified"]), [20.0, 40.0]) + + +class TestFeaturePreprocessorDynamicMode(unittest.TestCase): + """FeaturePreprocessor(use_dynamic=True) drives the dynamic pipeline.""" + + def _build(self) -> FeaturePreprocessor: + preprocessor = FeaturePreprocessor(name="feature", use_dynamic=True) + preprocessor.add_processing_step( + layer_creator=PreprocessorLayerFactory.cast_to_float32_layer, name="cast" + ) + preprocessor.add_processing_step( + layer_class="Rescaling", scale=3.0, name="rescale" + ) + return preprocessor + + def test_transform_returns_the_final_layer_output(self): + """Dynamic transform chains every step instead of raising.""" + result = self._build().transform(tf.constant([[2]], dtype=tf.int32)) + self.assertEqual(_flat(result), [6.0]) + + def test_chain_builds_a_usable_keras_graph(self): + """Dynamic chain() returns a symbolic tensor usable in a Keras model.""" + preprocessor = self._build() + inputs = keras.Input(shape=(1,), dtype=tf.int32) + model = keras.Model(inputs, preprocessor.chain(inputs)) + self.assertEqual(_flat(model(tf.constant([[2]], dtype=tf.int32))), [6.0]) + + def test_empty_dynamic_pipeline_is_a_passthrough(self): + """With no steps added, the input flows through untouched.""" + preprocessor = FeaturePreprocessor(name="feature", use_dynamic=True) + data = tf.constant([[5.0]]) + self.assertEqual(_flat(preprocessor.transform(data)), [5.0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_inference_paths.py b/test/test_inference_paths.py new file mode 100644 index 0000000..5c642fe --- /dev/null +++ b/test/test_inference_paths.py @@ -0,0 +1,290 @@ +"""Tests for the paths a served model actually takes. + +Building a preprocessor and calling it are separate failure surfaces: several +defects only appeared when the model was invoked, and specifically when it was +invoked the way serving does it -- one row at a time. +""" + +import keras +import unittest + +import keras +import numpy as np +import pandas as pd +import pytest +import tensorflow as tf + +from kdp import PreprocessingModel +from kdp.features import FeatureType, PassthroughFeature +from kdp.layers.date_parsing_layer import DateParsingLayer +from kdp.layers.distribution_aware_encoder_layer import DistributionAwareEncoder + + +def _write_dataset(directory, rows: int = 200): + """Write a CSV covering the feature types these tests exercise.""" + rng = np.random.default_rng(7) + csv_path = directory / "data.csv" + pd.DataFrame( + { + "age": rng.normal(40, 12, rows), + "income": rng.lognormal(10, 1, rows), + "rating": rng.integers(0, 5, rows).astype(float), + "city": rng.choice(["paris", "tokyo", "lima"], rows), + "joined": pd.date_range("2020-01-01", periods=rows).strftime("%Y-%m-%d"), + "record_id": rng.normal(size=rows), + } + ).to_csv(csv_path, index=False) + return csv_path + + +def _single_row_batch() -> dict: + """One row of every feature, i.e. what a served model is handed.""" + return { + "age": tf.constant([[35.0]]), + "income": tf.constant([[50000.0]]), + "rating": tf.constant([[2.0]]), + "city": tf.constant([["paris"]]), + "joined": tf.constant([["2021-06-15"]]), + "record_id": tf.constant([[0.5]]), + } + + +ALL_SPECS = { + "age": FeatureType.FLOAT_NORMALIZED, + "income": FeatureType.FLOAT_RESCALED, + "rating": FeatureType.FLOAT_DISCRETIZED, + "city": FeatureType.STRING_CATEGORICAL, + "joined": FeatureType.DATE, +} + + +class TestDateParsingBatchSizes(unittest.TestCase): + """DateParsingLayer must handle any batch size, one row included.""" + + def test_single_row_batch(self): + """A (1, 1) batch parses instead of collapsing to a scalar.""" + parsed = DateParsingLayer()(tf.constant([["2021-06-15"]])).numpy() + self.assertEqual(parsed.shape, (1, 4)) + np.testing.assert_array_equal(parsed[0][:3], [2021, 6, 15]) + + def test_batch_sizes_agree(self): + """Parsing one row gives the same answer as parsing it in a larger batch.""" + alone = DateParsingLayer()(tf.constant([["2020-02-29"]])).numpy() + together = DateParsingLayer()( + tf.constant([["2020-02-29"], ["1999-12-31"], ["2024-07-04"]]) + ).numpy() + np.testing.assert_array_equal(alone[0], together[0]) + + def test_one_dimensional_input(self): + """A flat vector of dates is accepted too.""" + parsed = DateParsingLayer()(tf.constant(["2021-06-15", "2022-01-02"])).numpy() + self.assertEqual(parsed.shape, (2, 4)) + + +class TestDistributionAwareConstantBatch(unittest.TestCase): + """Distribution detection must survive a batch with no spread.""" + + def test_constant_batch_does_not_raise(self): + """min == max is exactly what a single-row batch looks like.""" + keras.backend.clear_session() + layer = DistributionAwareEncoder() + output = layer(tf.constant([[50000.0]])) + self.assertEqual(output.shape[0], 1) + + def test_large_magnitude_constant_batch(self): + """A fixed epsilon vanishes at large magnitudes; the range must still widen.""" + keras.backend.clear_session() + layer = DistributionAwareEncoder() + output = layer(tf.constant([[1e7], [1e7], [1e7]])) + self.assertTrue(np.all(np.isfinite(np.asarray(output)))) + + +class TestDiscretizedFeature(unittest.TestCase): + """FLOAT_DISCRETIZED must produce a callable model from num_bins alone.""" + + def test_num_bins_only_is_usable( + self, + ): + """The documented `num_bins` configuration builds *and* runs.""" + keras.backend.clear_session() + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + csv_path = _write_dataset(tmp_path) + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs={"rating": FeatureType.FLOAT_DISCRETIZED}, + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + preprocessor.build_preprocessor() + output = np.asarray(preprocessor.model({"rating": tf.constant([[2.0]])})) + + # One-hot over the derived bins: exactly one active entry. + self.assertEqual(output.sum(), 1.0) + + def test_derived_boundaries_are_strictly_increasing(self): + """Boundaries must be usable even for a feature with no variance.""" + boundaries = PreprocessingModel._derive_bin_boundaries( + 5, {"mean": 3.0, "var": 0.0} + ) + self.assertEqual(len(boundaries), 4) + self.assertTrue(all(b < c for b, c in zip(boundaries, boundaries[1:]))) + + def test_single_bin_has_no_interior_boundaries(self): + """A one-bucket request needs no cut points.""" + self.assertEqual( + PreprocessingModel._derive_bin_boundaries(1, {"mean": 0.0, "var": 1.0}), [] + ) + + +@pytest.mark.parametrize( + "config", + [ + {}, + {"use_distribution_aware": True}, + {"tabular_attention": True}, + {"output_mode": "dict"}, + ], + ids=["basic", "distribution_aware", "tabular_attention", "dict_mode"], +) +def test_single_row_prediction(config, tmp_path): + """A model built from a full feature set answers a one-row request.""" + keras.backend.clear_session() + csv_path = _write_dataset(tmp_path) + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs=dict(ALL_SPECS), + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + **config, + ) + preprocessor.build_preprocessor() + + batch = _single_row_batch() + batch.pop("record_id") + output = preprocessor.model(batch) + + if isinstance(output, dict): + for value in output.values(): + assert int(value.shape[0]) == 1 + else: + assert int(output.shape[0]) == 1 + + +class TestPassthroughExcludedFromOutput(unittest.TestCase): + """include_passthrough_in_output=False must still build a valid graph.""" + + def _build(self, tmp_path): + csv_path = _write_dataset(tmp_path) + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs={ + "age": FeatureType.FLOAT_NORMALIZED, + "record_id": PassthroughFeature(name="record_id", dtype=tf.float32), + }, + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + include_passthrough_in_output=False, + ) + preprocessor.build_preprocessor() + return preprocessor + + def test_model_builds_and_passes_values_through_untouched(self): + """A raw Input cannot be an output; the values must still be unchanged.""" + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + preprocessor = self._build(Path(tmp)) + output = preprocessor.model( + {"age": tf.constant([[35.0]]), "record_id": tf.constant([[0.5]])} + ) + + self.assertIn("processed", output) + self.assertIn("passthrough", output) + np.testing.assert_allclose( + np.asarray(output["passthrough"]["record_id"]), [[0.5]] + ) + + +class TestTimeSeriesRowAlignment(unittest.TestCase): + """Time series features must stay concatenable with ordinary ones.""" + + def _build(self, tmp_path, **ts_config): + from kdp import TimeSeriesFeature + + rng = np.random.default_rng(3) + rows = 120 + csv_path = tmp_path / "series.csv" + pd.DataFrame( + { + "date": pd.date_range("2022-01-01", periods=rows).strftime("%Y-%m-%d"), + "sales": np.linspace(100, 300, rows) + rng.normal(0, 5, rows), + "temp": rng.normal(20, 3, rows), + } + ).to_csv(csv_path, index=False) + + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs={ + "sales": TimeSeriesFeature(name="sales", sort_by="date", **ts_config), + "temp": FeatureType.FLOAT_NORMALIZED, + }, + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + preprocessor.build_preprocessor() + return preprocessor + + def test_time_series_feature_concatenates_with_a_numeric_feature(self): + """Dropping warm-up rows left the column shorter than its neighbours.""" + import tempfile + from pathlib import Path + + for config in ( + {"lag_config": {"lags": [1, 7]}}, + {"rolling_stats_config": {"window_size": 7, "statistics": ["mean"]}}, + {"differencing_config": {"order": 1}}, + {"moving_average_config": {"periods": [7]}}, + ): + with self.subTest(config=next(iter(config))): + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + preprocessor = self._build(Path(tmp), **config) + rows = 20 + output = preprocessor.model( + { + "sales": tf.constant( + np.linspace(100, 200, rows).reshape(-1, 1), + dtype=tf.float32, + ), + "temp": tf.constant( + np.full((rows, 1), 20.0), dtype=tf.float32 + ), + } + ) + self.assertEqual(int(output.shape[0]), rows) + + def test_explicit_drop_na_is_overridden_for_models(self): + """An explicit drop_na=True cannot be honoured inside a model.""" + from kdp import TimeSeriesFeature + + feature = TimeSeriesFeature( + name="sales", + sort_by="date", + lag_config={"lags": [1, 3], "drop_na": True}, + ) + model_layers = feature.build_layers() + self.assertFalse(model_layers[0].drop_na) + + # Driving the layers directly still honours the request. + standalone = feature.build_layers(row_preserving=False) + self.assertTrue(standalone[0].drop_na) + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_model_serialization.py b/test/test_model_serialization.py new file mode 100644 index 0000000..f0cad8b --- /dev/null +++ b/test/test_model_serialization.py @@ -0,0 +1,166 @@ +"""Round-trip serialization tests for the preprocessing model. + +These tests guard the "deploy your preprocessing together with your model" +promise: every KDP layer must be registered with the Keras serialization +registry and must rebuild its sub-layers eagerly, otherwise +``PreprocessingModel.load_model`` raises instead of returning a usable model. +""" + +import keras +import tempfile +import unittest +from pathlib import Path + +import numpy as np +import pandas as pd +import pytest +import tensorflow as tf + +from kdp import PreprocessingModel +from kdp.features import FeatureType + +FEATURES_SPECS = { + "age": FeatureType.FLOAT_NORMALIZED, + "income": FeatureType.FLOAT_RESCALED, + "occupation": FeatureType.STRING_CATEGORICAL, + "description": FeatureType.TEXT, + "signup_date": FeatureType.DATE, +} + +SAMPLE_BATCH = { + "age": tf.constant([[0.5], [1.5]]), + "income": tf.constant([[100.0], [200.0]]), + "occupation": tf.constant([["engineer"], ["doctor"]]), + "description": tf.constant([["hello world"], ["lorem ipsum"]]), + "signup_date": tf.constant([["2021-05-04"], ["2022-11-30"]]), +} + +# Each entry is a different preprocessing configuration that must survive a +# save/load round-trip. +ROUND_TRIP_CONFIGS = [ + ("basic", {}), + ("distribution_aware", {"use_distribution_aware": True}), + ("tabular_attention", {"tabular_attention": True}), + ("transformer_blocks", {"transfo_nr_blocks": 1}), + ("feature_selection", {"feature_selection_placement": "all_features"}), + ("numerical_embedding", {"use_advanced_numerical_embedding": True}), + ("global_numerical_embedding", {"use_global_numerical_embedding": True}), + ("feature_moe", {"use_feature_moe": True, "feature_moe_num_experts": 2}), + ("dict_output", {"output_mode": "dict"}), +] + + +def _write_dataset(directory: Path) -> Path: + """Write a small CSV covering every feature type used by these tests.""" + rng = np.random.default_rng(42) + csv_path = directory / "data.csv" + pd.DataFrame( + { + "age": rng.normal(size=120).astype("float32"), + "income": (rng.random(120) * 1000).astype("float32"), + "occupation": rng.choice(["engineer", "doctor", "teacher"], 120), + "description": rng.choice( + ["hello world", "lorem ipsum", "foo bar baz"], 120 + ), + "signup_date": pd.date_range("2020-01-01", periods=120).strftime( + "%Y-%m-%d" + ), + } + ).to_csv(csv_path, index=False) + return csv_path + + +def _assert_same_output(expected, actual) -> None: + """Assert two model outputs match, handling both concat and dict modes.""" + if isinstance(expected, dict): + assert set(expected) == set(actual) + for key in expected: + np.testing.assert_allclose( + np.asarray(expected[key]), np.asarray(actual[key]), atol=1e-5 + ) + else: + np.testing.assert_allclose(np.asarray(expected), np.asarray(actual), atol=1e-5) + + +@pytest.mark.parametrize( + "config_name,config", ROUND_TRIP_CONFIGS, ids=[c[0] for c in ROUND_TRIP_CONFIGS] +) +def test_model_survives_save_load_round_trip(config_name, config, tmp_path): + """A saved preprocessor reloads and produces byte-identical predictions.""" + keras.backend.clear_session() + csv_path = _write_dataset(tmp_path) + + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs=dict(FEATURES_SPECS), + features_stats_path=str(tmp_path / f"stats_{config_name}.json"), + overwrite_stats=True, + **config, + ) + preprocessor.build_preprocessor() + expected = preprocessor.model(SAMPLE_BATCH) + + save_dir = tmp_path / f"model_{config_name}" + preprocessor.save_model(str(save_dir)) + + loaded_model, metadata = PreprocessingModel.load_model(str(save_dir)) + _assert_same_output(expected, loaded_model(SAMPLE_BATCH)) + assert "output_mode" in metadata + + +class TestLayerRegistration(unittest.TestCase): + """Every KDP layer must be discoverable by the Keras deserializer.""" + + def test_all_kdp_layers_are_registered(self): + """Each exported layer resolves through the Keras custom object registry.""" + from keras.saving import get_registered_name, get_registered_object + + import kdp.layers as kdp_layers + + unregistered = [] + for name in kdp_layers.__all__: + obj = getattr(kdp_layers, name) + if not (isinstance(obj, type) and issubclass(obj, keras.layers.Layer)): + continue # e.g. the DistributionType enum + # An unregistered class serializes under its bare name and cannot be + # looked up again; a registered one gets a "package>Name" key. + registered_name = get_registered_name(obj) + if get_registered_object(registered_name) is not obj: + unregistered.append(name) + + self.assertEqual( + unregistered, + [], + f"Layers missing @register_keras_serializable: {unregistered}", + ) + + def test_layer_registration_survives_reserialization(self): + """A registered layer can be serialized and deserialized standalone.""" + from kdp.layers import DateParsingLayer + + layer = DateParsingLayer(date_format="YYYY-MM-DD", name="parse") + restored = keras.layers.deserialize(keras.layers.serialize(layer)) + self.assertIsInstance(restored, DateParsingLayer) + self.assertEqual(restored.date_format, "YYYY-MM-DD") + + +class TestSaveModelErrors(unittest.TestCase): + """Failure modes of save/load should stay explicit.""" + + def test_save_before_build_raises(self): + """Saving an unbuilt preprocessor is an error, not a silent no-op.""" + with tempfile.TemporaryDirectory() as tmp: + csv_path = _write_dataset(Path(tmp)) + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs={"age": FeatureType.FLOAT_NORMALIZED}, + features_stats_path=str(Path(tmp) / "stats.json"), + overwrite_stats=True, + ) + with self.assertRaises(ValueError): + preprocessor.save_model(str(Path(tmp) / "out")) + + def test_load_from_missing_directory_raises(self): + """Loading a non-existent model directory raises a clear error.""" + with self.assertRaises(ValueError): + PreprocessingModel.load_model("/nonexistent/path/to/model") diff --git a/test/test_pipeline.py b/test/test_pipeline.py index fe79cd3..0fd6794 100644 --- a/test/test_pipeline.py +++ b/test/test_pipeline.py @@ -1,3 +1,4 @@ +import keras import unittest from unittest.mock import MagicMock @@ -11,22 +12,22 @@ class TestProcessingStep(unittest.TestCase): def test_processing_step_initialization_and_process(self): """Test that a processing step is initialized correctly and applies processing correctly.""" - input_data = tf.keras.Input(shape=(10,)) - mock_layer_creator = MagicMock(return_value=tf.keras.layers.Dense(units=5)) + input_data = keras.Input(shape=(10,)) + mock_layer_creator = MagicMock(return_value=keras.layers.Dense(units=5)) step = ProcessingStep(layer_creator=mock_layer_creator, units=5) output = step.process(input_data) - self.assertTrue(tf.keras.utils.is_keras_tensor(output)) + self.assertTrue(keras.utils.is_keras_tensor(output)) mock_layer_creator.assert_called_once_with(units=5) def test_connect_step_to_input_layer(self): """Test connecting a processing step to an input layer.""" - input_layer = tf.keras.Input(shape=(10,)) - mock_layer_creator = MagicMock(return_value=tf.keras.layers.Dense(units=5)) + input_layer = keras.Input(shape=(10,)) + mock_layer_creator = MagicMock(return_value=keras.layers.Dense(units=5)) step = ProcessingStep(layer_creator=mock_layer_creator, units=5) output_layer = step.connect(input_layer) - self.assertTrue(tf.keras.utils.is_keras_tensor(output_layer)) + self.assertTrue(keras.utils.is_keras_tensor(output_layer)) class TestPipeline(unittest.TestCase): @@ -53,7 +54,7 @@ def test_chain_steps(self): self.pipeline.add_step(mock_step1) self.pipeline.add_step(mock_step2) - input_layer = tf.keras.Input(shape=(10,)) + input_layer = keras.Input(shape=(10,)) self.pipeline.chain(input_layer) mock_step1.connect.assert_called_once() @@ -66,23 +67,23 @@ class TestFeaturePreprocessor(unittest.TestCase): def test_add_processing_step_and_preprocess(self): """Test adding a processing step and preprocessing input data.""" preprocessor = FeaturePreprocessor(name="test_preprocessor") - mock_layer_creator = MagicMock(return_value=tf.keras.layers.Dense(units=5)) + mock_layer_creator = MagicMock(return_value=keras.layers.Dense(units=5)) preprocessor.add_processing_step(layer_creator=mock_layer_creator, units=5) - input_data = tf.keras.Input(shape=(10,)) + input_data = keras.Input(shape=(10,)) output = preprocessor.chain(input_data) - self.assertTrue(tf.keras.utils.is_keras_tensor(output)) + self.assertTrue(keras.utils.is_keras_tensor(output)) mock_layer_creator.assert_called_once_with(units=5) def test_chain_processing_steps(self): """Test chaining processing steps through the preprocessor.""" preprocessor = FeaturePreprocessor(name="test_chain") - mock_layer_creator = MagicMock(return_value=tf.keras.layers.Dense(units=5)) + mock_layer_creator = MagicMock(return_value=keras.layers.Dense(units=5)) preprocessor.add_processing_step(layer_creator=mock_layer_creator, units=5) - input_layer = tf.keras.Input(shape=(10,)) + input_layer = keras.Input(shape=(10,)) output_layer = preprocessor.chain(input_layer) - self.assertTrue(tf.keras.utils.is_keras_tensor(output_layer)) + self.assertTrue(keras.utils.is_keras_tensor(output_layer)) if __name__ == "__main__": diff --git a/test/test_processor.py b/test/test_processor.py index 7651a3b..0133410 100644 --- a/test/test_processor.py +++ b/test/test_processor.py @@ -1,3 +1,4 @@ +import keras import tempfile import unittest from pathlib import Path @@ -47,7 +48,12 @@ class instances (NumericalFeature, CategoricalFeature, TextFeature, DateFeature) "feat1": FeatureType.FLOAT_NORMALIZED, "feat2": FeatureType.STRING_CATEGORICAL, "feat3": NumericalFeature(name="feat1", feature_type=FeatureType.FLOAT), - "feat4": DateFeature(name="date", feature_type=FeatureType.DATE, date_format="%Y-%m-%d", output_format="year"), + "feat4": DateFeature( + name="date", + feature_type=FeatureType.DATE, + date_format="%Y-%m-%d", + output_format="year", + ), # Other features... } df = generate_fake_data(features_specs, num_rows=100) @@ -389,8 +395,8 @@ def setUpClass(cls): name="feat9", feature_type=FeatureType.FLOAT_NORMALIZED, preprocessors=[ - tf.keras.layers.Rescaling, - tf.keras.layers.Normalization, + keras.layers.Rescaling, + keras.layers.Normalization, ], # leyers required kwargs scale=1, @@ -428,7 +434,7 @@ def test_build_preprocessor_base_features(self): result = ppr.build_preprocessor() # checking if we have model as output - self.assertIsInstance(result["model"], tf.keras.Model) + self.assertIsInstance(result["model"], keras.Model) def test_build_preprocessor_with_crosses(self): """Test building the preprocessor model.""" @@ -812,7 +818,7 @@ def test_preprocessor_with_tabular_attention(self): # Build and verify preprocessor result = ppr.build_preprocessor() - self.assertIsInstance(result["model"], tf.keras.Model) + self.assertIsInstance(result["model"], keras.Model) # Verify TabularAttention layer is present self.assertTrue( @@ -871,7 +877,7 @@ def test_preprocessor_with_multi_resolution_attention(self): # Build and verify preprocessor result = ppr.build_preprocessor() - self.assertIsInstance(result["model"], tf.keras.Model) + self.assertIsInstance(result["model"], keras.Model) # Verify MultiResolutionTabularAttention layer is present self.assertTrue( @@ -926,7 +932,7 @@ def test_preprocessor_attention_with_feature_crosses(self): # Build and verify preprocessor result = ppr.build_preprocessor() - self.assertIsInstance(result["model"], tf.keras.Model) + self.assertIsInstance(result["model"], keras.Model) # Test with a small batch test_data = generate_fake_data(features_specs, num_rows=5) @@ -1840,7 +1846,7 @@ def test_preprocessor_with_passthrough_feature(self): result = ppr.build_preprocessor() # Check if the model was created - self.assertIsInstance(result["model"], tf.keras.Model) + self.assertIsInstance(result["model"], keras.Model) # Check if both features are in the inputs input_names = [input_layer.name for input_layer in result["model"].inputs] diff --git a/test/test_public_api_surface.py b/test/test_public_api_surface.py new file mode 100644 index 0000000..8b0c2cb --- /dev/null +++ b/test/test_public_api_surface.py @@ -0,0 +1,306 @@ +"""Tests for the top-level entry points users are pointed at. + +`auto_configure`, `DatasetStatistics` and `PreprocessingModel.predict` are all +documented as standalone starting points, and each had a defect that only shows +up when they are used that way rather than through the processor's own wiring. +""" + +import unittest + +import keras +import numpy as np +import pandas as pd +import tensorflow as tf + +from kdp import DatasetStatistics, PreprocessingModel, auto_configure +from kdp.features import FeatureType + +SPECS = { + "age": FeatureType.FLOAT_NORMALIZED, + "city": FeatureType.STRING_CATEGORICAL, + "bio": FeatureType.TEXT, + "joined": FeatureType.DATE, +} + + +def _write_dataset(directory, rows: int = 120): + """Write a CSV covering the four feature types used here.""" + rng = np.random.default_rng(11) + csv_path = directory / "data.csv" + pd.DataFrame( + { + "age": rng.normal(40, 12, rows), + "city": rng.choice(["paris", "tokyo", "lima"], rows), + "bio": rng.choice(["hello world", "data science"], rows), + "joined": pd.date_range("2020-01-01", periods=rows).strftime("%Y-%m-%d"), + } + ).to_csv(csv_path, index=False) + return csv_path + + +class TestDatasetStatisticsFromSpecs(unittest.TestCase): + """features_specs alone must be enough to compute statistics.""" + + def test_specs_populate_the_per_type_feature_lists( + self, + ): + """The lists used to stay empty, so no accumulator was ever created.""" + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + stats = DatasetStatistics( + path_data=str(_write_dataset(tmp_path)), + features_specs=dict(SPECS), + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + self.assertEqual(stats.numeric_features, ["age"]) + self.assertEqual(stats.categorical_features, ["city"]) + self.assertEqual(stats.text_features, ["bio"]) + self.assertEqual(stats.date_features, ["joined"]) + + def test_main_returns_statistics_rather_than_an_empty_dict(self): + """A statistics run driven by specs produces real numbers.""" + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + computed = DatasetStatistics( + path_data=str(_write_dataset(tmp_path)), + features_specs=dict(SPECS), + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ).main() + + self.assertIn("numeric_stats", computed) + self.assertIn("categorical_stats", computed) + self.assertIn("age", computed["numeric_stats"]) + self.assertGreater(computed["numeric_stats"]["age"]["count"], 0) + self.assertIn("paris", computed["categorical_stats"]["city"]["vocab"]) + + def test_explicit_lists_still_win(self): + """Passing the lists directly overrides derivation from the specs.""" + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + stats = DatasetStatistics( + path_data=str(_write_dataset(tmp_path)), + features_specs=dict(SPECS), + numeric_features=["age"], + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + self.assertEqual(stats.numeric_features, ["age"]) + self.assertEqual(stats.categorical_features, []) + + +class TestAutoConfigure(unittest.TestCase): + """auto_configure is the headline "let KDP configure itself" entry point.""" + + def test_returns_per_feature_recommendations(self): + """It used to return an empty recommendation set for every dataset.""" + import tempfile + from pathlib import Path + + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + config = auto_configure( + str(_write_dataset(tmp_path)), + features_specs=dict(SPECS), + stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + + self.assertIn("features", config["recommendations"]) + self.assertIn("age", config["recommendations"]["features"]) + self.assertIn( + "FLOAT_NORMALIZED", + config["recommendations"]["features"]["age"]["preprocessing"], + ) + self.assertIn("global_config", config["recommendations"]) + self.assertGreater(len(config["code_snippet"]), 0) + + +class TestPredictAcceptsPlainPython(unittest.TestCase): + """predict() has to take the dictionaries the docs show.""" + + def test_predict_with_numpy_string_columns(self): + """Keras rejects NumPy string arrays, so they must be converted first.""" + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + preprocessor = PreprocessingModel( + path_data=str(_write_dataset(tmp_path)), + features_specs=dict(SPECS), + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ) + preprocessor.build_preprocessor() + + result = preprocessor.predict( + { + "age": np.array([[35.0]], dtype=np.float32), + "city": np.array([["paris"]]), + "bio": np.array([["hello world"]]), + "joined": np.array([["2021-06-15"]]), + } + ) + + self.assertEqual(np.asarray(result).shape[0], 1) + + def test_build_result_is_callable_with_plain_lists(self): + """The returned CallableDict converts plain Python values too.""" + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + result = PreprocessingModel( + path_data=str(_write_dataset(tmp_path)), + features_specs=dict(SPECS), + features_stats_path=str(tmp_path / "stats.json"), + overwrite_stats=True, + ).build_preprocessor() + + output = result( + { + "age": [[35.0]], + "city": [["paris"]], + "bio": [["hello world"]], + "joined": [["2021-06-15"]], + } + ) + + self.assertEqual(int(tf.shape(output)[0]), 1) + + +class TestLegacyStatsFileDetection(unittest.TestCase): + """Statistics written before the bytes fix must not be trusted.""" + + def test_repr_encoded_vocabulary_is_detected(self): + """`b'paris'` is a byte repr, not a category.""" + self.assertTrue( + DatasetStatistics._has_repr_encoded_vocabulary( + {"categorical_stats": {"city": {"vocab": ["b'paris'", "b'lima'"]}}} + ) + ) + + def test_correct_vocabulary_is_left_alone(self): + """A properly decoded vocabulary is not mistaken for the old format.""" + self.assertFalse( + DatasetStatistics._has_repr_encoded_vocabulary( + {"categorical_stats": {"city": {"vocab": ["paris", "lima"]}}} + ) + ) + + def test_single_letter_b_category_is_not_a_false_positive(self): + """A category that merely starts with "b" is still a category.""" + self.assertFalse( + DatasetStatistics._has_repr_encoded_vocabulary( + {"categorical_stats": {"c": {"vocab": ["b", "berlin", "b'"]}}} + ) + ) + + def test_legacy_file_is_recomputed_rather_than_reused(self): + """Reusing it would map every category to the OOV slot.""" + import json + import tempfile + from pathlib import Path + + keras.backend.clear_session() + with tempfile.TemporaryDirectory() as tmp: + tmp_path = Path(tmp) + csv_path = _write_dataset(tmp_path) + stats_path = tmp_path / "features_stats.json" + # Exactly what the previous release wrote. + stats_path.write_text( + json.dumps( + { + "categorical_stats": { + "city": { + "size": 3, + "vocab": ["b'paris'", "b'tokyo'", "b'lima'"], + "dtype": "string", + } + } + } + ) + ) + + preprocessor = PreprocessingModel( + path_data=str(csv_path), + features_specs={"city": FeatureType.STRING_CATEGORICAL}, + features_stats_path=str(stats_path), + ) + preprocessor.build_preprocessor() + + output = np.asarray( + preprocessor.model( + {"city": tf.constant([["paris"], ["tokyo"], ["UNSEEN"]])} + ) + ) + + # Known categories must not collapse onto the out-of-vocabulary vector. + self.assertFalse(np.allclose(output[0], output[2])) + self.assertFalse(np.allclose(output[0], output[1])) + + +class TestStatisticsAreOnlyComputedWhenNeeded(unittest.TestCase): + """A model whose features learn nothing from the data needs no dataset.""" + + def test_hashing_only_model_builds_without_any_data(self): + """Hashing sizes its own buckets, so no statistics pass is required.""" + from kdp.features import CategoricalFeature, CategoryEncodingOptions + + keras.backend.clear_session() + preprocessor = PreprocessingModel( + features_specs={ + "city": CategoricalFeature( + name="city", + feature_type=FeatureType.STRING_CATEGORICAL, + category_encoding=CategoryEncodingOptions.HASHING, + hash_bucket_size=32, + ), + }, + ) + preprocessor.build_preprocessor() + output = preprocessor.model({"city": tf.constant([["paris"], ["lima"]])}) + self.assertEqual(tuple(output.shape), (2, 32)) + + def test_numeric_feature_without_data_raises_a_clear_error(self): + """The old failure was a pathlib TypeError from deep inside the stats pass.""" + keras.backend.clear_session() + with self.assertRaises(ValueError) as ctx: + PreprocessingModel( + features_specs={"age": FeatureType.FLOAT_NORMALIZED}, + ).build_preprocessor() + self.assertIn("path_data", str(ctx.exception)) + + def test_hashing_without_explicit_bucket_size_still_needs_statistics(self): + """Without a bucket count the size is derived from the vocabulary.""" + from kdp.features import CategoricalFeature, CategoryEncodingOptions + + keras.backend.clear_session() + with self.assertRaises(ValueError): + PreprocessingModel( + features_specs={ + "city": CategoricalFeature( + name="city", + feature_type=FeatureType.STRING_CATEGORICAL, + category_encoding=CategoryEncodingOptions.HASHING, + ), + }, + ).build_preprocessor() + + +if __name__ == "__main__": + unittest.main() diff --git a/test/test_time_series_batches.py b/test/test_time_series_batches.py index 10d0852..8e3172a 100644 --- a/test/test_time_series_batches.py +++ b/test/test_time_series_batches.py @@ -1,8 +1,10 @@ +import keras import os import shutil import tempfile import unittest import numpy as np +from pathlib import Path import pandas as pd import tensorflow as tf from tensorflow.test import TestCase # For tf-specific assertions @@ -485,10 +487,10 @@ def test_time_series_training_with_batches(self): dataset = dataset.batch(2) # Small batch size to test batching # Create a simple model - inputs = tf.keras.layers.Input(shape=(2,)) - x = tf.keras.layers.Dense(16, activation="relu")(inputs) - outputs = tf.keras.layers.Dense(1)(x) - model = tf.keras.Model(inputs=inputs, outputs=outputs) + inputs = keras.layers.Input(shape=(2,)) + x = keras.layers.Dense(16, activation="relu")(inputs) + outputs = keras.layers.Dense(1)(x) + model = keras.Model(inputs=inputs, outputs=outputs) # Compile the model model.compile(optimizer="adam", loss="mse", metrics=["mae"]) @@ -656,13 +658,13 @@ def test_large_time_series_dataset_with_ordering(self): prediction_output = preprocessor_model(new_batch) # Verify the prediction output has the expected shape - # With time series features, the number of rows in the output may be reduced - # due to grouping and processing by store_id expected_feature_dim = full_output["sales"].shape[1] self.assertEqual(prediction_output["sales"].shape[1], expected_feature_dim) - # In this particular case, the time series feature layers reduce the data to one row per store - self.assertEqual(prediction_output["sales"].shape[0], num_stores) + # One output row per input row. The layers used to drop their warm-up + # rows, so a caller got back fewer rows than it sent with no way to tell + # which ones survived -- unusable next to other features or to labels. + self.assertEqual(prediction_output["sales"].shape[0], len(new_df)) # Test with new batches containing data for only some stores # This tests that the model handles partial data correctly @@ -909,8 +911,19 @@ def test_time_series_inference_validation(self): "store_id": FeatureType.STRING_CATEGORICAL, } + # KDP reads its statistics from CSV data on disk, so `path_data` takes a + # file path rather than an in-memory frame. + work_dir = Path(tempfile.mkdtemp(prefix="kdp_ts_inference_validation_")) + csv_path = work_dir / "data.csv" + df.to_csv(csv_path, index=False) + # Create and build the preprocessor - preprocessor = PreprocessingModel(features_specs=features, path_data=df) + preprocessor = PreprocessingModel( + features_specs=features, + path_data=str(csv_path), + features_stats_path=str(work_dir / "features_stats.json"), + overwrite_stats=True, + ) preprocessor.build_preprocessor() # Test 1: Single point inference should fail @@ -1016,10 +1029,15 @@ def test_no_validation_without_time_series(self): } # Create preprocessor with our feature stats + work_dir = Path(tempfile.mkdtemp(prefix="kdp_ts_validation_")) + csv_path = work_dir / "data.csv" + df.to_csv(csv_path, index=False) + preprocessor = PreprocessingModel( features_specs=features, - path_data=df, + path_data=str(csv_path), features_stats=feature_stats, + features_stats_path=str(work_dir / "features_stats.json"), overwrite_stats=True, ) diff --git a/test/test_time_series_feature.py b/test/test_time_series_feature.py index 4b96451..c8b5e6b 100644 --- a/test/test_time_series_feature.py +++ b/test/test_time_series_feature.py @@ -143,8 +143,9 @@ def test_output_dim(self): lag_config={"lags": [1, 7], "keep_original": False}, rolling_stats_config={"window_size": 7, "statistics": ["mean", "std"]}, ) - # 2 lags + 2 stats = 4 - self.assertEqual(feature.get_output_dim(), 4) + # The rolling-stats layer keeps its input alongside the two statistics, + # and its input is the 2 lag columns: 2 * (1 + 2) = 6 + self.assertEqual(feature.get_output_dim(), 6) # Test with all configs feature = TimeSeriesFeature( @@ -154,8 +155,10 @@ def test_output_dim(self): differencing_config={"order": 1}, moving_average_config={"periods": [7, 14]}, ) - # Original + 2 lags + 2 stats + 1 diff + 2 MAs = 8 - self.assertEqual(feature.get_output_dim(), 8) + # Each layer keeps its input and appends to it, so the widths compose + # multiplicatively: 1 -> lags (1+2)=3 -> stats 3*(1+2)=9 + # -> differencing 9*2=18 -> moving averages 18*(1+2)=54 + self.assertEqual(feature.get_output_dim(), 54) @parameterized.expand( [ @@ -193,7 +196,8 @@ def test_output_dim(self): "lag_config": {"lags": [1, 7]}, "differencing_config": {"order": 1}, }, - 5, + # 3 lag columns, differenced while keeping the originals: 3 * 2 + 6, ], ] ) @@ -299,6 +303,45 @@ def test_build_layers_with_new_transforms(self): self.assertEqual(layers[2].features, ["month", "day_of_week"]) self.assertTrue(layers[2].cyclic_encoding) + @parameterized.expand( + [ + [{"name": "sales", "lag_config": {"lags": [1, 3]}}], + [ + { + "name": "sales", + "lag_config": {"lags": [1, 3]}, + "differencing_config": {"order": 1}, + } + ], + [ + { + "name": "sales", + "rolling_stats_config": { + "window_size": 3, + "statistics": ["mean", "std"], + }, + } + ], + [{"name": "sales", "moving_average_config": {"periods": [2, 3]}}], + [ + { + "name": "sales", + "lag_config": {"lags": [1, 3]}, + "rolling_stats_config": {"window_size": 3, "statistics": ["mean"]}, + "moving_average_config": {"periods": [2]}, + } + ], + ] + ) + def test_output_dim_matches_actual_layer_output(self, config): + """get_output_dim must agree with the width the layer stack produces.""" + feature = TimeSeriesFeature(**config) + data = tf.constant(np.arange(1, 41, dtype=np.float32).reshape(-1, 1)) + for layer in feature.build_layers(): + data = layer(data) + actual_columns = int(data.shape[-1]) if data.shape.rank > 1 else 1 + self.assertEqual(feature.get_output_dim(), actual_columns) + def test_output_dim_with_new_transforms(self): """Test output dimension calculation with the new transform layers.""" # Test with wavelet transform