diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f5d2cad2..319716324 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -202,6 +202,8 @@ jobs: print('\n'.join(names)) if len(names) != 5: sys.exit(f'expected 5 wheels, built {len(names)}') + if len({n.split('-')[0] for n in names}) != 5: + sys.exit('expected one wheel per distribution') if len(versions) != 1: sys.exit(f'distributions disagree on the version: {sorted(versions)}') pathlib.Path('dist/VERSION').write_text(versions.pop()) @@ -210,12 +212,15 @@ jobs: # only one distribution carries makes the set unsatisfiable, and the # resolver is the only thing that says so. Every one is named by file: # once a version is on PyPI, `--find-links` with `==$VERSION` resolves - # the published release as readily as the wheel built here. + # the published release as readily as the wheel built here. The + # pure-Python wheels are named in full; the engine's tag is the runner's, + # and the check above leaves exactly one. - run: | VERSION=$(cat dist/VERSION) pip install --dry-run \ - "$(ls dist/structured_data_models_client-$VERSION-*.whl)[relational,sqlite,duckdb]" \ - dist/structured_data_models_{api,pql,connectors,relational_engine}-$VERSION-*.whl + "dist/structured_data_models_client-$VERSION-py3-none-any.whl[relational,sqlite,duckdb]" \ + dist/structured_data_models_{api,pql,connectors}-$VERSION-py3-none-any.whl \ + dist/structured_data_models_relational_engine-$VERSION-*.whl # The wheels only: dist/ also carries VERSION for the jobs downstream, # and twine refuses anything that is not a distribution. - run: pip install --upgrade twine && twine check dist/*.whl @@ -256,25 +261,16 @@ jobs: runs-on: ${{ matrix.runner }} steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - # The engine pins its siblings by exact version, which exist only as - # wheels on disk; see the release workflow. + # The engine pins its siblings by exact version. Its `before-test` + # installs these by file before the engine, so the smoke test never + # takes the published release of the same version instead. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: wheels path: _sibling_wheels - # Only Linux runs in a container, where the engine's - # `[tool.cibuildwheel.linux]` already names /project/_sibling_wheels. - - name: Point the wheel test environments at the sibling wheels + - name: Build only this row's architecture shell: bash - run: | - echo "${{ matrix.archs }}" >> "$GITHUB_ENV" - case "${{ runner.os }}" in - Linux) ;; - # cibuildwheel layers its own `environment` over the host's, so a - # plain variable reaches the build and test environments without - # displacing what the engine's pyproject sets per platform. - *) echo "PIP_FIND_LINKS=${{ github.workspace }}/_sibling_wheels" >> "$GITHUB_ENV" ;; - esac + run: echo "${{ matrix.archs }}" >> "$GITHUB_ENV" - uses: pypa/cibuildwheel@b3395e271a35f73dad77f21802abe9b986dfe299 # v4.3.1 with: @@ -332,23 +328,37 @@ jobs: pip install --upgrade pip pytest requests-mock VERSION=$(cat dist/VERSION) pip install \ - "$(ls dist/structured_data_models_client-$VERSION-*.whl)[sqlite,duckdb]" \ - dist/structured_data_models_{api,pql,connectors}-$VERSION-*.whl + "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb]" \ + dist/structured_data_models_{api,pql,connectors}-$VERSION-py3-none-any.whl - name: The installed distribution is what gets imported, and it is typed working-directory: /tmp run: | python - <<'EOF' - import json, pathlib + import json, os, pathlib from importlib.metadata import distribution + from urllib.parse import urlparse + from urllib.request import url2pathname import predictive_query, sdm_api, sdm_client, sdm_connectors # The same version on PyPI resolves as readily as the wheel built # here, and then this job tests the published release instead. - for name in ('client', 'api', 'pql', 'connectors'): - recorded = distribution(f'structured-data-models-{name}').read_text('direct_url.json') - source = json.loads(recorded or '{}').get('url', 'an index') - assert '/dist/' in source, ( - f'structured-data-models-{name} was installed from {source}, ' + built = pathlib.Path(os.environ['GITHUB_WORKSPACE'], 'dist').resolve() + for name in ( + 'structured-data-models-client', + 'structured-data-models-api', + 'structured-data-models-pql', + 'structured-data-models-connectors', + ): + recorded = distribution(name).read_text('direct_url.json') + url = json.loads(recorded or '{}').get('url', '') + parsed = urlparse(url) + source = ( + pathlib.Path(url2pathname(parsed.path)).resolve().parent + if parsed.scheme == 'file' + else None + ) + assert source == built, ( + f'{name} was installed from {url or "an index"}, ' f'not from the wheel built here' ) @@ -389,8 +399,8 @@ jobs: pip install --upgrade pip cyclonedx-bom VERSION=$(cat dist/VERSION) pip install \ - "$(ls dist/structured_data_models_client-$VERSION-*.whl)[sqlite,duckdb,postgres,databricks,snowflake,s3]" \ - dist/structured_data_models_{api,pql,connectors}-$VERSION-*.whl + "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb,postgres,databricks,snowflake,s3]" \ + dist/structured_data_models_{api,pql,connectors}-$VERSION-py3-none-any.whl cyclonedx-py environment --of JSON --validate -o dist/sbom.cyclonedx.json python -c "import json,sys; d=json.load(open('dist/sbom.cyclonedx.json')); names={c['name'] for c in d.get('components', [])}; missing={'structured-data-models-client','structured-data-models-api','structured-data-models-pql','structured-data-models-connectors'} - names; print(f'{len(names)} components'); sys.exit(f'SBOM missing {sorted(missing)}' if missing else 0)" - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 22d6a5606..4c59999f2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -165,27 +165,16 @@ jobs: # The engine pins its siblings by exact version, and at this point in a # release those versions exist only as the artifact `build` uploaded. - # cibuildwheel's test environment resolves from PyPI unless told - # otherwise, so without this the smoke test cannot find them. + # The engine's `before-test` installs them by file before the engine, so + # the smoke test tests these and not a published release. - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 with: name: pure-distributions path: _sibling_wheels - # Only Linux runs in a container, where the engine's - # `[tool.cibuildwheel.linux]` already names /project/_sibling_wheels. - - name: Point the wheel test environments at the sibling wheels + - name: Build only this row's architecture shell: bash - run: | - echo "${{ matrix.archs }}" >> "$GITHUB_ENV" - case "${{ runner.os }}" in - Linux) ;; - # cibuildwheel layers its own `environment` over the host's, so a - # plain variable reaches the build and test environments without - # displacing what the engine's pyproject sets per platform. - *) echo "PIP_FIND_LINKS=${{ github.workspace }}/_sibling_wheels" >> "$GITHUB_ENV" ;; - esac - + run: echo "${{ matrix.archs }}" >> "$GITHUB_ENV" - uses: pypa/cibuildwheel@b3395e271a35f73dad77f21802abe9b986dfe299 # v4.3.1 with: @@ -359,12 +348,17 @@ jobs: - name: Software bill of materials run: | python -m pip install --upgrade cyclonedx-bom - # --find-links dist for the same reason as the engine's test env: the - # exact sibling pins resolve to the wheels just built, not to whatever - # happens to be on PyPI. Without it this describes a previous release, - # or fails outright before the first one. - python -m pip install --find-links dist \ - "./packages/structured-data-models-client[relational,sqlite,duckdb,postgres,databricks,snowflake,s3]" + # Every distribution from dist/, never by name and version alone: + # after a partial upload, PyPI's release of this version resolves as + # readily as the wheels being released, and this would describe it. + # The engine has a wheel per platform, so pip picks this runner's + # from dist/ and nowhere else. + version="${GITHUB_REF_NAME#v}" + python -m pip install --no-index --no-deps --find-links dist \ + "structured-data-models-relational-engine==$version" + python -m pip install \ + "dist/structured_data_models_client-$version-py3-none-any.whl[relational,sqlite,duckdb,postgres,databricks,snowflake,s3]" \ + dist/structured_data_models_{api,pql,connectors}-$version-py3-none-any.whl cyclonedx-py environment --of JSON --validate -o dist/sbom.cyclonedx.json python -c "import json,sys; d=json.load(open('dist/sbom.cyclonedx.json')); names={c['name'] for c in d.get('components', [])}; missing={'structured-data-models-client','structured-data-models-api','structured-data-models-pql','structured-data-models-connectors','structured-data-models-relational-engine'} - names; print(f'{len(names)} components'); sys.exit(f'SBOM missing {sorted(missing)}' if missing else 0)" diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index ef8865635..a7f0478cd 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -193,17 +193,25 @@ build: print('\n'.join(names)) if len(names) != 5: sys.exit(f'expected 5 wheels, built {len(names)}') + if len({n.split('-')[0] for n in names}) != 5: + sys.exit('expected one wheel per distribution') if len(versions) != 1: sys.exit(f'distributions disagree on the version: {sorted(versions)}') pathlib.Path('dist/VERSION').write_text(versions.pop()) EOF # Building each one proves nothing about installing them together. A pin # that only one distribution carries makes the set unsatisfiable, and the - # resolver is the only thing that says so. + # resolver is the only thing that says so. Every one is named by file: + # once a version is on PyPI, `--find-links` with `==$VERSION` resolves the + # published release as readily as the wheel built here. - | VERSION=$(cat dist/VERSION) - pip install --dry-run --find-links "$CI_PROJECT_DIR/dist" \ - "structured-data-models-client[relational,sqlite,duckdb]==$VERSION" + pip install --dry-run \ + "dist/structured_data_models_client-$VERSION-py3-none-any.whl[relational,sqlite,duckdb]" \ + "dist/structured_data_models_api-$VERSION-py3-none-any.whl" \ + "dist/structured_data_models_pql-$VERSION-py3-none-any.whl" \ + "dist/structured_data_models_connectors-$VERSION-py3-none-any.whl" \ + dist/structured_data_models_relational_engine-"$VERSION"-*.whl artifacts: paths: - dist/ @@ -245,8 +253,7 @@ wheels: variables: DOCKER_HOST: tcp://docker:2375 DOCKER_TLS_CERTDIR: "" - # Into the manylinux container too. cibuildwheel prepends passed variables - # to the `environment` pyproject sets, so PIP_FIND_LINKS still applies. + # Into the manylinux container too, where the smoke test installs. CIBW_ENVIRONMENT_PASS_LINUX: PIP_INDEX_URL parallel: matrix: @@ -272,7 +279,7 @@ wheels: fi - pip install -q build 'cibuildwheel==4.3.0' # The engine's smoke test installs it against these, which exist nowhere - # but here until a release; `[tool.cibuildwheel.linux]` points pip at them. + # but here until a release; its `before-test` installs them by file. - | for package in structured-data-models-api structured-data-models-pql structured-data-models-client structured-data-models-connectors; do ( cd "packages/$package" && python -m build --wheel --outdir "$CI_PROJECT_DIR/_sibling_wheels" . ) @@ -336,7 +343,13 @@ installed-wheel: script: - pip install -q pytest requests-mock pytest-cov - VERSION=$(cat dist/VERSION) - - pip install -q --find-links "$CI_PROJECT_DIR/dist" "structured-data-models-client[sqlite,duckdb]==$VERSION" + # By file: the same version on PyPI resolves as readily as the wheel built + # here, and then this job tests the published release instead. + - pip install -q + "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb]" + "dist/structured_data_models_api-$VERSION-py3-none-any.whl" + "dist/structured_data_models_pql-$VERSION-py3-none-any.whl" + "dist/structured_data_models_connectors-$VERSION-py3-none-any.whl" # Prove the installed distribution, not the checkout, is what is imported, # and that the typed marker survived the build. Written as one script that # raises, rather than a chain ending in `|| true`: a check that cannot fail @@ -344,9 +357,32 @@ installed-wheel: - cd /tmp - | python - <<'EOF' - import pathlib + import json, os, pathlib + from importlib.metadata import distribution + from urllib.parse import urlparse + from urllib.request import url2pathname import predictive_query, sdm_api, sdm_client, sdm_connectors + built = pathlib.Path(os.environ['CI_PROJECT_DIR'], 'dist').resolve() + for name in ( + 'structured-data-models-client', + 'structured-data-models-api', + 'structured-data-models-pql', + 'structured-data-models-connectors', + ): + recorded = distribution(name).read_text('direct_url.json') + url = json.loads(recorded or '{}').get('url', '') + parsed = urlparse(url) + source = ( + pathlib.Path(url2pathname(parsed.path)).resolve().parent + if parsed.scheme == 'file' + else None + ) + assert source == built, ( + f'{name} was installed from {url or "an index"}, ' + f'not from the wheel built here' + ) + for module in (sdm_api, sdm_client, sdm_connectors, predictive_query): location = pathlib.Path(module.__file__).resolve() assert 'site-packages' in location.parts, ( @@ -412,8 +448,11 @@ sbom: script: - pip install -q cyclonedx-bom - VERSION=$(cat dist/VERSION) - - pip install -q --find-links "$CI_PROJECT_DIR/dist" - "structured-data-models-client[sqlite,duckdb,postgres,databricks,snowflake,s3]==$VERSION" + - pip install -q + "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb,postgres,databricks,snowflake,s3]" + "dist/structured_data_models_api-$VERSION-py3-none-any.whl" + "dist/structured_data_models_pql-$VERSION-py3-none-any.whl" + "dist/structured_data_models_connectors-$VERSION-py3-none-any.whl" - cyclonedx-py environment --of JSON --validate -o "$CI_PROJECT_DIR/dist/sbom.cyclonedx.json" - python -c "import json,sys; d=json.load(open('dist/sbom.cyclonedx.json')); names={c['name'] for c in d.get('components', [])}; diff --git a/CHANGELOG.md b/CHANGELOG.md index 16e6f6926..b59dce298 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -36,13 +36,6 @@ these packages. All five distributions share one version and one entry. value that cannot be read as its type, such as a `run_mode` that is not text, is refused with `INVALID_REQUEST` before sending, where a NIM answered `422 VALIDATION_FAILED`. -- The `duckdb` extra holds `duckdb` below 2. The 2.0 release warns on the - Arrow fetch the relational engine reads DuckDB tables through, and this - release is tested against 1.5. -- `structured-data-models-client` and the relational engine hold `pandas` - below 3.1. The 3.1 release raises taking a date offset from the earliest - timestamp, which the engine does for every graph without a time column, - and this release is tested against 3.0. ### Fixed @@ -50,6 +43,15 @@ these packages. All five distributions share one version and one entry. request carries the query in `metadata.source_query` for logging, which the contract caps at 1,024 characters, so a NIM refused a longer one with `VALIDATION_FAILED`. The logged copy is now cut to fit. +- DuckDB 2.0 no longer warns on every read. The relational engine and + `read_table` read a DuckDB result through a relation rather than the DB-API + cursor, which 2.0 warns makes an extra copy of the rows. +- A relational prediction over a graph without a time column works on pandas + 3.1. Its anchor is the earliest timestamp pandas holds, and 3.1 refused to + take even an empty date offset from it. +- A request's description leaves out a wire table carrying an integer past + the float range on every pandas release. pandas 3.1 had started to build + a frame around one, where earlier releases refused it. ## 1.0.0 - 2026-10-15 diff --git a/packages/structured-data-models-client/pyproject.toml b/packages/structured-data-models-client/pyproject.toml index 42c700625..9dfbb274b 100644 --- a/packages/structured-data-models-client/pyproject.toml +++ b/packages/structured-data-models-client/pyproject.toml @@ -41,10 +41,7 @@ classifiers = [ dependencies = [ "structured-data-models-api==1.0.0", "structured-data-models-pql==1.0.0", - # 3.1 builds a frame around an integer past the float range where earlier - # releases refused one, and a request's description relies on that - # refusal to leave out the table carrying it. - "pandas>=2.2.2,<3.1", + "pandas>=2.2.2", "numpy>=1.23", "requests>=2.28", "urllib3>=1.26", diff --git a/packages/structured-data-models-client/src/sdm_client/request.py b/packages/structured-data-models-client/src/sdm_client/request.py index a67618bd4..148a7cc7d 100644 --- a/packages/structured-data-models-client/src/sdm_client/request.py +++ b/packages/structured-data-models-client/src/sdm_client/request.py @@ -32,6 +32,7 @@ import json import math import re +import sys import warnings from collections.abc import Mapping, Sequence from dataclasses import dataclass, field @@ -877,23 +878,39 @@ def _entity_table_in(payloads: Sequence[Mapping[str, Any]]) -> str: def _wire_frame(table: Any) -> pd.DataFrame | None: - r"""One wire table as a frame, or nothing if it cannot be read as one.""" + r"""One wire table as a frame, or nothing if it cannot be read as one. + + An integer past the float range is refused here rather than left to + pandas, whose releases disagree on whether a frame can hold one. + """ if not isinstance(table, Mapping): return None columns = table.get('columns') rows = table.get('rows') if not isinstance(columns, list) or not isinstance(rows, list): return None + if any( + _past_the_float_range(value) + for row in rows + if isinstance(row, (list, tuple)) + for value in row + ): + return None try: return pd.DataFrame(rows, columns=columns) except (TypeError, ValueError, OverflowError): - # A row wider than its header raises rather than padding, and a - # number too large for pandas to hold raises as well. Reading a + # A row wider than its header raises rather than padding. Reading a # request is a description of it, and a description that cannot be # made is not worth failing the call for. return None +def _past_the_float_range(value: Any) -> bool: + return isinstance(value, int) and not ( + -sys.float_info.max <= value <= sys.float_info.max + ) + + #: What ties a related row to the instance it hangs off. INSTANCE_ID = 'instance_id' diff --git a/packages/structured-data-models-client/tests/test_both_routes_describe_themselves_alike.py b/packages/structured-data-models-client/tests/test_both_routes_describe_themselves_alike.py index dca72b00f..e536a6436 100644 --- a/packages/structured-data-models-client/tests/test_both_routes_describe_themselves_alike.py +++ b/packages/structured-data-models-client/tests/test_both_routes_describe_themselves_alike.py @@ -619,15 +619,17 @@ def test_an_infinite_id_is_not_something_to_move_past(): ] -def test_a_number_too_large_to_hold_is_not_a_description(): - """Pandas raises building a frame around it. A description that cannot be - made is not worth failing the call for, so the envelope contributes - nothing and the readable one beside it is unaffected.""" +@pytest.mark.parametrize('rows', [[[10**400]], [['x'], [-(10**400)]]]) +def test_a_number_too_large_to_hold_is_not_a_description(rows): + """No float holds it, and pandas releases disagree on whether a frame + can. A description that cannot be made is not worth failing the call + for, so the envelope contributes nothing and the readable one beside it + is unaffected.""" huge = { 'context': { 'instance_table': { 'columns': ['instance_id'], - 'rows': [[10**400]], + 'rows': rows, } } } diff --git a/packages/structured-data-models-connectors/pyproject.toml b/packages/structured-data-models-connectors/pyproject.toml index 6bce18c6a..b96d010d4 100644 --- a/packages/structured-data-models-connectors/pyproject.toml +++ b/packages/structured-data-models-connectors/pyproject.toml @@ -61,10 +61,7 @@ sqlite = [ "adbc-driver-sqlite", ] duckdb = [ - # 2.0 warns on an Arrow fetch from a DB-API cursor, which the engine - # makes for every table and sample it reads. Lifting this moves those - # reads off the cursor. - "duckdb>=1.5.0,<2", + "duckdb>=1.5.0", ] snowflake = [ # 5.0 empties `DEFAULT_CONFIGURATION` and rewrites keyword names through diff --git a/packages/structured-data-models-connectors/src/sdm_connectors/reader.py b/packages/structured-data-models-connectors/src/sdm_connectors/reader.py index d91c547b6..a001a09e3 100644 --- a/packages/structured-data-models-connectors/src/sdm_connectors/reader.py +++ b/packages/structured-data-models-connectors/src/sdm_connectors/reader.py @@ -176,6 +176,8 @@ def read_table( ): cursor, owned = _query_handle(connection) try: + if _is_duckdb(cursor): + return _read_duckdb(cursor, sql) with driver_guard( 'QUERY_FAILED', 'query execution failed', @@ -218,6 +220,37 @@ def read_table( cursor.close() +def _is_duckdb(handle: Any) -> bool: + r"""Whether ``handle`` is a DuckDB connection, without importing DuckDB.""" + duckdb = sys.modules.get('duckdb') + return duckdb is not None and isinstance(handle, duckdb.DuckDBPyConnection) + + +def _read_duckdb(connection: Any, sql: str) -> pd.DataFrame: + r"""Reads a DuckDB result through a relation, not the DB-API cursor. + + DuckDB 2.0 warns that Arrow from a DB-API result is an extra copy. A + statement that returns no result reads as an empty frame. + """ + with driver_guard( + 'QUERY_FAILED', + 'query execution failed', + sql=sql, + type_error_is_the_caller_s=False, + ): + relation = connection.sql(sql) + arrow_table = None if relation is None else relation.to_arrow_table() + if arrow_table is None: + return pd.DataFrame() + with driver_guard( + 'QUERY_FAILED', + 'reading the result failed', + sql=sql, + type_error_is_the_caller_s=False, + ): + return _arrow_to_pandas(arrow_table) + + def _own_transaction(connection: Any) -> AbstractContextManager[Any]: r"""The transaction one read runs in, which only psycopg needs.""" # Only if already imported, so reading through another driver imports none. diff --git a/packages/structured-data-models-connectors/tests/test_reader.py b/packages/structured-data-models-connectors/tests/test_reader.py index db892a4b5..b0a58f701 100644 --- a/packages/structured-data-models-connectors/tests/test_reader.py +++ b/packages/structured-data-models-connectors/tests/test_reader.py @@ -5,6 +5,7 @@ from __future__ import annotations import sqlite3 +import warnings from pathlib import PurePosixPath import pandas as pd @@ -179,6 +180,23 @@ def test_read_table_over_duckdb_by_table(tmp_path): connection.close() +def test_read_table_over_duckdb_does_not_warn(): + """DuckDB 2.0 warns on Arrow from a DB-API result, so the read must not + take one.""" + import duckdb + + connection = duckdb.connect() + try: + connection.execute( + 'CREATE TABLE items AS SELECT * FROM range(5) AS t(a)' + ) + with warnings.catch_warnings(): + warnings.simplefilter('error') + assert len(read_table(connection, table='items')) == 5 + finally: + connection.close() + + def test_an_integer_column_has_one_dtype_however_it_is_fetched(): """Plain rows (sqlite3) and Arrow (DuckDB) give the same dtypes: `int64` without a null, `Int64` with one.""" diff --git a/packages/structured-data-models-relational-engine/pyproject.toml b/packages/structured-data-models-relational-engine/pyproject.toml index cbd7e1c31..3ce23d8bf 100644 --- a/packages/structured-data-models-relational-engine/pyproject.toml +++ b/packages/structured-data-models-relational-engine/pyproject.toml @@ -118,15 +118,18 @@ editable.verbose=false # workflow all drive cibuildwheel from here, so the wheel set CI proves is the # one a tag uploads. [tool.cibuildwheel] -# The default `build` frontend, not `build[uv]`: the smoke test below installs -# this wheel against sibling wheels that exist only on disk, found through -# PIP_FIND_LINKS. uv reads no PIP_* variable, and the uv the manylinux images -# shipped had no environment form of --find-links at all, so under `build[uv]` -# the smoke test could never resolve structured-data-models-api. +# The default `build` frontend, not `build[uv]`: `before-test` installs with +# the test environment's own pip, and an environment uv creates has none. build = "cp310-* cp311-* cp312-* cp313-* cp314-*" # musllinux is not a libc this project has ever built or tested against, and # leaving it on would silently double the Linux wheel count. skip = "*-musllinux_*" +# The engine pins its siblings by exact version, and before a release they +# exist only as wheels on disk. Every workflow puts them in `_sibling_wheels/` +# at the root cibuildwheel runs from, and they are installed by file first, so +# the engine's pins are met by them and never by a published release of the +# same version. +before-test = "python -c \"import glob, subprocess, sys; subprocess.check_call([sys.executable, '-m', 'pip', 'install', *glob.glob(sys.argv[1] + '/*.whl')])\" \"{project}/_sibling_wheels\"" # Proof per wheel, on the machine that built it: the compiled sampler loads and # the public graph surface imports. test-command = "python -c \"from sdm_relational_engine.rfm import Graph, KumoRelational; import sdm_relational_engine.relationallib as relationallib; assert hasattr(relationallib.NeighborSampler, 'seed')\"" @@ -138,14 +141,6 @@ test-command = "python -c \"from sdm_relational_engine.rfm import Graph, KumoRel archs = ["x86_64", "aarch64"] manylinux-x86_64-image = "manylinux_2_28" manylinux-aarch64-image = "manylinux_2_28" -# The engine pins its siblings by exact version, and before a release they exist -# only as wheels on disk. cibuildwheel copies the directory it is run from into -# the container as /project, and every workflow runs it from the repository -# root, so this is `_sibling_wheels/` at that root. -# Only Linux builds in a container, so only Linux has a path that is the same -# on every machine; macOS and Windows are given the checkout's absolute path by -# their workflows instead. -environment = { PIP_FIND_LINKS = "/project/_sibling_wheels" } [tool.cibuildwheel.macos] # The workflows narrow this to one arch per runner, so every wheel is built and diff --git a/packages/structured-data-models-relational-engine/setup.py b/packages/structured-data-models-relational-engine/setup.py index 65c1aa3a5..0f289c9a9 100644 --- a/packages/structured-data-models-relational-engine/setup.py +++ b/packages/structured-data-models-relational-engine/setup.py @@ -39,9 +39,7 @@ setup( install_requires=[ 'numpy', - # 3.1 raises taking a `DateOffset` from `Timestamp.min`, which is the - # anchor time of every graph without a time column. - 'pandas<3.1', + 'pandas', # The warehouse backends read results as `pd.ArrowDtype`, which pandas # refuses below pyarrow 10.0.1. Databricks Runtime 14.3 LTS and # serverless environment 1 ship pyarrow 8, and failed there on import. diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/sampler.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/sampler.py index ee2ac7a44..515fa64c1 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/sampler.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/sampler.py @@ -164,8 +164,7 @@ def _by_pkey( with self._connection.cursor() as cursor: cursor.register(tmp_name, tmp) - cursor.execute(sql) - table = cursor.to_arrow_table() + table = cursor.sql(sql).to_arrow_table() batch = table['__kumo_batch__'].to_numpy() self._refuse_an_id_scored_twice( @@ -233,8 +232,7 @@ def _by_fkey( with self._connection.cursor() as cursor: cursor.register(tmp_name, tmp) - cursor.execute(sql) - table = cursor.to_arrow_table() + table = cursor.sql(sql).to_arrow_table() batch = table['__kumo_batch__'].to_numpy() batch_index = table.schema.get_field_index('__kumo_batch__') @@ -288,8 +286,7 @@ def _by_time( with self._connection.cursor() as cursor: cursor.register(tmp_name, tmp) - cursor.execute(sql) - table = cursor.to_arrow_table() + table = cursor.sql(sql).to_arrow_table() batch = table['__kumo_batch__'].to_numpy() batch_index = table.schema.get_field_index('__kumo_batch__') @@ -412,8 +409,7 @@ def _sample_entity_table( ) with self._connection.cursor() as cursor: - cursor.execute(sql, parameters) - table = cursor.to_arrow_table() + table = cursor.sql(sql, params=parameters).to_arrow_table() return Table._sanitize( df=table.to_pandas(types_mapper=pd.ArrowDtype), diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/table.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/table.py index fbfa8fbb9..7ab427ee7 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/table.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/backend/duckdb/table.py @@ -166,8 +166,7 @@ def _get_source_sample_df(self) -> pd.DataFrame: f'FROM {self._quoted_source_name} ' f'LIMIT {self._NUM_SAMPLE_ROWS}' ) - cursor.execute(sql) - table = cursor.to_arrow_table() + table = cursor.sql(sql).to_arrow_table() if len(table) == 0: raise RuntimeError(f"Table '{self.source_name}' is empty") @@ -219,8 +218,7 @@ def _get_expr_sample_df( f'FROM {self._quoted_source_name} ' f'LIMIT {self._NUM_SAMPLE_ROWS}' ) - cursor.execute(sql) - table = cursor.to_arrow_table() + table = cursor.sql(sql).to_arrow_table() if len(table) == 0: raise RuntimeError(f"Table '{self.source_name}' is empty") diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/sampler.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/sampler.py index 5f4e2f2ee..7c00fa805 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/sampler.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/sampler.py @@ -25,6 +25,7 @@ from sdm_relational_engine.rfm.base.utils import ( TimeBound, date_offset_to_seconds, + shift_time, to_naive_utc, ) from sdm_relational_engine.rfm.diagnostics import GraphSanitizationReport @@ -1205,7 +1206,7 @@ def _window_fits_after( ends.append(window.end_date_offset) if not ends: return False - return anchor_time + max_date_offset(*ends) <= max_time + return shift_time(anchor_time, max_date_offset(*ends)) <= max_time def _entity_filter_condition(query: ValidatedPredictiveQuery) -> str | None: diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/utils.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/utils.py index 9ac5695ef..3be1d0954 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/utils.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/base/utils.py @@ -102,6 +102,47 @@ def to_datetime(ser: pd.Series, column_name: str | None = None) -> pd.Series: return ser +_RELATIVE_FIELDS = frozenset( + { + 'years', + 'months', + 'weeks', + 'days', + 'hours', + 'minutes', + 'seconds', + 'milliseconds', + 'microseconds', + 'nanoseconds', + } +) + + +def shift_time(time: pd.Timestamp, offset: pd.DateOffset) -> pd.Timestamp: + r"""``time`` moved by ``offset``, unchanged by an offset that moves nothing. + + pandas 3.1 takes even an empty offset through microseconds, which puts + :obj:`pandas.Timestamp.min`, the anchor of a graph without a time column, + out of range. + """ + return time if _moves_nothing(offset) else time + offset + + +def _moves_nothing(offset: pd.DateOffset) -> bool: + r"""Whether ``offset`` is a plain relative offset of no length. + + Any other offset is applied, since one that replaces a field, rolls to an + anchor or normalizes can move a time even with a zero multiple. + """ + if type(offset) is not pd.DateOffset or offset.normalize: + return False + if not set(offset.kwds) <= _RELATIVE_FIELDS: + return False + return offset.n == 0 or ( + bool(offset.kwds) and not any(offset.kwds.values()) + ) + + _MIN_DAYS_IN_MONTH = 28 _MAX_DAYS_IN_MONTH = 31 _MIN_DAYS_IN_YEAR = 365 diff --git a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/rfm.py b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/rfm.py index c870816e4..edaa49d61 100644 --- a/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/rfm.py +++ b/packages/structured-data-models-relational-engine/src/sdm_relational_engine/rfm/rfm.py @@ -49,7 +49,11 @@ ) from sdm_relational_engine.rfm import Graph, TaskTable from sdm_relational_engine.rfm.base import DataBackend, Sampler, composite_key -from sdm_relational_engine.rfm.base.utils import Timestamp, to_naive_utc +from sdm_relational_engine.rfm.base.utils import ( + Timestamp, + shift_time, + to_naive_utc, +) from sdm_relational_engine.rfm.diagnostics import GraphSanitizationReport from sdm_relational_engine.rfm.explain.result import ( ExplainConfig, @@ -2129,7 +2133,7 @@ def _validate_time( elif ( query.query_type == QueryType.TEMPORAL and context_anchor_time is not None - and context_anchor_time + end_offset > anchor_time + and shift_time(context_anchor_time, end_offset) > anchor_time ): warnings.warn( f'Aggregation for context examples at timestamp ' @@ -2141,10 +2145,14 @@ def _validate_time( elif ( context_anchor_time is not None - and context_anchor_time - end_offset * query.num_forecasts + and shift_time( + context_anchor_time, -end_offset * query.num_forecasts + ) < min_time ): - _time = context_anchor_time - end_offset * query.num_forecasts + _time = shift_time( + context_anchor_time, -end_offset * query.num_forecasts + ) warnings.warn( f'Context anchor timestamp is too early or ' f'aggregation time range is too large. To form ' @@ -2160,13 +2168,12 @@ def _validate_time( f'make sure this is intended.' ) - if ( - evaluate - and anchor_time > max_time - end_offset * query.num_forecasts + if evaluate and anchor_time > shift_time( + max_time, -end_offset * query.num_forecasts ): raise ValueError( f'Anchor timestamp for evaluation is after the latest ' - f"supported timestamp '{max_time - end_offset}'." + f"supported timestamp '{shift_time(max_time, -end_offset)}'." ) def _get_task_table( @@ -2233,7 +2240,9 @@ def _get_task_table( if anchor_time is None: anchor_time = self._get_default_anchor_time(query) if num_test_examples > 0: - anchor_time = anchor_time - step_offset * query.num_forecasts + anchor_time = shift_time( + anchor_time, -step_offset * query.num_forecasts + ) if logger is not None: assert isinstance(anchor_time, pd.Timestamp) @@ -2256,7 +2265,7 @@ def _get_task_table( 'for context and prediction examples' ) if context_anchor_time is None: - context_anchor_time = anchor_time - step_offset + context_anchor_time = shift_time(anchor_time, -step_offset) self._validate_time( query, anchor_time, diff --git a/packages/structured-data-models-relational-engine/tests/base/test_utils.py b/packages/structured-data-models-relational-engine/tests/base/test_utils.py index df223d4c4..c3a2472ef 100644 --- a/packages/structured-data-models-relational-engine/tests/base/test_utils.py +++ b/packages/structured-data-models-relational-engine/tests/base/test_utils.py @@ -7,7 +7,11 @@ import pandas as pd import pytest -from sdm_relational_engine.rfm.base.utils import Timestamp, to_datetime +from sdm_relational_engine.rfm.base.utils import ( + Timestamp, + shift_time, + to_datetime, +) def test_timestamp() -> None: @@ -66,3 +70,32 @@ def test_to_datetime_does_not_warn_about_already_null_values() -> None: with warnings.catch_warnings(): warnings.simplefilter('error') assert to_datetime(ser, 'TS').isna().tolist() == [False, True, True] + + +@pytest.mark.parametrize( + 'offset', + [ + pd.DateOffset(), + pd.DateOffset(days=1), + pd.DateOffset(months=1, days=-31), + -pd.DateOffset(days=2) * 3, + pd.DateOffset(n=0, year=2022), + pd.DateOffset(days=0, normalize=True), + pd.offsets.MonthEnd(0), + ], +) +def test_shift_time_moves_by_an_offset_that_moves( + offset: pd.DateOffset, +) -> None: + time = pd.Timestamp('2020-02-01 13:30') + assert shift_time(time, offset) == time + offset + + +@pytest.mark.parametrize( + 'offset', + [pd.DateOffset(0), pd.DateOffset(days=0), pd.DateOffset(days=1) * 0], +) +def test_shift_time_leaves_the_earliest_time_where_it_is( + offset: pd.DateOffset, +) -> None: + assert shift_time(pd.Timestamp.min, offset) == pd.Timestamp.min diff --git a/packages/structured-data-models-relational-engine/tests/test_rfm.py b/packages/structured-data-models-relational-engine/tests/test_rfm.py index f63d03002..6f0fd9d7c 100644 --- a/packages/structured-data-models-relational-engine/tests/test_rfm.py +++ b/packages/structured-data-models-relational-engine/tests/test_rfm.py @@ -1003,6 +1003,26 @@ def test_forecasting_single_entity_validation( ) +@pytest.mark.parametrize('indices', [[0], None]) +def test_a_graph_without_a_time_column_is_anchored_at_the_earliest_time( + indices: list[int] | None, +) -> None: + """A static query steps by an empty offset, and pandas 3.1 refused to + take one from `Timestamp.min`. + """ + df = pd.DataFrame({'USER_ID': [0, 1, 2, 3], 'AGE': [20, 30, 40, 50]}) + model = KumoRelational(Graph.from_data({'USERS': df}, verbose=False)) + query = ValidatedPredictiveQuery( + target_ast=Column(fqn='USERS.AGE', stype_maybe=Stype.numerical), + entity_ast=Column(fqn='USERS.USER_ID'), + ) + + task_table = model._get_task_table(query, indices=indices) + + anchors = task_table._context_df['ANCHOR_TIMESTAMP'] + assert (anchors == pd.Timestamp.min).all() + + def test_end_time_column() -> None: df = pd.DataFrame( { diff --git a/uv.lock b/uv.lock index ba3219cb0..be6a79353 100644 --- a/uv.lock +++ b/uv.lock @@ -3166,7 +3166,7 @@ test = [ requires-dist = [ { name = "databricks-sdk", marker = "extra == 'databricks-serving'", specifier = ">=0.68.0,<1.0" }, { name = "numpy", specifier = ">=1.23" }, - { name = "pandas", specifier = ">=2.2.2,<3.1" }, + { name = "pandas", specifier = ">=2.2.2" }, { name = "pytest", marker = "extra == 'test'" }, { name = "requests", specifier = ">=2.28" }, { name = "requests-mock", marker = "extra == 'test'" }, @@ -3241,7 +3241,7 @@ requires-dist = [ { name = "databricks-sdk", marker = "extra == 'databricks'", specifier = ">=0.89,<1.0" }, { name = "databricks-sdk", marker = "extra == 'lakebase'", specifier = ">=0.89,<1.0" }, { name = "databricks-sql-connector", marker = "extra == 'databricks'", specifier = ">=4.1.3" }, - { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.5.0,<2" }, + { name = "duckdb", marker = "extra == 'duckdb'", specifier = ">=1.5.0" }, { name = "moto", extras = ["server"], marker = "extra == 'test'", specifier = ">=5" }, { name = "pandas", specifier = ">=2.2.2" }, { name = "psycopg", marker = "extra == 'lakebase'", specifier = ">=3.1" }, @@ -3348,7 +3348,7 @@ requires-dist = [ { name = "numpy" }, { name = "openai", marker = "extra == 'explain'", specifier = ">=1.40" }, { name = "packaging", marker = "extra == 'test'" }, - { name = "pandas", specifier = "<3.1" }, + { name = "pandas" }, { name = "pooch", marker = "extra == 'relbench'" }, { name = "pooch", marker = "extra == 'test'" }, { name = "pyarrow", specifier = ">=10.0.1" },