Support DuckDB 2 and pandas 3.1 on main, and test the wheels CI built

.github/workflows/ci.yml

@@ -202,6 +202,8 @@ jobs:
           print('\n'.join(names))
           if len(names) != 5:
               sys.exit(f'expected 5 wheels, built {len(names)}')
+          if len({n.split('-')[0] for n in names}) != 5:
+              sys.exit('expected one wheel per distribution')
           if len(versions) != 1:
               sys.exit(f'distributions disagree on the version: {sorted(versions)}')
           pathlib.Path('dist/VERSION').write_text(versions.pop())
@@ -210,12 +212,15 @@ jobs:
       # only one distribution carries makes the set unsatisfiable, and the
       # resolver is the only thing that says so. Every one is named by file:
       # once a version is on PyPI, `--find-links` with `==$VERSION` resolves
-      # the published release as readily as the wheel built here.
+      # the published release as readily as the wheel built here. The
+      # pure-Python wheels are named in full; the engine's tag is the runner's,
+      # and the check above leaves exactly one.
       - run: |
           VERSION=$(cat dist/VERSION)
           pip install --dry-run \
-            "$(ls dist/structured_data_models_client-$VERSION-*.whl)[relational,sqlite,duckdb]" \
-            dist/structured_data_models_{api,pql,connectors,relational_engine}-$VERSION-*.whl
+            "dist/structured_data_models_client-$VERSION-py3-none-any.whl[relational,sqlite,duckdb]" \
+            dist/structured_data_models_{api,pql,connectors}-$VERSION-py3-none-any.whl \
+            dist/structured_data_models_relational_engine-$VERSION-*.whl
       # The wheels only: dist/ also carries VERSION for the jobs downstream,
       # and twine refuses anything that is not a distribution.
       - run: pip install --upgrade twine && twine check dist/*.whl
@@ -256,25 +261,16 @@ jobs:
     runs-on: ${{ matrix.runner }}
     steps:
       - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
-      # The engine pins its siblings by exact version, which exist only as
-      # wheels on disk; see the release workflow.
+      # The engine pins its siblings by exact version. Its `before-test`
+      # installs these by file before the engine, so the smoke test never
+      # takes the published release of the same version instead.
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: wheels
           path: _sibling_wheels
-      # Only Linux runs in a container, where the engine's
-      # `[tool.cibuildwheel.linux]` already names /project/_sibling_wheels.
-      - name: Point the wheel test environments at the sibling wheels
+      - name: Build only this row's architecture
         shell: bash
-        run: |
-          echo "${{ matrix.archs }}" >> "$GITHUB_ENV"
-          case "${{ runner.os }}" in
-            Linux) ;;
-            # cibuildwheel layers its own `environment` over the host's, so a
-            # plain variable reaches the build and test environments without
-            # displacing what the engine's pyproject sets per platform.
-            *) echo "PIP_FIND_LINKS=${{ github.workspace }}/_sibling_wheels" >> "$GITHUB_ENV" ;;
-          esac
+        run: echo "${{ matrix.archs }}" >> "$GITHUB_ENV"
 
       - uses: pypa/cibuildwheel@b3395e271a35f73dad77f21802abe9b986dfe299 # v4.3.1
         with:
@@ -332,23 +328,37 @@ jobs:
           pip install --upgrade pip pytest requests-mock
           VERSION=$(cat dist/VERSION)
           pip install \
-            "$(ls dist/structured_data_models_client-$VERSION-*.whl)[sqlite,duckdb]" \
-            dist/structured_data_models_{api,pql,connectors}-$VERSION-*.whl
+            "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb]" \
+            dist/structured_data_models_{api,pql,connectors}-$VERSION-py3-none-any.whl
       - name: The installed distribution is what gets imported, and it is typed
         working-directory: /tmp
         run: |
           python - <<'EOF'
-          import json, pathlib
+          import json, os, pathlib
           from importlib.metadata import distribution
+          from urllib.parse import urlparse
+          from urllib.request import url2pathname
           import predictive_query, sdm_api, sdm_client, sdm_connectors
 
           # The same version on PyPI resolves as readily as the wheel built
           # here, and then this job tests the published release instead.
-          for name in ('client', 'api', 'pql', 'connectors'):
-              recorded = distribution(f'structured-data-models-{name}').read_text('direct_url.json')
-              source = json.loads(recorded or '{}').get('url', 'an index')
-              assert '/dist/' in source, (
-                  f'structured-data-models-{name} was installed from {source}, '
+          built = pathlib.Path(os.environ['GITHUB_WORKSPACE'], 'dist').resolve()
+          for name in (
+              'structured-data-models-client',
+              'structured-data-models-api',
+              'structured-data-models-pql',
+              'structured-data-models-connectors',
+          ):
+              recorded = distribution(name).read_text('direct_url.json')
+              url = json.loads(recorded or '{}').get('url', '')
+              parsed = urlparse(url)
+              source = (
+                  pathlib.Path(url2pathname(parsed.path)).resolve().parent
+                  if parsed.scheme == 'file'
+                  else None
+              )
+              assert source == built, (
+                  f'{name} was installed from {url or "an index"}, '
                   f'not from the wheel built here'
               )
 
@@ -389,8 +399,8 @@ jobs:
           pip install --upgrade pip cyclonedx-bom
           VERSION=$(cat dist/VERSION)
           pip install \
-            "$(ls dist/structured_data_models_client-$VERSION-*.whl)[sqlite,duckdb,postgres,databricks,snowflake,s3]" \
-            dist/structured_data_models_{api,pql,connectors}-$VERSION-*.whl
+            "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb,postgres,databricks,snowflake,s3]" \
+            dist/structured_data_models_{api,pql,connectors}-$VERSION-py3-none-any.whl
           cyclonedx-py environment --of JSON --validate -o dist/sbom.cyclonedx.json
           python -c "import json,sys; d=json.load(open('dist/sbom.cyclonedx.json')); names={c['name'] for c in d.get('components', [])}; missing={'structured-data-models-client','structured-data-models-api','structured-data-models-pql','structured-data-models-connectors'} - names; print(f'{len(names)} components'); sys.exit(f'SBOM missing {sorted(missing)}' if missing else 0)"
       - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2

.github/workflows/publish.yml

@@ -165,27 +165,16 @@ jobs:
 
       # The engine pins its siblings by exact version, and at this point in a
       # release those versions exist only as the artifact `build` uploaded.
-      # cibuildwheel's test environment resolves from PyPI unless told
-      # otherwise, so without this the smoke test cannot find them.
+      # The engine's `before-test` installs them by file before the engine, so
+      # the smoke test tests these and not a published release.
       - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
         with:
           name: pure-distributions
           path: _sibling_wheels
 
-      # Only Linux runs in a container, where the engine's
-      # `[tool.cibuildwheel.linux]` already names /project/_sibling_wheels.
-      - name: Point the wheel test environments at the sibling wheels
+      - name: Build only this row's architecture
         shell: bash
-        run: |
-          echo "${{ matrix.archs }}" >> "$GITHUB_ENV"
-          case "${{ runner.os }}" in
-            Linux) ;;
-            # cibuildwheel layers its own `environment` over the host's, so a
-            # plain variable reaches the build and test environments without
-            # displacing what the engine's pyproject sets per platform.
-            *) echo "PIP_FIND_LINKS=${{ github.workspace }}/_sibling_wheels" >> "$GITHUB_ENV" ;;
-          esac
-
+        run: echo "${{ matrix.archs }}" >> "$GITHUB_ENV"
 
       - uses: pypa/cibuildwheel@b3395e271a35f73dad77f21802abe9b986dfe299 # v4.3.1
         with:
@@ -359,12 +348,17 @@ jobs:
       - name: Software bill of materials
         run: |
           python -m pip install --upgrade cyclonedx-bom
-          # --find-links dist for the same reason as the engine's test env: the
-          # exact sibling pins resolve to the wheels just built, not to whatever
-          # happens to be on PyPI. Without it this describes a previous release,
-          # or fails outright before the first one.
-          python -m pip install --find-links dist \
-            "./packages/structured-data-models-client[relational,sqlite,duckdb,postgres,databricks,snowflake,s3]"
+          # Every distribution from dist/, never by name and version alone:
+          # after a partial upload, PyPI's release of this version resolves as
+          # readily as the wheels being released, and this would describe it.
+          # The engine has a wheel per platform, so pip picks this runner's
+          # from dist/ and nowhere else.
+          version="${GITHUB_REF_NAME#v}"
+          python -m pip install --no-index --no-deps --find-links dist \
+            "structured-data-models-relational-engine==$version"
+          python -m pip install \
+            "dist/structured_data_models_client-$version-py3-none-any.whl[relational,sqlite,duckdb,postgres,databricks,snowflake,s3]" \
+            dist/structured_data_models_{api,pql,connectors}-$version-py3-none-any.whl
           cyclonedx-py environment --of JSON --validate -o dist/sbom.cyclonedx.json
           python -c "import json,sys; d=json.load(open('dist/sbom.cyclonedx.json')); names={c['name'] for c in d.get('components', [])}; missing={'structured-data-models-client','structured-data-models-api','structured-data-models-pql','structured-data-models-connectors','structured-data-models-relational-engine'} - names; print(f'{len(names)} components'); sys.exit(f'SBOM missing {sorted(missing)}' if missing else 0)"
 

.gitlab-ci.yml

@@ -193,17 +193,25 @@ build:
       print('\n'.join(names))
       if len(names) != 5:
           sys.exit(f'expected 5 wheels, built {len(names)}')
+      if len({n.split('-')[0] for n in names}) != 5:
+          sys.exit('expected one wheel per distribution')
       if len(versions) != 1:
           sys.exit(f'distributions disagree on the version: {sorted(versions)}')
       pathlib.Path('dist/VERSION').write_text(versions.pop())
       EOF
     # Building each one proves nothing about installing them together. A pin
     # that only one distribution carries makes the set unsatisfiable, and the
-    # resolver is the only thing that says so.
+    # resolver is the only thing that says so. Every one is named by file:
+    # once a version is on PyPI, `--find-links` with `==$VERSION` resolves the
+    # published release as readily as the wheel built here.
     - |
       VERSION=$(cat dist/VERSION)
-      pip install --dry-run --find-links "$CI_PROJECT_DIR/dist" \
-        "structured-data-models-client[relational,sqlite,duckdb]==$VERSION"
+      pip install --dry-run \
+        "dist/structured_data_models_client-$VERSION-py3-none-any.whl[relational,sqlite,duckdb]" \
+        "dist/structured_data_models_api-$VERSION-py3-none-any.whl" \
+        "dist/structured_data_models_pql-$VERSION-py3-none-any.whl" \
+        "dist/structured_data_models_connectors-$VERSION-py3-none-any.whl" \
+        dist/structured_data_models_relational_engine-"$VERSION"-*.whl
   artifacts:
     paths:
       - dist/
@@ -245,8 +253,7 @@ wheels:
   variables:
     DOCKER_HOST: tcp://docker:2375
     DOCKER_TLS_CERTDIR: ""
-    # Into the manylinux container too. cibuildwheel prepends passed variables
-    # to the `environment` pyproject sets, so PIP_FIND_LINKS still applies.
+    # Into the manylinux container too, where the smoke test installs.
     CIBW_ENVIRONMENT_PASS_LINUX: PIP_INDEX_URL
   parallel:
     matrix:
@@ -272,7 +279,7 @@ wheels:
       fi
     - pip install -q build 'cibuildwheel==4.3.0'
     # The engine's smoke test installs it against these, which exist nowhere
-    # but here until a release; `[tool.cibuildwheel.linux]` points pip at them.
+    # but here until a release; its `before-test` installs them by file.
     - |
       for package in structured-data-models-api structured-data-models-pql structured-data-models-client structured-data-models-connectors; do
         ( cd "packages/$package" && python -m build --wheel --outdir "$CI_PROJECT_DIR/_sibling_wheels" . )
@@ -336,17 +343,46 @@ installed-wheel:
   script:
     - pip install -q pytest requests-mock pytest-cov
     - VERSION=$(cat dist/VERSION)
-    - pip install -q --find-links "$CI_PROJECT_DIR/dist" "structured-data-models-client[sqlite,duckdb]==$VERSION"
+    # By file: the same version on PyPI resolves as readily as the wheel built
+    # here, and then this job tests the published release instead.
+    - pip install -q
+        "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb]"
+        "dist/structured_data_models_api-$VERSION-py3-none-any.whl"
+        "dist/structured_data_models_pql-$VERSION-py3-none-any.whl"
+        "dist/structured_data_models_connectors-$VERSION-py3-none-any.whl"
     # Prove the installed distribution, not the checkout, is what is imported,
     # and that the typed marker survived the build. Written as one script that
     # raises, rather than a chain ending in `|| true`: a check that cannot fail
     # is worse than no check, because it reads like one that passed.
     - cd /tmp
     - |
       python - <<'EOF'
-      import pathlib
+      import json, os, pathlib
+      from importlib.metadata import distribution
+      from urllib.parse import urlparse
+      from urllib.request import url2pathname
       import predictive_query, sdm_api, sdm_client, sdm_connectors
 
+      built = pathlib.Path(os.environ['CI_PROJECT_DIR'], 'dist').resolve()
+      for name in (
+          'structured-data-models-client',
+          'structured-data-models-api',
+          'structured-data-models-pql',
+          'structured-data-models-connectors',
+      ):
+          recorded = distribution(name).read_text('direct_url.json')
+          url = json.loads(recorded or '{}').get('url', '')
+          parsed = urlparse(url)
+          source = (
+              pathlib.Path(url2pathname(parsed.path)).resolve().parent
+              if parsed.scheme == 'file'
+              else None
+          )
+          assert source == built, (
+              f'{name} was installed from {url or "an index"}, '
+              f'not from the wheel built here'
+          )
+
       for module in (sdm_api, sdm_client, sdm_connectors, predictive_query):
           location = pathlib.Path(module.__file__).resolve()
           assert 'site-packages' in location.parts, (
@@ -412,8 +448,11 @@ sbom:
   script:
     - pip install -q cyclonedx-bom
     - VERSION=$(cat dist/VERSION)
-    - pip install -q --find-links "$CI_PROJECT_DIR/dist"
-        "structured-data-models-client[sqlite,duckdb,postgres,databricks,snowflake,s3]==$VERSION"
+    - pip install -q
+        "dist/structured_data_models_client-$VERSION-py3-none-any.whl[sqlite,duckdb,postgres,databricks,snowflake,s3]"
+        "dist/structured_data_models_api-$VERSION-py3-none-any.whl"
+        "dist/structured_data_models_pql-$VERSION-py3-none-any.whl"
+        "dist/structured_data_models_connectors-$VERSION-py3-none-any.whl"
     - cyclonedx-py environment --of JSON --validate -o "$CI_PROJECT_DIR/dist/sbom.cyclonedx.json"
     - python -c "import json,sys; d=json.load(open('dist/sbom.cyclonedx.json'));
         names={c['name'] for c in d.get('components', [])};

CHANGELOG.md

@@ -36,20 +36,22 @@ these packages. All five distributions share one version and one entry.
   value that cannot be read as its type, such as a `run_mode` that is not
   text, is refused with `INVALID_REQUEST` before sending, where a NIM
   answered `422 VALIDATION_FAILED`.
-- The `duckdb` extra holds `duckdb` below 2. The 2.0 release warns on the
-  Arrow fetch the relational engine reads DuckDB tables through, and this
-  release is tested against 1.5.
-- `structured-data-models-client` and the relational engine hold `pandas`
-  below 3.1. The 3.1 release raises taking a date offset from the earliest
-  timestamp, which the engine does for every graph without a time column,
-  and this release is tested against 3.0.
 
 ### Fixed
 
 - A predictive query longer than 1,024 characters is answered. A relational
   request carries the query in `metadata.source_query` for logging, which the
   contract caps at 1,024 characters, so a NIM refused a longer one with
   `VALIDATION_FAILED`. The logged copy is now cut to fit.
+- DuckDB 2.0 no longer warns on every read. The relational engine and
+  `read_table` read a DuckDB result through a relation rather than the DB-API
+  cursor, which 2.0 warns makes an extra copy of the rows.
+- A relational prediction over a graph without a time column works on pandas
+  3.1. Its anchor is the earliest timestamp pandas holds, and 3.1 refused to
+  take even an empty date offset from it.
+- A request's description leaves out a wire table carrying an integer past
+  the float range on every pandas release. pandas 3.1 had started to build
+  a frame around one, where earlier releases refused it.
 
 ## 1.0.0 - 2026-10-15
 

packages/structured-data-models-client/pyproject.toml

@@ -41,10 +41,7 @@ classifiers = [
 dependencies = [
     "structured-data-models-api==1.0.0",
     "structured-data-models-pql==1.0.0",
-    # 3.1 builds a frame around an integer past the float range where earlier
-    # releases refused one, and a request's description relies on that
-    # refusal to leave out the table carrying it.
-    "pandas>=2.2.2,<3.1",
+    "pandas>=2.2.2",
     "numpy>=1.23",
     "requests>=2.28",
     "urllib3>=1.26",

packages/structured-data-models-client/src/sdm_client/request.py

@@ -32,6 +32,7 @@
 import json
 import math
 import re
+import sys
 import warnings
 from collections.abc import Mapping, Sequence
 from dataclasses import dataclass, field
@@ -877,23 +878,39 @@ def _entity_table_in(payloads: Sequence[Mapping[str, Any]]) -> str:
 
 
 def _wire_frame(table: Any) -> pd.DataFrame | None:
-    r"""One wire table as a frame, or nothing if it cannot be read as one."""
+    r"""One wire table as a frame, or nothing if it cannot be read as one.
+
+    An integer past the float range is refused here rather than left to
+    pandas, whose releases disagree on whether a frame can hold one.
+    """
     if not isinstance(table, Mapping):
         return None
     columns = table.get('columns')
     rows = table.get('rows')
     if not isinstance(columns, list) or not isinstance(rows, list):
         return None
+    if any(
+        _past_the_float_range(value)
+        for row in rows
+        if isinstance(row, (list, tuple))
+        for value in row
+    ):
+        return None
     try:
         return pd.DataFrame(rows, columns=columns)
     except (TypeError, ValueError, OverflowError):
-        # A row wider than its header raises rather than padding, and a
-        # number too large for pandas to hold raises as well. Reading a
+        # A row wider than its header raises rather than padding. Reading a
         # request is a description of it, and a description that cannot be
         # made is not worth failing the call for.
         return None
 
 
+def _past_the_float_range(value: Any) -> bool:
+    return isinstance(value, int) and not (
+        -sys.float_info.max <= value <= sys.float_info.max
+    )
+
+
 #: What ties a related row to the instance it hangs off.
 INSTANCE_ID = 'instance_id'