From dc2ae92900fc5b622f6e3d7e92e53a2596c33a2d Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Mon, 31 Aug 2026 12:04:20 +0200 Subject: [PATCH 1/2] Pin serverless databricks-connect to MAJOR.MINOR, not bare major MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A serverless environment's constraint list (protobuf, grpcio, pyarrow, etc.) is generated from a specific runtime snapshot, so it is only validated against the MAJOR.MINOR release series present at generation time. The previous bare-major pin (``~=18.0``, resolving ``>=18.0, <19``) allowed newer minor releases whose bundled pyspark protobuf stubs may be incompatible with the pinned runtime libraries. Concretely, ``databricks-connect~=18.0`` resolved to 18.3.4, whose pyspark pb2 files use a ``_globals`` pattern intended for protobuf 6.x's ``BuildTopDescriptorsAndMessages``. With the v5 constraint list's ``protobuf~=5.29.4`` (the runtime version), that method does not inject ``_globals``, so the ``if not _USE_C_DESCRIPTORS:`` branch raises ``NameError: name '_globals' is not defined``. Fix ``dbconnect_pin()`` to pin to MAJOR.MINOR.0 (``~=18.0.0``, resolving ``>=18.0.0, <18.1``) — matching what was generated for DBR environments all along — and regenerate the six serverless pyprojects. Co-authored-by: Isaac --- .github/scripts/envgen.py | 12 +++++++----- python/serverless/serverless-v1/pyproject.toml | 2 +- python/serverless/serverless-v2/pyproject.toml | 2 +- python/serverless/serverless-v3/pyproject.toml | 2 +- python/serverless/serverless-v4/pyproject.toml | 2 +- python/serverless/serverless-v5-ml/pyproject.toml | 2 +- python/serverless/serverless-v5/pyproject.toml | 2 +- 7 files changed, 13 insertions(+), 11 deletions(-) diff --git a/.github/scripts/envgen.py b/.github/scripts/envgen.py index f4cfdc5..37cd41d 100644 --- a/.github/scripts/envgen.py +++ b/.github/scripts/envgen.py @@ -156,15 +156,17 @@ def _filtered(pkgs, env_name=None): def dbconnect_pin(pkgs): """Return the dev-group databricks-connect requirement, or None. - Serverless lists a concrete databricks-connect (e.g. '17.3.1'), but a serverless - environment version tracks a whole major line, not a single point release. So the - pin is by bare major — ``~=MAJOR.0`` (e.g. '17.3.1' -> ``databricks-connect~=17.0``, - resolving ``>=17.0, <18.0``) — to pick up the latest release within that major. + Serverless lists a concrete databricks-connect (e.g. '18.0.9'). The pin is by + MAJOR.MINOR — ``~=MAJOR.MINOR.0`` (e.g. '18.0.9' -> ``databricks-connect~=18.0.0``, + resolving ``>=18.0.0, <18.1``) — matching the minor series the constraint list was + generated from. A broader major-only pin (``~=18.0``) allows newer minor releases + whose bundled pyspark protobuf stubs may be incompatible with the pinned runtime + libraries (protobuf, grpcio) in the constraint list. """ v = pkgs.get("databricks-connect") if not v: return None - return f"databricks-connect~={v.split('.')[0]}.0" + return f"databricks-connect~={'.'.join(v.split('.')[:2])}.0" def build_pyproject(pkgs, env_name, python_version, dbconnect=None): diff --git a/python/serverless/serverless-v1/pyproject.toml b/python/serverless/serverless-v1/pyproject.toml index 601f54d..f26c2d2 100644 --- a/python/serverless/serverless-v1/pyproject.toml +++ b/python/serverless/serverless-v1/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.10.*" [dependency-groups] dev = [ - "databricks-connect~=14.0", + "databricks-connect~=14.3.0", ] [tool.uv] diff --git a/python/serverless/serverless-v2/pyproject.toml b/python/serverless/serverless-v2/pyproject.toml index 497624e..bd154dc 100644 --- a/python/serverless/serverless-v2/pyproject.toml +++ b/python/serverless/serverless-v2/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.11.*" [dependency-groups] dev = [ - "databricks-connect~=15.0", + "databricks-connect~=15.4.0", ] [tool.uv] diff --git a/python/serverless/serverless-v3/pyproject.toml b/python/serverless/serverless-v3/pyproject.toml index 89c4b0c..dde7b22 100644 --- a/python/serverless/serverless-v3/pyproject.toml +++ b/python/serverless/serverless-v3/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=16.0", + "databricks-connect~=16.4.0", ] [tool.uv] diff --git a/python/serverless/serverless-v4/pyproject.toml b/python/serverless/serverless-v4/pyproject.toml index 7440c67..1f3f040 100644 --- a/python/serverless/serverless-v4/pyproject.toml +++ b/python/serverless/serverless-v4/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=17.0", + "databricks-connect~=17.3.0", ] [tool.uv] diff --git a/python/serverless/serverless-v5-ml/pyproject.toml b/python/serverless/serverless-v5-ml/pyproject.toml index 9861787..9a1ef8c 100644 --- a/python/serverless/serverless-v5-ml/pyproject.toml +++ b/python/serverless/serverless-v5-ml/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=18.0", + "databricks-connect~=18.0.0", ] [tool.uv] diff --git a/python/serverless/serverless-v5/pyproject.toml b/python/serverless/serverless-v5/pyproject.toml index e344eb3..53f1496 100644 --- a/python/serverless/serverless-v5/pyproject.toml +++ b/python/serverless/serverless-v5/pyproject.toml @@ -7,7 +7,7 @@ requires-python = "==3.12.*" [dependency-groups] dev = [ - "databricks-connect~=18.0", + "databricks-connect~=18.0.0", ] [tool.uv] From 737e221975d0f529b7709dea7b1408a4adac74d3 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Mon, 31 Aug 2026 12:50:12 +0200 Subject: [PATCH 2/2] Fix DbconnectPinTest to expect MAJOR.MINOR.0 pin Update the expected value from ``~=17.0`` (the old bare-major pin) to ``~=17.3.0`` (MAJOR.MINOR.0) to match the corrected dbconnect_pin() and also fix the comment that still said ``~=MAJOR.0``. Co-authored-by: Isaac --- .github/scripts/test_envgen.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/test_envgen.py b/.github/scripts/test_envgen.py index 4eaf8e6..4ba2fa1 100644 --- a/.github/scripts/test_envgen.py +++ b/.github/scripts/test_envgen.py @@ -132,12 +132,12 @@ def test_constraints(self): class DbconnectPinTest(unittest.TestCase): def test_strips_local_version_segment(self): # databricks-connect is installed from the dev group as a plain PyPI release; - # the pin is normalized to ~=MAJOR.0, so a local segment in the release-notes + # the pin is normalized to ~=MAJOR.MINOR.0, so a local segment in the release-notes # version is discarded and never lands in an artifact. dbconnect_pin reads raw # pkgs (not _filtered), so this guards that the normalization does the stripping. self.assertEqual( dbconnect_pin({"databricks-connect": "17.3.1+db1"}), - "databricks-connect~=17.0", + "databricks-connect~=17.3.0", ) def test_none_when_absent(self):