From b84e5bc09f97a96a3a72ed93635909136a614f07 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Mon, 31 Aug 2026 11:55:43 +0200 Subject: [PATCH 1/2] Publish end-of-support DBR runtimes still requested by clusters (12.2, 16.1) EoS runtime lines are dropped from the release-notes index and skipped by dbr_point_releases' is_eos check, so discover_dbr never sees them and there is no way to enumerate them. VPEX E_ENV_UNSUPPORTED telemetry shows a few EoS env_keys still requested by live clusters with no environment published: dbr/12.2.x-scala2.12 (8 events / 2 workspaces) and dbr/16.1.x-scala2.12 (1 / 1). Add a narrow, telemetry-curated allowlist (sync.DBR_EOS_PUBLISH) of EoS page slugs to publish anyway. Factor sync_dbr's per-slug body into _sync_dbr_page and add sync_dbr_eos, which runs the allowlist through it directly (bypassing the EoS skip); each is a pre-18-style page so it publishes only its own .x folder. 16.1 is Python 3.12 on pandas 1.5.3 (no cp312 wheel), so it joins DROP_BY_ENV like 16.4/serverless-v3; 12.2 is Python 3.9 and keeps pandas. Both generated envs pass uv sync. Standard (non-ML) only: no EoS ML line has traffic. Co-authored-by: Isaac --- .github/scripts/envgen.py | 7 +- .github/scripts/sync.py | 91 +++++++---- .github/scripts/test_envgen.py | 7 +- .github/scripts/test_sync.py | 30 ++++ python/dbr/12.2.x-scala2.12/constraints.txt | 124 +++++++++++++++ python/dbr/12.2.x-scala2.12/pyproject.toml | 137 ++++++++++++++++ python/dbr/16.1.x-scala2.12/constraints.txt | 152 ++++++++++++++++++ python/dbr/16.1.x-scala2.12/pyproject.toml | 165 ++++++++++++++++++++ 8 files changed, 681 insertions(+), 32 deletions(-) create mode 100644 python/dbr/12.2.x-scala2.12/constraints.txt create mode 100644 python/dbr/12.2.x-scala2.12/pyproject.toml create mode 100644 python/dbr/16.1.x-scala2.12/constraints.txt create mode 100644 python/dbr/16.1.x-scala2.12/pyproject.toml diff --git a/.github/scripts/envgen.py b/.github/scripts/envgen.py index 37cd41d..36b2ad3 100644 --- a/.github/scripts/envgen.py +++ b/.github/scripts/envgen.py @@ -65,8 +65,8 @@ # Environment-scoped drops: removed for the named environments only (DROP is global). # pandas 1.5.3 has no cp312 wheel — pandas ships CPython 3.12 wheels only from 2.1.1 — # so `uv sync` on a Python 3.12 runtime can't install it and falls back to a failing -# source build. DBR 16.4 and serverless-v3 are the only 3.12 runtimes still pinned to -# pandas 1.5.x: 13.3/14.3/15.4 predate 3.12, and 17.3+ / serverless-v4+ already ship +# source build. DBR 16.1, 16.4 and serverless-v3 are the only 3.12 runtimes still pinned +# to pandas 1.5.x: 13.3/14.3/15.4 predate 3.12, and 17.3+ / serverless-v4+ already ship # pandas 2.x (which has cp312 wheels). No in-range (~=1.5) version has a cp312 wheel, # so the pin can't be salvaged by widening — it is dropped for just these envs, and # pandas resolves to an installable version locally. See issue #18. @@ -75,6 +75,9 @@ # and self-expiring: if one of these runtimes is ever re-pinned to a cp312-wheel 2.x, # the version no longer matches and the pin is kept rather than silently discarded. DROP_BY_ENV = { + # DBR 16.1 is an EoS line published from sync.DBR_EOS_PUBLISH; it too is Python 3.12 + # on pandas 1.5.3, so it carries the same drop. + "16.1.x-scala2.12": {"pandas": "1.5."}, "16.4.x-scala2.12": {"pandas": "1.5."}, "16.4.x-cpu-ml-scala2.12": {"pandas": "1.5."}, "16.4.x-gpu-ml-scala2.12": {"pandas": "1.5."}, diff --git a/.github/scripts/sync.py b/.github/scripts/sync.py index 8899301..8fc7a87 100755 --- a/.github/scripts/sync.py +++ b/.github/scripts/sync.py @@ -53,6 +53,17 @@ # Index entries that aren't a runtime version page. DBR_NON_VERSION = {"maintenance-updates", "databricks-runtime-ver", "eos"} +# End-of-support runtime lines to publish anyway, as page slugs. EoS lines are dropped +# from the release-notes index (so discover_dbr never sees them) and skipped by +# dbr_point_releases' EoS check, and the 'eos' page links none — so they can't be +# discovered, only listed explicitly. Live clusters still request a few of them, and VPEX +# telemetry then records E_ENV_UNSUPPORTED (an owned target with no environment published), +# so we make a deliberate, narrow exception to the EoS-not-published policy for exactly the +# lines with real traffic. Add a slug ONLY when telemetry shows its exact env_key failing: +# 12.2 -> dbr/12.2.x-scala2.12 (8 events / 2 workspaces) +# 16.1 -> dbr/16.1.x-scala2.12 (1 event / 1 workspace) +DBR_EOS_PUBLISH = ["12.2", "16.1"] + def fetch(url): req = urllib.request.Request(url, headers={"User-Agent": "databricks-environments-sync"}) @@ -440,40 +451,62 @@ def _write_env(key, pkgs, python_version, dbconnect): print(f" + dbr/{key} (python {python_version}, {len(pkgs)} packages)") +def _sync_dbr_page(slug, point_release, umbrella_major): + """Fetch one standard runtime page and write its folder(s). + + A live point release publishes its own '.x' folder (pre-18: the whole line; + EoS allowlist lines: their own line). The newest live release of a DBR 18+ line ALSO + publishes the bare-major '.x' umbrella that a cluster's spark_version resolves + to — in ADDITION to its own folder, so the umbrella page (18.2) yields both '18.2.x' + and '18.x' (``umbrella_major`` set). A point-release folder pins databricks-connect to + its exact minor (18.2 -> ~=18.2.0); the bare-major umbrella instead pins the whole major + line (18 -> ~=18.0), so a cluster addressing the line by its bare major resolves the + newest databricks-connect in the major.""" + try: + html = fetch(DBR_PAGE.format(slug=slug)) + except Exception as e: + print(f" ! dbr [{slug}]: fetch failed ({e}); skipping") + return + meta = dbr_meta(html) + pkgs, _ = parse_dbr_page(html) + if not meta or not pkgs: + print(f" ! dbr [{slug}]: no meta / Python table; skipping") + return + key_ver, dbconnect_ver, scalas, python_version = meta + folder_vers = ([key_ver] if point_release else []) + ([umbrella_major] if umbrella_major else []) + for folder_ver in folder_vers: + # The bare-major umbrella tracks the whole major line (18 -> ~=18.0), like a + # serverless major; a point release (and every EoS allowlist line) keeps its exact + # minor (18.2 -> ~=18.2.0). See the docstring. + dbconnect = umbrella_major if folder_ver == umbrella_major else dbconnect_ver + for scala in scalas: + _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect) + + def sync_dbr(): for index_slug in discover_dbr(): major = _umbrella_major(index_slug) pointrelease_slugs, umbrella_slug = dbr_point_releases(index_slug, ml=False) # The umbrella slug is usually also one of the point releases — fetch each page once. for slug in dict.fromkeys(pointrelease_slugs + ([umbrella_slug] if umbrella_slug else [])): - try: - html = fetch(DBR_PAGE.format(slug=slug)) - except Exception as e: - print(f" ! dbr [{slug}]: fetch failed ({e}); skipping") - continue - meta = dbr_meta(html) - pkgs, _ = parse_dbr_page(html) - if not meta or not pkgs: - print(f" ! dbr [{slug}]: no meta / Python table; skipping") - continue - key_ver, dbconnect_ver, scalas, python_version = meta - # A live point release publishes its own '.x' folder (pre-18: the whole - # line). The newest live one ALSO publishes the bare-major '.x' umbrella - # that a DBR 18+ cluster's spark_version resolves to — in ADDITION to its own - # folder, so the umbrella page (18.2) yields both '18.2.x' and '18.x'. Both use - # this page's dbconnect_ver (the latest point release's databricks-connect minor). - folder_vers = [key_ver] if slug in pointrelease_slugs else [] - if major and slug == umbrella_slug: - folder_vers.append(major) - for folder_ver in folder_vers: - # A point-release folder pins its exact minor (18.2 -> ~=18.2.0). The bare- - # major umbrella tracks the whole major line, like a serverless major, so it - # pins ~=MAJOR.0 (18 -> ~=18.0): a cluster addressing the line by its bare - # major resolves the latest databricks-connect in the major, and a new point - # release doesn't need a regen to be covered. - dbconnect = major if folder_ver == major else dbconnect_ver - for scala in scalas: - _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect) + _sync_dbr_page( + slug, + point_release=slug in pointrelease_slugs, + umbrella_major=major if slug == umbrella_slug else None, + ) + + +def sync_dbr_eos(): + """Publish the end-of-support lines in DBR_EOS_PUBLISH (see its comment for why they + exist and the telemetry rule for adding one). + + These are dropped from the index and skipped by dbr_point_releases' EoS check, so they + are fetched directly here rather than discovered. Each is a self-contained runtime page + with the real minor in its title (pre-18-style), so it publishes only its own + '.x' folder(s) and no bare-major umbrella. They can't collide with the + index-driven sync_dbr (that path never yields an EoS slug).""" + for slug in DBR_EOS_PUBLISH: + _sync_dbr_page(slug, point_release=True, umbrella_major=None) def ml_variant_pkgs(ml_html, variant): @@ -611,6 +644,8 @@ def main(): sync_serverless() print("Syncing DBR runtimes from docs.databricks.com ...") sync_dbr() + print("Publishing end-of-support DBR runtimes still requested by clusters ...") + sync_dbr_eos() print("Syncing DBR ML runtimes (CPU + GPU) from docs.databricks.com ...") sync_dbr_ml() changed = reconcile() diff --git a/.github/scripts/test_envgen.py b/.github/scripts/test_envgen.py index 4ba2fa1..3e35ce7 100644 --- a/.github/scripts/test_envgen.py +++ b/.github/scripts/test_envgen.py @@ -43,12 +43,15 @@ def test_keeps_installable_pins(self): class EnvScopedDropTest(unittest.TestCase): def test_pandas_dropped_on_py312_1_5_runtimes(self): - # DBR 16.4 + serverless-v3 are the only Python-3.12 runtimes still on + # DBR 16.1, 16.4 + serverless-v3 are the only Python-3.12 runtimes still on # pandas 1.5.x, which has no cp312 wheel — dropped for just these envs. DBR 16.4 # ships both Scala images off one page (same Python packages), so the scala2.13 - # variants carry the identical pandas problem and are dropped the same way. + # variants carry the identical pandas problem and are dropped the same way. DBR + # 16.1 is an EoS line published from the telemetry allowlist and is Python 3.12 on + # pandas 1.5.3 too, so it needs the same drop. pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"} for env in ( + "16.1.x-scala2.12", "16.4.x-scala2.12", "16.4.x-cpu-ml-scala2.12", "16.4.x-gpu-ml-scala2.12", diff --git a/.github/scripts/test_sync.py b/.github/scripts/test_sync.py index 30ee16f..4ab663a 100644 --- a/.github/scripts/test_sync.py +++ b/.github/scripts/test_sync.py @@ -420,5 +420,35 @@ def test_umbrella_ml_line_publishes_point_releases_and_umbrella(self): ) +class SyncDbrEosTest(unittest.TestCase): + def test_publishes_allowlisted_eos_line_despite_eos_marker(self): + # A line on the EoS allowlist is published from its own page even though its title + # carries '(EoS)' — the reason the allowlist exists (these lines are dropped from + # the index and skipped by dbr_point_releases' EoS check). Like a pre-18 line, it + # writes only its own '.x' folder and no bare-major umbrella. + pages = {"16.1": _runtime_page("Databricks Runtime 16.1 (EoS)", scala="2.12")} + writes = [] + with mock.patch.object(sync, "DBR_EOS_PUBLISH", ["16.1"]), \ + mock.patch.object(sync, "fetch", _fake_fetch(pages)), \ + mock.patch.object(sync, "_write_env", + lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))): + sync.sync_dbr_eos() + self.assertEqual(writes, [("16.1.x-scala2.12", "16.1")]) + + def test_skips_an_allowlisted_slug_whose_page_is_gone(self): + # An EoS page can eventually be removed from the docs; a fetch failure on one + # allowlisted slug is logged and skipped without aborting the rest of the run. + def boom(url): + raise Exception("HTTP 404") + + writes = [] + with mock.patch.object(sync, "DBR_EOS_PUBLISH", ["12.2"]), \ + mock.patch.object(sync, "fetch", boom), \ + mock.patch.object(sync, "_write_env", + lambda *a, **k: writes.append(a)): + sync.sync_dbr_eos() + self.assertEqual(writes, []) + + if __name__ == "__main__": unittest.main() diff --git a/python/dbr/12.2.x-scala2.12/constraints.txt b/python/dbr/12.2.x-scala2.12/constraints.txt new file mode 100644 index 0000000..6bd098e --- /dev/null +++ b/python/dbr/12.2.x-scala2.12/constraints.txt @@ -0,0 +1,124 @@ +# constraints.txt file for Databricks Runtime 12.2.x-scala2.12 + +argon2-cffi~=21.3.0 +argon2-cffi-bindings~=21.2.0 +asttokens~=2.0.5 +attrs~=21.4.0 +backcall~=0.2.0 +backports-entry-points-selectable~=1.2.0 +beautifulsoup4~=4.11.1 +black~=22.3.0 +bleach~=4.1.0 +boto3~=1.21.32 +botocore~=1.24.32 +certifi~=2021.10.8 +cffi~=1.15.0 +chardet~=4.0.0 +charset-normalizer~=2.0.4 +click~=8.0.4 +cryptography~=3.4.8 +cycler~=0.11.0 +cython~=0.29.28 +debugpy~=1.5.1 +decorator~=5.1.1 +defusedxml~=0.7.1 +distlib~=0.3.6 +docstring-to-markdown~=0.11 +entrypoints~=0.4 +executing~=0.8.3 +facets-overview~=1.0.0 +fastjsonschema~=2.16.2 +filelock~=3.9.0 +fonttools~=4.25.0 +idna~=3.3 +ipykernel~=6.15.3 +ipython~=8.5.0 +ipython-genutils~=0.2.0 +ipywidgets~=7.7.2 +jedi~=0.18.1 +jinja2~=2.11.3 +jmespath~=0.10.0 +joblib~=1.1.1 +jsonschema~=4.4.0 +jupyter-client~=6.1.12 +jupyter-core~=4.11.2 +jupyterlab-pygments~=0.1.2 +jupyterlab-widgets~=1.0.0 +kiwisolver~=1.3.2 +markupsafe~=2.0.1 +matplotlib~=3.5.1 +matplotlib-inline~=0.1.2 +mccabe~=0.7.0 +mistune~=0.8.4 +mypy-extensions~=0.4.3 +nbclient~=0.5.13 +nbconvert~=6.4.4 +nbformat~=5.3.0 +nest-asyncio~=1.5.5 +nodeenv~=1.7.0 +notebook~=6.4.8 +numpy~=1.21.5 +packaging~=21.3 +pandas~=1.4.2 +pandocfilters~=1.5.0 +parso~=0.8.3 +pathspec~=0.9.0 +patsy~=0.5.2 +pexpect~=4.8.0 +pickleshare~=0.7.5 +pillow~=9.0.1 +platformdirs~=2.6.2 +plotly~=5.6.0 +pluggy~=1.0.0 +prometheus-client~=0.13.1 +prompt-toolkit~=3.0.20 +protobuf~=3.19.4 +psutil~=5.8.0 +psycopg2~=2.9.3 +ptyprocess~=0.7.0 +pure-eval~=0.2.2 +pyarrow~=7.0.0 +pycparser~=2.21 +pyflakes~=2.5.0 +pygments~=2.11.2 +pyodbc~=4.0.32 +pyparsing~=3.0.4 +pyright~=1.1.283 +pyrsistent~=0.18.0 +python-dateutil~=2.8.2 +python-lsp-jsonrpc~=1.0.0 +python-lsp-server~=1.6.0 +pytz~=2021.3 +pyzmq~=22.3.0 +requests~=2.27.1 +requests-unixsocket~=0.2.0 +rope~=0.22.0 +s3transfer~=0.5.0 +scikit-learn~=1.0.2 +scipy~=1.7.3 +seaborn~=0.11.2 +send2trash~=1.8.0 +setuptools~=61.2.0 +six~=1.16.0 +soupsieve~=2.3.1 +ssh-import-id~=5.10 +stack-data~=0.2.0 +statsmodels~=0.13.2 +tenacity~=8.0.1 +terminado~=0.13.1 +testpath~=0.5.0 +threadpoolctl~=2.2.0 +tokenize-rt~=4.2.1 +tomli~=1.2.2 +tornado~=6.1 +traitlets~=5.1.1 +typing-extensions~=4.1.1 +ujson~=5.1.0 +urllib3~=1.26.9 +virtualenv~=20.8.0 +wcwidth~=0.2.5 +webencodings~=0.5.1 +whatthepatch~=1.0.4 +wheel~=0.37.0 +widgetsnbextension~=3.6.1 +yapf~=0.31.0 diff --git a/python/dbr/12.2.x-scala2.12/pyproject.toml b/python/dbr/12.2.x-scala2.12/pyproject.toml new file mode 100644 index 0000000..4e14f09 --- /dev/null +++ b/python/dbr/12.2.x-scala2.12/pyproject.toml @@ -0,0 +1,137 @@ +# pyproject.toml file for Databricks Runtime 12.2.x-scala2.12 + +[project] +name = "constraint-env-12-2-x-scala2-12" +version = "0.1.0" +requires-python = "==3.9.*" + +[dependency-groups] +dev = [ + "databricks-connect~=12.2.0", +] + +[tool.uv] +constraint-dependencies = [ + "argon2-cffi~=21.3.0", + "argon2-cffi-bindings~=21.2.0", + "asttokens~=2.0.5", + "attrs~=21.4.0", + "backcall~=0.2.0", + "backports-entry-points-selectable~=1.2.0", + "beautifulsoup4~=4.11.1", + "black~=22.3.0", + "bleach~=4.1.0", + "boto3~=1.21.32", + "botocore~=1.24.32", + "certifi~=2021.10.8", + "cffi~=1.15.0", + "chardet~=4.0.0", + "charset-normalizer~=2.0.4", + "click~=8.0.4", + "cryptography~=3.4.8", + "cycler~=0.11.0", + "cython~=0.29.28", + "debugpy~=1.5.1", + "decorator~=5.1.1", + "defusedxml~=0.7.1", + "distlib~=0.3.6", + "docstring-to-markdown~=0.11", + "entrypoints~=0.4", + "executing~=0.8.3", + "facets-overview~=1.0.0", + "fastjsonschema~=2.16.2", + "filelock~=3.9.0", + "fonttools~=4.25.0", + "idna~=3.3", + "ipykernel~=6.15.3", + "ipython~=8.5.0", + "ipython-genutils~=0.2.0", + "ipywidgets~=7.7.2", + "jedi~=0.18.1", + "jinja2~=2.11.3", + "jmespath~=0.10.0", + "joblib~=1.1.1", + "jsonschema~=4.4.0", + "jupyter-client~=6.1.12", + "jupyter-core~=4.11.2", + "jupyterlab-pygments~=0.1.2", + "jupyterlab-widgets~=1.0.0", + "kiwisolver~=1.3.2", + "markupsafe~=2.0.1", + "matplotlib~=3.5.1", + "matplotlib-inline~=0.1.2", + "mccabe~=0.7.0", + "mistune~=0.8.4", + "mypy-extensions~=0.4.3", + "nbclient~=0.5.13", + "nbconvert~=6.4.4", + "nbformat~=5.3.0", + "nest-asyncio~=1.5.5", + "nodeenv~=1.7.0", + "notebook~=6.4.8", + "numpy~=1.21.5", + "packaging~=21.3", + "pandas~=1.4.2", + "pandocfilters~=1.5.0", + "parso~=0.8.3", + "pathspec~=0.9.0", + "patsy~=0.5.2", + "pexpect~=4.8.0", + "pickleshare~=0.7.5", + "pillow~=9.0.1", + "platformdirs~=2.6.2", + "plotly~=5.6.0", + "pluggy~=1.0.0", + "prometheus-client~=0.13.1", + "prompt-toolkit~=3.0.20", + "protobuf~=3.19.4", + "psutil~=5.8.0", + "psycopg2~=2.9.3", + "ptyprocess~=0.7.0", + "pure-eval~=0.2.2", + "pyarrow~=7.0.0", + "pycparser~=2.21", + "pyflakes~=2.5.0", + "pygments~=2.11.2", + "pyodbc~=4.0.32", + "pyparsing~=3.0.4", + "pyright~=1.1.283", + "pyrsistent~=0.18.0", + "python-dateutil~=2.8.2", + "python-lsp-jsonrpc~=1.0.0", + "python-lsp-server~=1.6.0", + "pytz~=2021.3", + "pyzmq~=22.3.0", + "requests~=2.27.1", + "requests-unixsocket~=0.2.0", + "rope~=0.22.0", + "s3transfer~=0.5.0", + "scikit-learn~=1.0.2", + "scipy~=1.7.3", + "seaborn~=0.11.2", + "send2trash~=1.8.0", + "setuptools~=61.2.0", + "six~=1.16.0", + "soupsieve~=2.3.1", + "ssh-import-id~=5.10", + "stack-data~=0.2.0", + "statsmodels~=0.13.2", + "tenacity~=8.0.1", + "terminado~=0.13.1", + "testpath~=0.5.0", + "threadpoolctl~=2.2.0", + "tokenize-rt~=4.2.1", + "tomli~=1.2.2", + "tornado~=6.1", + "traitlets~=5.1.1", + "typing-extensions~=4.1.1", + "ujson~=5.1.0", + "urllib3~=1.26.9", + "virtualenv~=20.8.0", + "wcwidth~=0.2.5", + "webencodings~=0.5.1", + "whatthepatch~=1.0.4", + "wheel~=0.37.0", + "widgetsnbextension~=3.6.1", + "yapf~=0.31.0", +] diff --git a/python/dbr/16.1.x-scala2.12/constraints.txt b/python/dbr/16.1.x-scala2.12/constraints.txt new file mode 100644 index 0000000..5d83a07 --- /dev/null +++ b/python/dbr/16.1.x-scala2.12/constraints.txt @@ -0,0 +1,152 @@ +# constraints.txt file for Databricks Runtime 16.1.x-scala2.12 + +annotated-types~=0.7.0 +asttokens~=2.0.5 +astunparse~=1.6.3 +azure-core~=1.31.0 +azure-storage-blob~=12.23.0 +azure-storage-file-datalake~=12.17.0 +black~=24.4.2 +blinker~=1.7.0 +boto3~=1.34.69 +botocore~=1.34.69 +cachetools~=5.3.3 +certifi~=2024.6.2 +cffi~=1.16.0 +chardet~=4.0.0 +charset-normalizer~=2.0.4 +click~=8.1.7 +cloudpickle~=2.2.1 +comm~=0.2.1 +contourpy~=1.2.0 +cryptography~=42.0.5 +cycler~=0.11.0 +cython~=3.0.11 +databricks-sdk~=0.30 +debugpy~=1.6.7 +decorator~=5.1.1 +deprecated~=1.2.14 +distlib~=0.3.8 +docstring-to-markdown~=0.11 +entrypoints~=0.4 +executing~=0.8.3 +facets-overview~=1.1.1 +filelock~=3.15.4 +fonttools~=4.51.0 +gitdb~=4.0.11 +gitpython~=3.1.37 +google-api-core~=2.20.0 +google-auth~=2.35.0 +google-cloud-core~=2.4.1 +google-cloud-storage~=2.18.2 +google-crc32c~=1.6.0 +google-resumable-media~=2.7.2 +googleapis-common-protos~=1.65.0 +grpcio~=1.60 +grpcio-status~=1.60 +httplib2~=0.20.4 +idna~=3.7 +importlib-metadata~=6.0.0 +ipyflow-core~=0.0.201 +ipykernel~=6.28.0 +ipython~=8.25.0 +ipython-genutils~=0.2.0 +ipywidgets~=7.7.2 +isodate~=0.6.1 +jedi~=0.19.1 +jmespath~=1.0.1 +joblib~=1.4.2 +jupyter-client~=8.6.0 +jupyter-core~=5.7.2 +kiwisolver~=1.4.4 +launchpadlib~=1.11.0 +lazr-restfulclient~=0.14.6 +lazr-uri~=1.0.6 +matplotlib~=3.8.4 +matplotlib-inline~=0.1.6 +mccabe~=0.7.0 +mlflow-skinny~=2.15.1 +mypy~=1.10.0 +mypy-extensions~=1.0.0 +nest-asyncio~=1.6.0 +nodeenv~=1.9.1 +numpy~=1.26.4 +oauthlib~=3.2.2 +opentelemetry-api~=1.27.0 +opentelemetry-sdk~=1.27.0 +opentelemetry-semantic-conventions~=0.48b0 +packaging~=24.1 +parso~=0.8.3 +pathspec~=0.10.3 +patsy~=0.5.6 +pexpect~=4.8.0 +pillow~=10.3.0 +platformdirs~=3.10.0 +plotly~=5.22.0 +pluggy~=1.0.0 +prompt-toolkit~=3.0.43 +proto-plus~=1.24.0 +protobuf~=4.24.1 +psutil~=5.9.0 +psycopg2~=2.9.3 +ptyprocess~=0.7.0 +pure-eval~=0.2.2 +pyarrow~=15.0.2 +pyasn1~=0.4.8 +pyasn1-modules~=0.2.8 +pyccolo~=0.0.65 +pycparser~=2.21 +pydantic~=2.8.2 +pydantic-core~=2.20.1 +pyflakes~=3.2.0 +pygments~=2.15.1 +pyjwt~=2.7.0 +pyodbc~=5.0.1 +pyparsing~=3.0.9 +pyright~=1.1.294 +python-dateutil~=2.9.0.post0 +python-lsp-jsonrpc~=1.1.2 +python-lsp-server~=1.10.0 +pytoolconfig~=1.2.6 +pytz~=2024.1 +pyyaml~=6.0.1 +pyzmq~=25.1.2 +requests~=2.32.2 +rope~=1.12.0 +rsa~=4.9 +s3transfer~=0.10.2 +scikit-learn~=1.4.2 +scipy~=1.13.1 +seaborn~=0.13.2 +setuptools~=74.0.0 +six~=1.16.0 +smmap~=5.0.0 +sqlparse~=0.5.1 +ssh-import-id~=5.11 +stack-data~=0.2.0 +statsmodels~=0.14.2 +tenacity~=8.2.2 +threadpoolctl~=2.2.0 +tokenize-rt~=4.2.1 +tomli~=2.0.1 +tornado~=6.4.1 +traitlets~=5.14.3 +types-protobuf~=3.20.3 +types-psutil~=5.9.0 +types-pytz~=2023.3.1.1 +types-pyyaml~=6.0.0 +types-requests~=2.31.0.0 +types-setuptools~=68.0.0.0 +types-six~=1.16.0 +types-urllib3~=1.26.25.14 +typing-extensions~=4.11.0 +ujson~=5.10.0 +urllib3~=1.26.16 +virtualenv~=20.26.2 +wadllib~=1.3.6 +wcwidth~=0.2.5 +whatthepatch~=1.0.2 +wheel~=0.43.0 +wrapt~=1.14.1 +yapf~=0.33.0 +zipp~=3.17.0 diff --git a/python/dbr/16.1.x-scala2.12/pyproject.toml b/python/dbr/16.1.x-scala2.12/pyproject.toml new file mode 100644 index 0000000..fb95634 --- /dev/null +++ b/python/dbr/16.1.x-scala2.12/pyproject.toml @@ -0,0 +1,165 @@ +# pyproject.toml file for Databricks Runtime 16.1.x-scala2.12 + +[project] +name = "constraint-env-16-1-x-scala2-12" +version = "0.1.0" +requires-python = "==3.12.*" + +[dependency-groups] +dev = [ + "databricks-connect~=16.1.0", +] + +[tool.uv] +constraint-dependencies = [ + "annotated-types~=0.7.0", + "asttokens~=2.0.5", + "astunparse~=1.6.3", + "azure-core~=1.31.0", + "azure-storage-blob~=12.23.0", + "azure-storage-file-datalake~=12.17.0", + "black~=24.4.2", + "blinker~=1.7.0", + "boto3~=1.34.69", + "botocore~=1.34.69", + "cachetools~=5.3.3", + "certifi~=2024.6.2", + "cffi~=1.16.0", + "chardet~=4.0.0", + "charset-normalizer~=2.0.4", + "click~=8.1.7", + "cloudpickle~=2.2.1", + "comm~=0.2.1", + "contourpy~=1.2.0", + "cryptography~=42.0.5", + "cycler~=0.11.0", + "cython~=3.0.11", + "databricks-sdk~=0.30", + "debugpy~=1.6.7", + "decorator~=5.1.1", + "deprecated~=1.2.14", + "distlib~=0.3.8", + "docstring-to-markdown~=0.11", + "entrypoints~=0.4", + "executing~=0.8.3", + "facets-overview~=1.1.1", + "filelock~=3.15.4", + "fonttools~=4.51.0", + "gitdb~=4.0.11", + "gitpython~=3.1.37", + "google-api-core~=2.20.0", + "google-auth~=2.35.0", + "google-cloud-core~=2.4.1", + "google-cloud-storage~=2.18.2", + "google-crc32c~=1.6.0", + "google-resumable-media~=2.7.2", + "googleapis-common-protos~=1.65.0", + "grpcio~=1.60", + "grpcio-status~=1.60", + "httplib2~=0.20.4", + "idna~=3.7", + "importlib-metadata~=6.0.0", + "ipyflow-core~=0.0.201", + "ipykernel~=6.28.0", + "ipython~=8.25.0", + "ipython-genutils~=0.2.0", + "ipywidgets~=7.7.2", + "isodate~=0.6.1", + "jedi~=0.19.1", + "jmespath~=1.0.1", + "joblib~=1.4.2", + "jupyter-client~=8.6.0", + "jupyter-core~=5.7.2", + "kiwisolver~=1.4.4", + "launchpadlib~=1.11.0", + "lazr-restfulclient~=0.14.6", + "lazr-uri~=1.0.6", + "matplotlib~=3.8.4", + "matplotlib-inline~=0.1.6", + "mccabe~=0.7.0", + "mlflow-skinny~=2.15.1", + "mypy~=1.10.0", + "mypy-extensions~=1.0.0", + "nest-asyncio~=1.6.0", + "nodeenv~=1.9.1", + "numpy~=1.26.4", + "oauthlib~=3.2.2", + "opentelemetry-api~=1.27.0", + "opentelemetry-sdk~=1.27.0", + "opentelemetry-semantic-conventions~=0.48b0", + "packaging~=24.1", + "parso~=0.8.3", + "pathspec~=0.10.3", + "patsy~=0.5.6", + "pexpect~=4.8.0", + "pillow~=10.3.0", + "platformdirs~=3.10.0", + "plotly~=5.22.0", + "pluggy~=1.0.0", + "prompt-toolkit~=3.0.43", + "proto-plus~=1.24.0", + "protobuf~=4.24.1", + "psutil~=5.9.0", + "psycopg2~=2.9.3", + "ptyprocess~=0.7.0", + "pure-eval~=0.2.2", + "pyarrow~=15.0.2", + "pyasn1~=0.4.8", + "pyasn1-modules~=0.2.8", + "pyccolo~=0.0.65", + "pycparser~=2.21", + "pydantic~=2.8.2", + "pydantic-core~=2.20.1", + "pyflakes~=3.2.0", + "pygments~=2.15.1", + "pyjwt~=2.7.0", + "pyodbc~=5.0.1", + "pyparsing~=3.0.9", + "pyright~=1.1.294", + "python-dateutil~=2.9.0.post0", + "python-lsp-jsonrpc~=1.1.2", + "python-lsp-server~=1.10.0", + "pytoolconfig~=1.2.6", + "pytz~=2024.1", + "pyyaml~=6.0.1", + "pyzmq~=25.1.2", + "requests~=2.32.2", + "rope~=1.12.0", + "rsa~=4.9", + "s3transfer~=0.10.2", + "scikit-learn~=1.4.2", + "scipy~=1.13.1", + "seaborn~=0.13.2", + "setuptools~=74.0.0", + "six~=1.16.0", + "smmap~=5.0.0", + "sqlparse~=0.5.1", + "ssh-import-id~=5.11", + "stack-data~=0.2.0", + "statsmodels~=0.14.2", + "tenacity~=8.2.2", + "threadpoolctl~=2.2.0", + "tokenize-rt~=4.2.1", + "tomli~=2.0.1", + "tornado~=6.4.1", + "traitlets~=5.14.3", + "types-protobuf~=3.20.3", + "types-psutil~=5.9.0", + "types-pytz~=2023.3.1.1", + "types-pyyaml~=6.0.0", + "types-requests~=2.31.0.0", + "types-setuptools~=68.0.0.0", + "types-six~=1.16.0", + "types-urllib3~=1.26.25.14", + "typing-extensions~=4.11.0", + "ujson~=5.10.0", + "urllib3~=1.26.16", + "virtualenv~=20.26.2", + "wadllib~=1.3.6", + "wcwidth~=0.2.5", + "whatthepatch~=1.0.2", + "wheel~=0.43.0", + "wrapt~=1.14.1", + "yapf~=0.33.0", + "zipp~=3.17.0", +] From c7a4fdd289db28da4d773ef8c1cc6bc159b9e5c1 Mon Sep 17 00:00:00 2001 From: Grigory Panov Date: Mon, 31 Aug 2026 12:09:43 +0200 Subject: [PATCH 2/2] Harden EoS publishing: reject bare-major slugs; document coverage rules MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review round (Isaac) surfaced two latent gaps in sync_dbr_eos, neither triggered by today's allowlist: - A bare-major 18+ slug would emit a .x umbrella folder colliding with the live umbrella sync_dbr builds. Reject such slugs (via _umbrella_major) with a clear message; specific EoS point-release slugs still publish normally. - The per-Scala DROP_BY_ENV coupling (a dual-Scala py3.12 EoS line would emit an undropped scala2.13 folder) is a manual-maintenance property shared with the existing 16.4 handling and caught by verify_resolve on add — documented in the DBR_EOS_PUBLISH comment rather than duplicated in code. Also update README: the "EoS point releases are skipped" note now records the DBR_EOS_PUBLISH exception. Generated artifacts unchanged; 46 unit tests pass. Co-authored-by: Isaac --- .github/scripts/sync.py | 15 ++++++++++++++- .github/scripts/test_sync.py | 15 +++++++++++++++ README.md | 4 +++- 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/.github/scripts/sync.py b/.github/scripts/sync.py index 8fc7a87..2010c69 100755 --- a/.github/scripts/sync.py +++ b/.github/scripts/sync.py @@ -62,6 +62,10 @@ # lines with real traffic. Add a slug ONLY when telemetry shows its exact env_key failing: # 12.2 -> dbr/12.2.x-scala2.12 (8 events / 2 workspaces) # 16.1 -> dbr/16.1.x-scala2.12 (1 event / 1 workspace) +# Each slug must be a pre-18 point-release page (the minor lives in its title, e.g. '12.2'); +# a bare-major 18+ slug is rejected by sync_dbr_eos. If a listed page ships two Scala images +# and is Python 3.12 on pandas 1.5.x, also add its scala2.13 env to envgen.DROP_BY_ENV (as +# 16.4 does) — verify_resolve.py flags an uncovered folder that then can't uv sync. DBR_EOS_PUBLISH = ["12.2", "16.1"] @@ -504,8 +508,17 @@ def sync_dbr_eos(): are fetched directly here rather than discovered. Each is a self-contained runtime page with the real minor in its title (pre-18-style), so it publishes only its own '.x' folder(s) and no bare-major umbrella. They can't collide with the - index-driven sync_dbr (that path never yields an EoS slug).""" + index-driven sync_dbr (that path never yields an EoS slug). + + A bare-major 18+ slug (e.g. '18') is rejected: it has no minor in its title, so this + point-release path would emit a '.x' umbrella folder that collides with the live + umbrella sync_dbr builds for that line. A specific EoS point release (even '18.5') is + fine — only the bare-major umbrella form is unsupported here.""" for slug in DBR_EOS_PUBLISH: + if _umbrella_major(slug): + print(f" ! dbr-eos [{slug}]: bare-major (18+) slug unsupported by the EoS path " + f"(would emit a colliding umbrella folder); skipping") + continue _sync_dbr_page(slug, point_release=True, umbrella_major=None) diff --git a/.github/scripts/test_sync.py b/.github/scripts/test_sync.py index 4ab663a..5615c7c 100644 --- a/.github/scripts/test_sync.py +++ b/.github/scripts/test_sync.py @@ -435,6 +435,21 @@ def test_publishes_allowlisted_eos_line_despite_eos_marker(self): sync.sync_dbr_eos() self.assertEqual(writes, [("16.1.x-scala2.12", "16.1")]) + def test_rejects_a_bare_major_slug_this_path_does_not_support(self): + # sync_dbr_eos publishes pre-18 point-release lines (the minor lives in the page + # title). A bare-major 18+ slug would emit a '.x' umbrella folder that + # collides with the live umbrella sync_dbr builds for that line, so it is rejected + # here rather than mispublished. A specific EoS point-release slug (even 18.5) is + # still fine — only the bare-major umbrella form is blocked. + pages = {"18": _runtime_page("Databricks Runtime 18 LTS (EoS)", scala="2.13")} + writes = [] + with mock.patch.object(sync, "DBR_EOS_PUBLISH", ["18"]), \ + mock.patch.object(sync, "fetch", _fake_fetch(pages)), \ + mock.patch.object(sync, "_write_env", + lambda key, pkgs, pv, dbconnect: writes.append(key)): + sync.sync_dbr_eos() + self.assertEqual(writes, []) + def test_skips_an_allowlisted_slug_whose_page_is_gone(self): # An EoS page can eventually be removed from the docs; a fetch failure on one # allowlisted slug is logged and skipped without aborting the rest of the run. diff --git a/README.md b/README.md index 204040d..7b2f4ef 100644 --- a/README.md +++ b/README.md @@ -119,7 +119,9 @@ is best-effort. Nobody hand-edits the `python/` artifacts. point-release folder pins `databricks-connect` to its exact minor (`18.2.x` → `~=18.2.0`), but the umbrella tracks the whole major line like a serverless major (`18.x` → `~=18.0`), so a cluster addressing the line by its bare major resolves the newest `databricks-connect` - in the major. EoS point releases are skipped. Pre-18 lines keep the old + in the major. EoS point releases are skipped, except a few pre-18 lines still requested by + live clusters, which are published from an explicit telemetry-curated allowlist + (`DBR_EOS_PUBLISH` in `sync.py`). Pre-18 lines keep the old scheme — one folder, keyed by the minor read from the page title. A release that ships two Scala images from one page (e.g. DBR 16.4 LTS — `Scala: 2.12.15 or 2.13.10`) yields one environment per