Publish end-of-support DBR runtimes still requested by clusters (12.2, 16.1)

.github/scripts/envgen.py

@@ -65,8 +65,8 @@
 # Environment-scoped drops: removed for the named environments only (DROP is global).
 # pandas 1.5.3 has no cp312 wheel — pandas ships CPython 3.12 wheels only from 2.1.1 —
 # so `uv sync` on a Python 3.12 runtime can't install it and falls back to a failing
-# source build. DBR 16.4 and serverless-v3 are the only 3.12 runtimes still pinned to
-# pandas 1.5.x: 13.3/14.3/15.4 predate 3.12, and 17.3+ / serverless-v4+ already ship
+# source build. DBR 16.1, 16.4 and serverless-v3 are the only 3.12 runtimes still pinned
+# to pandas 1.5.x: 13.3/14.3/15.4 predate 3.12, and 17.3+ / serverless-v4+ already ship
 # pandas 2.x (which has cp312 wheels). No in-range (~=1.5) version has a cp312 wheel,
 # so the pin can't be salvaged by widening — it is dropped for just these envs, and
 # pandas resolves to an installable version locally. See issue #18.
@@ -75,6 +75,9 @@
 # and self-expiring: if one of these runtimes is ever re-pinned to a cp312-wheel 2.x,
 # the version no longer matches and the pin is kept rather than silently discarded.
 DROP_BY_ENV = {
+    # DBR 16.1 is an EoS line published from sync.DBR_EOS_PUBLISH; it too is Python 3.12
+    # on pandas 1.5.3, so it carries the same drop.
+    "16.1.x-scala2.12": {"pandas": "1.5."},
     "16.4.x-scala2.12": {"pandas": "1.5."},
     "16.4.x-cpu-ml-scala2.12": {"pandas": "1.5."},
     "16.4.x-gpu-ml-scala2.12": {"pandas": "1.5."},

.github/scripts/sync.py

@@ -53,6 +53,21 @@
 # Index entries that aren't a runtime version page.
 DBR_NON_VERSION = {"maintenance-updates", "databricks-runtime-ver", "eos"}
 
+# End-of-support runtime lines to publish anyway, as page slugs. EoS lines are dropped
+# from the release-notes index (so discover_dbr never sees them) and skipped by
+# dbr_point_releases' EoS check, and the 'eos' page links none — so they can't be
+# discovered, only listed explicitly. Live clusters still request a few of them, and VPEX
+# telemetry then records E_ENV_UNSUPPORTED (an owned target with no environment published),
+# so we make a deliberate, narrow exception to the EoS-not-published policy for exactly the
+# lines with real traffic. Add a slug ONLY when telemetry shows its exact env_key failing:
+#   12.2 -> dbr/12.2.x-scala2.12   (8 events / 2 workspaces)
+#   16.1 -> dbr/16.1.x-scala2.12   (1 event  / 1 workspace)
+# Each slug must be a pre-18 point-release page (the minor lives in its title, e.g. '12.2');
+# a bare-major 18+ slug is rejected by sync_dbr_eos. If a listed page ships two Scala images
+# and is Python 3.12 on pandas 1.5.x, also add its scala2.13 env to envgen.DROP_BY_ENV (as
+# 16.4 does) — verify_resolve.py flags an uncovered folder that then can't uv sync.
+DBR_EOS_PUBLISH = ["12.2", "16.1"]
+
 
 def fetch(url):
     req = urllib.request.Request(url, headers={"User-Agent": "databricks-environments-sync"})
@@ -440,40 +455,71 @@ def _write_env(key, pkgs, python_version, dbconnect):
     print(f"  + dbr/{key} (python {python_version}, {len(pkgs)} packages)")
 
 
+def _sync_dbr_page(slug, point_release, umbrella_major):
+    """Fetch one standard runtime page and write its folder(s).
+
+    A live point release publishes its own '<key_ver>.x' folder (pre-18: the whole line;
+    EoS allowlist lines: their own line). The newest live release of a DBR 18+ line ALSO
+    publishes the bare-major '<major>.x' umbrella that a cluster's spark_version resolves
+    to — in ADDITION to its own folder, so the umbrella page (18.2) yields both '18.2.x'
+    and '18.x' (``umbrella_major`` set). A point-release folder pins databricks-connect to
+    its exact minor (18.2 -> ~=18.2.0); the bare-major umbrella instead pins the whole major
+    line (18 -> ~=18.0), so a cluster addressing the line by its bare major resolves the
+    newest databricks-connect in the major."""
+    try:
+        html = fetch(DBR_PAGE.format(slug=slug))
+    except Exception as e:
+        print(f"  ! dbr [{slug}]: fetch failed ({e}); skipping")
+        return
+    meta = dbr_meta(html)
+    pkgs, _ = parse_dbr_page(html)
+    if not meta or not pkgs:
+        print(f"  ! dbr [{slug}]: no meta / Python table; skipping")
+        return
+    key_ver, dbconnect_ver, scalas, python_version = meta
+    folder_vers = ([key_ver] if point_release else []) + ([umbrella_major] if umbrella_major else [])
+    for folder_ver in folder_vers:
+        # The bare-major umbrella tracks the whole major line (18 -> ~=18.0), like a
+        # serverless major; a point release (and every EoS allowlist line) keeps its exact
+        # minor (18.2 -> ~=18.2.0). See the docstring.
+        dbconnect = umbrella_major if folder_ver == umbrella_major else dbconnect_ver
+        for scala in scalas:
+            _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect)
+
+
 def sync_dbr():
     for index_slug in discover_dbr():
         major = _umbrella_major(index_slug)
         pointrelease_slugs, umbrella_slug = dbr_point_releases(index_slug, ml=False)
         # The umbrella slug is usually also one of the point releases — fetch each page once.
         for slug in dict.fromkeys(pointrelease_slugs + ([umbrella_slug] if umbrella_slug else [])):
-            try:
-                html = fetch(DBR_PAGE.format(slug=slug))
-            except Exception as e:
-                print(f"  ! dbr [{slug}]: fetch failed ({e}); skipping")
-                continue
-            meta = dbr_meta(html)
-            pkgs, _ = parse_dbr_page(html)
-            if not meta or not pkgs:
-                print(f"  ! dbr [{slug}]: no meta / Python table; skipping")
-                continue
-            key_ver, dbconnect_ver, scalas, python_version = meta
-            # A live point release publishes its own '<minor>.x' folder (pre-18: the whole
-            # line). The newest live one ALSO publishes the bare-major '<major>.x' umbrella
-            # that a DBR 18+ cluster's spark_version resolves to — in ADDITION to its own
-            # folder, so the umbrella page (18.2) yields both '18.2.x' and '18.x'. Both use
-            # this page's dbconnect_ver (the latest point release's databricks-connect minor).
-            folder_vers = [key_ver] if slug in pointrelease_slugs else []
-            if major and slug == umbrella_slug:
-                folder_vers.append(major)
-            for folder_ver in folder_vers:
-                # A point-release folder pins its exact minor (18.2 -> ~=18.2.0). The bare-
-                # major umbrella tracks the whole major line, like a serverless major, so it
-                # pins ~=MAJOR.0 (18 -> ~=18.0): a cluster addressing the line by its bare
-                # major resolves the latest databricks-connect in the major, and a new point
-                # release doesn't need a regen to be covered.
-                dbconnect = major if folder_ver == major else dbconnect_ver
-                for scala in scalas:
-                    _write_env(f"{folder_ver}.x-scala{scala}", pkgs, python_version, dbconnect)
+            _sync_dbr_page(
+                slug,
+                point_release=slug in pointrelease_slugs,
+                umbrella_major=major if slug == umbrella_slug else None,
+            )
+
+
+def sync_dbr_eos():
+    """Publish the end-of-support lines in DBR_EOS_PUBLISH (see its comment for why they
+    exist and the telemetry rule for adding one).
+
+    These are dropped from the index and skipped by dbr_point_releases' EoS check, so they
+    are fetched directly here rather than discovered. Each is a self-contained runtime page
+    with the real minor in its title (pre-18-style), so it publishes only its own
+    '<key_ver>.x' folder(s) and no bare-major umbrella. They can't collide with the
+    index-driven sync_dbr (that path never yields an EoS slug).
+
+    A bare-major 18+ slug (e.g. '18') is rejected: it has no minor in its title, so this
+    point-release path would emit a '<major>.x' umbrella folder that collides with the live
+    umbrella sync_dbr builds for that line. A specific EoS point release (even '18.5') is
+    fine — only the bare-major umbrella form is unsupported here."""
+    for slug in DBR_EOS_PUBLISH:
+        if _umbrella_major(slug):
+            print(f"  ! dbr-eos [{slug}]: bare-major (18+) slug unsupported by the EoS path "
+                  f"(would emit a colliding umbrella folder); skipping")
+            continue
+        _sync_dbr_page(slug, point_release=True, umbrella_major=None)
 
 
 def ml_variant_pkgs(ml_html, variant):
@@ -611,6 +657,8 @@ def main():
     sync_serverless()
     print("Syncing DBR runtimes from docs.databricks.com ...")
     sync_dbr()
+    print("Publishing end-of-support DBR runtimes still requested by clusters ...")
+    sync_dbr_eos()
     print("Syncing DBR ML runtimes (CPU + GPU) from docs.databricks.com ...")
     sync_dbr_ml()
     changed = reconcile()

.github/scripts/test_envgen.py

@@ -43,12 +43,15 @@ def test_keeps_installable_pins(self):
 
 class EnvScopedDropTest(unittest.TestCase):
     def test_pandas_dropped_on_py312_1_5_runtimes(self):
-        # DBR 16.4 + serverless-v3 are the only Python-3.12 runtimes still on
+        # DBR 16.1, 16.4 + serverless-v3 are the only Python-3.12 runtimes still on
         # pandas 1.5.x, which has no cp312 wheel — dropped for just these envs. DBR 16.4
         # ships both Scala images off one page (same Python packages), so the scala2.13
-        # variants carry the identical pandas problem and are dropped the same way.
+        # variants carry the identical pandas problem and are dropped the same way. DBR
+        # 16.1 is an EoS line published from the telemetry allowlist and is Python 3.12 on
+        # pandas 1.5.3 too, so it needs the same drop.
         pkgs = {"pandas": "1.5.3", "numpy": "2.1.3"}
         for env in (
+            "16.1.x-scala2.12",
             "16.4.x-scala2.12",
             "16.4.x-cpu-ml-scala2.12",
             "16.4.x-gpu-ml-scala2.12",

.github/scripts/test_sync.py

@@ -420,5 +420,50 @@ def test_umbrella_ml_line_publishes_point_releases_and_umbrella(self):
         )
 
 
+class SyncDbrEosTest(unittest.TestCase):
+    def test_publishes_allowlisted_eos_line_despite_eos_marker(self):
+        # A line on the EoS allowlist is published from its own page even though its title
+        # carries '(EoS)' — the reason the allowlist exists (these lines are dropped from
+        # the index and skipped by dbr_point_releases' EoS check). Like a pre-18 line, it
+        # writes only its own '<key_ver>.x' folder and no bare-major umbrella.
+        pages = {"16.1": _runtime_page("Databricks Runtime 16.1 (EoS)", scala="2.12")}
+        writes = []
+        with mock.patch.object(sync, "DBR_EOS_PUBLISH", ["16.1"]), \
+                mock.patch.object(sync, "fetch", _fake_fetch(pages)), \
+                mock.patch.object(sync, "_write_env",
+                                  lambda key, pkgs, pv, dbconnect: writes.append((key, dbconnect))):
+            sync.sync_dbr_eos()
+        self.assertEqual(writes, [("16.1.x-scala2.12", "16.1")])
+
+    def test_rejects_a_bare_major_slug_this_path_does_not_support(self):
+        # sync_dbr_eos publishes pre-18 point-release lines (the minor lives in the page
+        # title). A bare-major 18+ slug would emit a '<major>.x' umbrella folder that
+        # collides with the live umbrella sync_dbr builds for that line, so it is rejected
+        # here rather than mispublished. A specific EoS point-release slug (even 18.5) is
+        # still fine — only the bare-major umbrella form is blocked.
+        pages = {"18": _runtime_page("Databricks Runtime 18 LTS (EoS)", scala="2.13")}
+        writes = []
+        with mock.patch.object(sync, "DBR_EOS_PUBLISH", ["18"]), \
+                mock.patch.object(sync, "fetch", _fake_fetch(pages)), \
+                mock.patch.object(sync, "_write_env",
+                                  lambda key, pkgs, pv, dbconnect: writes.append(key)):
+            sync.sync_dbr_eos()
+        self.assertEqual(writes, [])
+
+    def test_skips_an_allowlisted_slug_whose_page_is_gone(self):
+        # An EoS page can eventually be removed from the docs; a fetch failure on one
+        # allowlisted slug is logged and skipped without aborting the rest of the run.
+        def boom(url):
+            raise Exception("HTTP 404")
+
+        writes = []
+        with mock.patch.object(sync, "DBR_EOS_PUBLISH", ["12.2"]), \
+                mock.patch.object(sync, "fetch", boom), \
+                mock.patch.object(sync, "_write_env",
+                                  lambda *a, **k: writes.append(a)):
+            sync.sync_dbr_eos()
+        self.assertEqual(writes, [])
+
+
 if __name__ == "__main__":
     unittest.main()

README.md

@@ -119,7 +119,9 @@ is best-effort. Nobody hand-edits the `python/` artifacts.
   point-release folder pins `databricks-connect` to its exact minor (`18.2.x` → `~=18.2.0`),
   but the umbrella tracks the whole major line like a serverless major (`18.x` → `~=18.0`),
   so a cluster addressing the line by its bare major resolves the newest `databricks-connect`
-  in the major. EoS point releases are skipped. Pre-18 lines keep the old
+  in the major. EoS point releases are skipped, except a few pre-18 lines still requested by
+  live clusters, which are published from an explicit telemetry-curated allowlist
+  (`DBR_EOS_PUBLISH` in `sync.py`). Pre-18 lines keep the old
   scheme — one folder, keyed by the minor read from the page title. A release that
   ships two Scala images from
   one page (e.g. DBR 16.4 LTS — `Scala: 2.12.15 or 2.13.10`) yields one environment per

python/dbr/12.2.x-scala2.12/constraints.txt

@@ -0,0 +1,124 @@
+# constraints.txt file for Databricks Runtime 12.2.x-scala2.12
+
+argon2-cffi~=21.3.0
+argon2-cffi-bindings~=21.2.0
+asttokens~=2.0.5
+attrs~=21.4.0
+backcall~=0.2.0
+backports-entry-points-selectable~=1.2.0
+beautifulsoup4~=4.11.1
+black~=22.3.0
+bleach~=4.1.0
+boto3~=1.21.32
+botocore~=1.24.32
+certifi~=2021.10.8
+cffi~=1.15.0
+chardet~=4.0.0
+charset-normalizer~=2.0.4
+click~=8.0.4
+cryptography~=3.4.8
+cycler~=0.11.0
+cython~=0.29.28
+debugpy~=1.5.1
+decorator~=5.1.1
+defusedxml~=0.7.1
+distlib~=0.3.6
+docstring-to-markdown~=0.11
+entrypoints~=0.4
+executing~=0.8.3
+facets-overview~=1.0.0
+fastjsonschema~=2.16.2
+filelock~=3.9.0
+fonttools~=4.25.0
+idna~=3.3
+ipykernel~=6.15.3
+ipython~=8.5.0
+ipython-genutils~=0.2.0
+ipywidgets~=7.7.2
+jedi~=0.18.1
+jinja2~=2.11.3
+jmespath~=0.10.0
+joblib~=1.1.1
+jsonschema~=4.4.0
+jupyter-client~=6.1.12
+jupyter-core~=4.11.2
+jupyterlab-pygments~=0.1.2
+jupyterlab-widgets~=1.0.0
+kiwisolver~=1.3.2
+markupsafe~=2.0.1
+matplotlib~=3.5.1
+matplotlib-inline~=0.1.2
+mccabe~=0.7.0
+mistune~=0.8.4
+mypy-extensions~=0.4.3
+nbclient~=0.5.13
+nbconvert~=6.4.4
+nbformat~=5.3.0
+nest-asyncio~=1.5.5
+nodeenv~=1.7.0
+notebook~=6.4.8
+numpy~=1.21.5
+packaging~=21.3
+pandas~=1.4.2
+pandocfilters~=1.5.0
+parso~=0.8.3
+pathspec~=0.9.0
+patsy~=0.5.2
+pexpect~=4.8.0
+pickleshare~=0.7.5
+pillow~=9.0.1
+platformdirs~=2.6.2
+plotly~=5.6.0
+pluggy~=1.0.0
+prometheus-client~=0.13.1
+prompt-toolkit~=3.0.20
+protobuf~=3.19.4
+psutil~=5.8.0
+psycopg2~=2.9.3
+ptyprocess~=0.7.0
+pure-eval~=0.2.2
+pyarrow~=7.0.0
+pycparser~=2.21
+pyflakes~=2.5.0
+pygments~=2.11.2
+pyodbc~=4.0.32
+pyparsing~=3.0.4
+pyright~=1.1.283
+pyrsistent~=0.18.0
+python-dateutil~=2.8.2
+python-lsp-jsonrpc~=1.0.0
+python-lsp-server~=1.6.0
+pytz~=2021.3
+pyzmq~=22.3.0
+requests~=2.27.1
+requests-unixsocket~=0.2.0
+rope~=0.22.0
+s3transfer~=0.5.0
+scikit-learn~=1.0.2
+scipy~=1.7.3
+seaborn~=0.11.2
+send2trash~=1.8.0
+setuptools~=61.2.0
+six~=1.16.0
+soupsieve~=2.3.1
+ssh-import-id~=5.10
+stack-data~=0.2.0
+statsmodels~=0.13.2
+tenacity~=8.0.1
+terminado~=0.13.1
+testpath~=0.5.0
+threadpoolctl~=2.2.0
+tokenize-rt~=4.2.1
+tomli~=1.2.2
+tornado~=6.1
+traitlets~=5.1.1
+typing-extensions~=4.1.1
+ujson~=5.1.0
+urllib3~=1.26.9
+virtualenv~=20.8.0
+wcwidth~=0.2.5
+webencodings~=0.5.1
+whatthepatch~=1.0.4
+wheel~=0.37.0
+widgetsnbextension~=3.6.1
+yapf~=0.31.0