From 0b71cea9aa4af89b8d2466c954a7174a49434c70 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Wed, 18 Nov 2026 12:22:56 +0100 Subject: [PATCH 1/5] feat(docker): support alternate authenticated JDK downloads Port alternate Java download sources and optional BuildKit-secret authentication from adobe/koperator#403. Upgrade the ZooKeeper base image to Debian Trixie while retaining Oracle and Java 25 defaults. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit 9d3330b1f153965a800a11150b62db21b177eeae) --- .github/workflows/release.yml | 7 + docker/zookeeper-image/Dockerfile | 44 ++++-- docker/zookeeper-image/README.md | 48 +++++++ scripts/test_zookeeper_version.py | 221 ++++++++++++++++++++++++++++++ 4 files changed, 306 insertions(+), 14 deletions(-) create mode 100644 docker/zookeeper-image/README.md create mode 100644 scripts/test_zookeeper_version.py diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c6712efde..dafefb97b 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -98,6 +98,13 @@ jobs: uses: docker/build-push-action@v6 with: context: docker/zookeeper-image + # Enable these together when selecting an alternate JDK release: + # secrets: | + # JAVA_DOWNLOAD_APIKEY=${{ secrets.JAVA_DOWNLOAD_APIKEY }} + # Add these overrides under build-args: + # java_version=25 + # java_download_base_url=${{ secrets.JAVA_DOWNLOAD_BASE_URL }} + # java_download_version=25.0.1.1 platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta-zk-apache.outputs.tags }} labels: ${{ steps.meta-zk-apache.outputs.labels }} diff --git a/docker/zookeeper-image/Dockerfile b/docker/zookeeper-image/Dockerfile index 5c098232a..953424f66 100644 --- a/docker/zookeeper-image/Dockerfile +++ b/docker/zookeeper-image/Dockerfile @@ -1,5 +1,11 @@ +# syntax=docker/dockerfile:1 + # backported from https://github.com/docker-library/openjdk/blob/4c2b1a57869e24c5fcfbea6a17e558c710d4f6bd/21/jdk/slim-bullseye/Dockerfile -FROM debian:bullseye-slim +FROM debian:trixie-slim + +ARG java_version=25 +ARG java_download_version=${java_version} +ARG java_download_base_url=https://download.oracle.com/java RUN set -eux; \ apt-get update; \ @@ -8,24 +14,30 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV JAVA_HOME /usr/local/openjdk-21 -ENV PATH $JAVA_HOME/bin:$PATH +ENV JAVA_HOME=/usr/local/openjdk-${java_version} +ENV PATH=$JAVA_HOME/bin:$PATH # Default to UTF-8 file.encoding -ENV LANG C.UTF-8 -ENV JAVA_VERSION 21 +ENV LANG=C.UTF-8 +ENV JAVA_VERSION=${java_version} -RUN set -eux; \ +RUN --mount=type=secret,id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY \ + set -eux; \ + \ + case "$java_download_base_url" in \ + https://download.oracle.com/*) java_download_base_url="${java_download_base_url}/${java_version}/latest" ;; \ + *) java_download_base_url="${java_download_base_url}/${java_version}/${java_download_version}" ;; \ + esac; \ \ arch="$(dpkg --print-architecture)"; \ case "$arch" in \ 'amd64') \ - downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz'; \ - downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz.sha256'; \ + downloadUrl="${java_download_base_url}/jdk-${java_download_version}_linux-x64_bin.tar.gz"; \ + downloadSha256="${downloadUrl}.sha256"; \ ;; \ 'arm64') \ - downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz'; \ - downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz.sha256'; \ + downloadUrl="${java_download_base_url}/jdk-${java_download_version}_linux-aarch64_bin.tar.gz"; \ + downloadSha256="${downloadUrl}.sha256"; \ ;; \ *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ esac; \ @@ -33,12 +45,16 @@ RUN set -eux; \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ apt-get install -y --no-install-recommends \ - wget \ + curl \ ; \ rm -rf /var/lib/apt/lists/*; \ \ - wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \ - SHA256=$(wget -qO- $downloadSha256) ; \ + set --; \ + if [ -n "${JAVA_DOWNLOAD_APIKEY:+set}" ]; then \ + set -- --variable '%JAVA_DOWNLOAD_APIKEY' --expand-header 'Authorization: Bearer {{JAVA_DOWNLOAD_APIKEY}}'; \ + fi; \ + curl --fail --location "$@" --output openjdk.tgz "$downloadUrl"; \ + SHA256="$(curl --fail --silent --show-error --location "$@" "$downloadSha256")"; \ echo "$SHA256 *openjdk.tgz" | sha256sum --strict --check -; \ \ mkdir -p "$JAVA_HOME"; \ @@ -160,7 +176,7 @@ ENV PATH=$PATH:/$DISTRO_NAME/bin \ # OCI standard labels LABEL org.opencontainers.image.title="Apache Zookeeper" -LABEL org.opencontainers.image.description="Apache Zookeeper 3.8.4 with OpenJDK 21" +LABEL org.opencontainers.image.description="Apache Zookeeper ${ZK_VERSION} with OpenJDK ${java_version}" LABEL org.opencontainers.image.source="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.url="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.documentation="https://github.com/adobe/zookeeper-operator" diff --git a/docker/zookeeper-image/README.md b/docker/zookeeper-image/README.md new file mode 100644 index 000000000..b0b8b88e6 --- /dev/null +++ b/docker/zookeeper-image/README.md @@ -0,0 +1,48 @@ +# Apache ZooKeeper base image + +Build from the repository root: + +```bash +docker buildx build \ + --build-arg ZK_VERSION="$(./scripts/zookeeper-version.sh)" \ + -t zookeeper-apache:local docker/zookeeper-image +``` + +The image uses Debian Trixie and Java 25. Oracle's public JDK download URL is +the default, without authentication. + +## Alternate JDK downloads + +Override `java_download_base_url` and `java_download_version` to use another +source. The Dockerfile appends `java_version` (default: `25`) and `latest` +for Oracle, or `java_version` and `java_download_version` for alternate +sources. Neither version belongs in the base URL. Alternate sources must +provide Oracle-style archive filenames and matching `.sha256` files for +each architecture (`x64` and `aarch64`). + +For example, add these options to the build command: + +```bash +--build-arg java_download_base_url=https://jdk.example.com/java \ +--build-arg java_download_version=25.0.1.1 +``` + +This requests `/java/25/25.0.1.1/jdk-25.0.1.1_linux-_bin.tar.gz` +and its `.sha256` file. Both downloads must succeed and the checksum must match. + +For authenticated downloads, pass an optional API key as a BuildKit secret: + +```bash +--secret id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY +``` + +The key is sent as an `Authorization: Bearer` header, is not stored in the +image, and is not forwarded to a different host on redirects. + +Release builds also default to Oracle. To use an alternate source, add the +commented overrides in `.github/workflows/release.yml` to the Apache ZooKeeper +build step's `build-args`, configure the GitHub Actions secret +`JAVA_DOWNLOAD_BASE_URL`, and select a matching `java_download_version`. +For authenticated downloads, also enable the commented `secrets` input and +configure `JAVA_DOWNLOAD_APIKEY`. PR image builds continue to use Oracle +without secrets. diff --git a/scripts/test_zookeeper_version.py b/scripts/test_zookeeper_version.py new file mode 100644 index 000000000..c79a59d9c --- /dev/null +++ b/scripts/test_zookeeper_version.py @@ -0,0 +1,221 @@ +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class JavaDownloadTest(unittest.TestCase): + def run_download(self, arch="amd64", base="https://download.oracle.com/java", + version="25", api_key=None, corrupt=False): + dockerfile = (ROOT / "docker/zookeeper-image/Dockerfile").read_text() + download = dockerfile.split("ENV JAVA_VERSION=", 1)[1].split( + "RUN ", 1 + )[1].split('mkdir -p "$JAVA_HOME"', 1)[0] + if download.startswith("--mount="): + download = download.split("\\\n", 1)[1] + download = download.replace("\\\n", "").rstrip() + "\n" + variables = {key: value for key, value in os.environ.items() + if key != "JAVA_DOWNLOAD_APIKEY"} + variables.update( + ARCH=arch, java_version="25", java_download_version=version, + java_download_base_url=base, + DOWNLOAD_SHA=hashlib.sha256( + b"wrong" if corrupt else b"artifact" + ).hexdigest(), + ) + if api_key is not None: + variables["JAVA_DOWNLOAD_APIKEY"] = api_key + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + ["bash", "-ec", + 'dpkg() { printf "%s\\n" "$ARCH"; }\n' + 'apt-mark() { :; }\n' + 'apt-get() { :; }\n' + 'rm() { :; }\n' + 'wget() { return 99; }\n' + 'curl() {\n' + ' python3 -c \'import json, sys; ' + 'open("downloads.jsonl", "a").write(' + 'json.dumps(sys.argv[1:]) + "\\n")\' "$@"\n' + ' for url; do :; done\n' + ' case "$url" in\n' + ' *.sha256) printf "%s\\n" "$DOWNLOAD_SHA" ;;\n' + ' *) printf artifact > openjdk.tgz ;;\n' + ' esac\n' + '}\n' + download], + cwd=directory, env=variables, text=True, capture_output=True, + ) + log = Path(directory) / "downloads.jsonl" + calls = [json.loads(line) for line in log.read_text().splitlines() + ] if log.exists() else [] + return result, calls + + def test_oracle_defaults_preserve_both_architecture_urls(self): + for arch, name in (("amd64", "x64"), ("arm64", "aarch64")): + with self.subTest(arch=arch): + result, calls = self.run_download(arch=arch) + self.assertEqual(result.returncode, 0, result.stderr) + url = ("https://download.oracle.com/java/25/latest/" + f"jdk-25_linux-{name}_bin.tar.gz") + self.assertEqual([call[-1] for call in calls], + [url, url + ".sha256"]) + self.assertNotIn("--expand-header", calls[0]) + + def test_alternate_source_uses_patch_version_and_optional_authentication(self): + for arch, name in (("amd64", "x64"), ("arm64", "aarch64")): + for key in (None, "synthetic-download-key"): + with self.subTest(arch=arch, authenticated=key is not None): + result, calls = self.run_download( + arch=arch, base="https://jdk.example.test/java", + version="25.0.1.1", api_key=key, + ) + self.assertEqual(result.returncode, 0, result.stderr) + url = ("https://jdk.example.test/java/25/25.0.1.1/" + f"jdk-25.0.1.1_linux-{name}_bin.tar.gz") + self.assertEqual([call[-1] for call in calls], + [url, url + ".sha256"]) + for call in calls: + self.assertIn("--fail", call) + self.assertIn("--location", call) + self.assertEqual("--expand-header" in call, key is not None) + if key is not None: + self.assertIn("%JAVA_DOWNLOAD_APIKEY", call) + self.assertIn("Authorization: Bearer {{JAVA_DOWNLOAD_APIKEY}}", + call) + self.assertNotIn(key, " ".join(call)) + self.assertNotIn(key, result.stderr) + + def test_checksum_mismatch_stops_the_build(self): + result, _ = self.run_download(corrupt=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("FAILED", result.stdout) + + def test_unsupported_architecture_stops_before_downloading(self): + result, calls = self.run_download(arch="s390x") + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsupported architecture", result.stderr) + self.assertEqual(calls, []) + + +class ZookeeperDownloadTest(unittest.TestCase): + def run_download(self, available_url, empty=False): + dockerfile = (ROOT / "docker/zookeeper-image/Dockerfile").read_text() + helper = "ddist() {" + dockerfile.split("ddist() {", 1)[1].split( + " ddist ", 1 + )[0] + helper = helper.replace("\\\n", "") + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + ["sh", "-ec", + 'wget() {\n' + ' while [ "$#" -gt 0 ]; do\n' + ' case "$1" in\n' + ' -O) shift; output="$1" ;;\n' + ' https://*) url="$1" ;;\n' + ' esac\n' + ' shift\n' + ' done\n' + ' [ "$url" = "$AVAILABLE_URL" ] || return 4\n' + ' : > "$output"\n' + ' [ "$EMPTY" = 1 ] || printf artifact > "$output"\n' + '}\n' + helper + '\nddist artifact.tgz zookeeper/release.tgz'], + cwd=directory, + env={**os.environ, "AVAILABLE_URL": available_url, + "EMPTY": "1" if empty else "0"}, + text=True, capture_output=True, + ) + target = Path(directory) / "artifact.tgz" + content = target.read_text() if target.exists() else None + return result, content + + def test_downloads_from_cdn_when_origin_is_unavailable(self): + result, content = self.run_download( + "https://dlcdn.apache.org/zookeeper/release.tgz" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(content, "artifact") + + def test_falls_back_to_archive_for_older_releases(self): + result, content = self.run_download( + "https://archive.apache.org/dist/zookeeper/release.tgz" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(content, "artifact") + + def test_fails_when_all_sources_are_unavailable_or_empty(self): + for empty in (False, True): + with self.subTest(empty=empty): + result, _ = self.run_download( + "https://archive.apache.org/dist/zookeeper/release.tgz" + if empty else "", empty=empty, + ) + self.assertNotEqual(result.returncode, 0) + + +class ZookeeperVersionTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + for name in ("Makefile", "docker/zu/build.gradle.kts", + "scripts/zookeeper-version.sh"): + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / name, target) + self.buildfile = self.root / "docker/zu/build.gradle.kts" + self.bin = self.root / "bin" + self.bin.mkdir() + docker = self.bin / "docker" + docker.write_text( + "#!/usr/bin/env python3\n" + "import json, sys\n" + "with open('docker-commands.log', 'a') as log:\n" + " log.write(json.dumps(sys.argv[1:]) + '\\n')\n" + ) + docker.chmod(0o755) + + def run_make(self): + return subprocess.run( + ["make", "build-zk-image", "VERSION=test", "GIT_SHA=test"], + cwd=self.root, + env={**os.environ, "PATH": f"{self.bin}:{os.environ['PATH']}"}, + text=True, + capture_output=True, + ) + + def test_reads_shared_version_outside_repository_cwd(self): + self.buildfile.write_text('val zookeeperVersion = "9.8.7"\n') + result = subprocess.run( + ["bash", str(self.root / "scripts/zookeeper-version.sh")], + cwd=self.bin, text=True, capture_output=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "9.8.7\n") + + def test_make_passes_updated_version_to_server_build(self): + self.buildfile.write_text('val zookeeperVersion = "9.8.7"\n') + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + commands = [ + json.loads(line) + for line in (self.root / "docker-commands.log").read_text().splitlines() + ] + self.assertIn("ZK_VERSION=9.8.7", commands[0]) + + def test_invalid_version_stops_before_docker_build(self): + for value in ("", 'val zookeeperVersion = ""\n', + 'val zookeeperVersion = "latest"\n', + 'val zookeeperVersion = "3.10.0-rc1"\n', + 'val zookeeperVersion = "3.9.6"\nval zookeeperVersion = "3.10.0"\n'): + with self.subTest(value=value): + self.buildfile.write_text(value) + result = self.run_make() + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.root / "docker-commands.log").exists()) From 3ad9e549d2a248ce67753eac1159afae23e3cbae Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Wed, 18 Nov 2026 12:55:59 +0100 Subject: [PATCH 2/5] fix(docker): support major-only JDK mirrors and Java 21 runtime Use the documented major-only layout for alternate JDK mirrors. Compile the zu helper to Java 21 bytecode while retaining JDK 25 as the builder and default runtime, and keep image descriptions accurate for alternate runtimes. Validated the complete amd64 image with authenticated Oracle JDK 21.0.13.1+4 downloads, ZooKeeper startup and live zu client access. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit 90bafb9f1490ab1283829e73fbf4b9e57d59313b) --- .github/workflows/release.yml | 5 +++-- docker/Dockerfile | 19 +++++++++++++------ docker/zookeeper-image/Dockerfile | 2 +- docker/zookeeper-image/README.md | 29 ++++++++++++++++++++++------- docker/zu/build.gradle.kts | 5 +++-- scripts/test_zookeeper_version.py | 10 +++++----- 6 files changed, 47 insertions(+), 23 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index dafefb97b..222961289 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -93,7 +93,8 @@ jobs: tags: | type=raw,value=3.8.4-apache-${{ needs.prepare.outputs.tag }} labels: | - org.opencontainers.image.description=Apache Zookeeper 3.8.4 with OpenJDK 21 + org.opencontainers.image.description=Apache Zookeeper ${{ needs.prepare.outputs.zk_version }} + # Rebuilt on every release with Debian security updates and the selected JDK. - name: Build and push zookeeper Apache image uses: docker/build-push-action@v6 with: @@ -146,7 +147,7 @@ jobs: tags: | type=raw,value=3.8.4-${{ needs.prepare.outputs.tag }} labels: | - org.opencontainers.image.description=Apache Zookeeper image build with java-21 + org.opencontainers.image.description=Apache Zookeeper image - name: Build and push zookeeper image uses: docker/build-push-action@v6 with: diff --git a/docker/Dockerfile b/docker/Dockerfile index 769601218..c67cd7bdb 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,4 @@ +# check=skip=InvalidDefaultArgInFrom # # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. # @@ -8,26 +9,32 @@ # http://www.apache.org/licenses/LICENSE-2.0 # -FROM gradle:jdk21 +# Base Apache ZooKeeper + OpenJDK image built from docker/zookeeper-image. +# It is not pinned here: the build-zookeeper release job builds that image +# first and passes its digest in, so both always come from the same commit +# and shared zookeeperVersion. For local builds use `make build-zk-image`. +ARG BASE_IMAGE + +# zu.jar is architecture-independent bytecode, so build it once on the native +# build platform instead of re-running Gradle under emulation for every target. +FROM --platform=$BUILDPLATFORM gradle:jdk25 AS zu-builder RUN mkdir /zu COPY zu /zu WORKDIR /zu RUN ./gradlew shadowJar --no-daemon -# use forked base zookeeper 3.8.4 docker image -# that runs on openjdk-21 -FROM adobe/zookeeper:3.8.4-apache-zk-20250914 +FROM ${BASE_IMAGE} COPY bin /usr/local/bin RUN chmod +x /usr/local/bin/* -COPY --from=0 /zu/build/libs/zu.jar /opt/libs/ +COPY --from=zu-builder /zu/build/libs/zu.jar /opt/libs/ RUN apt-get -q update && \ apt-get install --no-install-recommends -y dnsutils curl procps socat # OCI standard labels LABEL org.opencontainers.image.title="Apache Zookeeper" -LABEL org.opencontainers.image.description="Apache Zookeeper image build with java-21" +LABEL org.opencontainers.image.description="Apache Zookeeper image build with java-${JAVA_VERSION}" LABEL org.opencontainers.image.source="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.url="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.documentation="https://github.com/adobe/zookeeper-operator" diff --git a/docker/zookeeper-image/Dockerfile b/docker/zookeeper-image/Dockerfile index 953424f66..5fb10f335 100644 --- a/docker/zookeeper-image/Dockerfile +++ b/docker/zookeeper-image/Dockerfile @@ -26,7 +26,7 @@ RUN --mount=type=secret,id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY \ \ case "$java_download_base_url" in \ https://download.oracle.com/*) java_download_base_url="${java_download_base_url}/${java_version}/latest" ;; \ - *) java_download_base_url="${java_download_base_url}/${java_version}/${java_download_version}" ;; \ + *) java_download_base_url="${java_download_base_url}/${java_version}" ;; \ esac; \ \ arch="$(dpkg --print-architecture)"; \ diff --git a/docker/zookeeper-image/README.md b/docker/zookeeper-image/README.md index b0b8b88e6..dd8a4d5eb 100644 --- a/docker/zookeeper-image/README.md +++ b/docker/zookeeper-image/README.md @@ -3,20 +3,20 @@ Build from the repository root: ```bash -docker buildx build \ +docker buildx build --load \ --build-arg ZK_VERSION="$(./scripts/zookeeper-version.sh)" \ -t zookeeper-apache:local docker/zookeeper-image ``` -The image uses Debian Trixie and Java 25. Oracle's public JDK download URL is -the default, without authentication. +The image uses Debian Trixie and defaults to Java 25. Oracle's public JDK +download URL is the default, without authentication. ## Alternate JDK downloads Override `java_download_base_url` and `java_download_version` to use another source. The Dockerfile appends `java_version` (default: `25`) and `latest` -for Oracle, or `java_version` and `java_download_version` for alternate -sources. Neither version belongs in the base URL. Alternate sources must +for Oracle, or only `java_version` for alternate sources. Do not include +the Java-major directory in the base URL. Alternate sources must provide Oracle-style archive filenames and matching `.sha256` files for each architecture (`x64` and `aarch64`). @@ -24,10 +24,11 @@ For example, add these options to the build command: ```bash --build-arg java_download_base_url=https://jdk.example.com/java \ ---build-arg java_download_version=25.0.1.1 +--build-arg java_version=21 \ +--build-arg java_download_version=21.0.13.1 ``` -This requests `/java/25/25.0.1.1/jdk-25.0.1.1_linux-_bin.tar.gz` +This requests `/java/21/jdk-21.0.13.1_linux-_bin.tar.gz` and its `.sha256` file. Both downloads must succeed and the checksum must match. For authenticated downloads, pass an optional API key as a BuildKit secret: @@ -46,3 +47,17 @@ build step's `build-args`, configure the GitHub Actions secret For authenticated downloads, also enable the commented `secrets` input and configure `JAVA_DOWNLOAD_APIKEY`. PR image builds continue to use Oracle without secrets. + +## Full ZooKeeper image + +After building the base image above, build the operator-flavoured image: + +```bash +docker buildx build --load \ + --build-arg BASE_IMAGE=zookeeper-apache:local \ + -t zookeeper:local docker +``` + +The `zu` helper targets Java 21 bytecode, so the full image supports both +Java 21 and the default Java 25 runtime. The helper is built with JDK 25 +regardless of the runtime selected for the base image. diff --git a/docker/zu/build.gradle.kts b/docker/zu/build.gradle.kts index 784580a60..eabe3783c 100644 --- a/docker/zu/build.gradle.kts +++ b/docker/zu/build.gradle.kts @@ -1,5 +1,6 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { kotlin("jvm") version "1.9.21" @@ -28,7 +29,7 @@ tasks.withType() { } tasks.withType { - kotlinOptions { - jvmTarget = "21" + compilerOptions { + jvmTarget.set(JvmTarget.JVM_21) } } diff --git a/scripts/test_zookeeper_version.py b/scripts/test_zookeeper_version.py index c79a59d9c..dc483ae78 100644 --- a/scripts/test_zookeeper_version.py +++ b/scripts/test_zookeeper_version.py @@ -13,7 +13,7 @@ class JavaDownloadTest(unittest.TestCase): def run_download(self, arch="amd64", base="https://download.oracle.com/java", - version="25", api_key=None, corrupt=False): + major="25", version="25", api_key=None, corrupt=False): dockerfile = (ROOT / "docker/zookeeper-image/Dockerfile").read_text() download = dockerfile.split("ENV JAVA_VERSION=", 1)[1].split( "RUN ", 1 @@ -24,7 +24,7 @@ def run_download(self, arch="amd64", base="https://download.oracle.com/java", variables = {key: value for key, value in os.environ.items() if key != "JAVA_DOWNLOAD_APIKEY"} variables.update( - ARCH=arch, java_version="25", java_download_version=version, + ARCH=arch, java_version=major, java_download_version=version, java_download_base_url=base, DOWNLOAD_SHA=hashlib.sha256( b"wrong" if corrupt else b"artifact" @@ -74,11 +74,11 @@ def test_alternate_source_uses_patch_version_and_optional_authentication(self): with self.subTest(arch=arch, authenticated=key is not None): result, calls = self.run_download( arch=arch, base="https://jdk.example.test/java", - version="25.0.1.1", api_key=key, + major="21", version="21.0.13.1", api_key=key, ) self.assertEqual(result.returncode, 0, result.stderr) - url = ("https://jdk.example.test/java/25/25.0.1.1/" - f"jdk-25.0.1.1_linux-{name}_bin.tar.gz") + url = ("https://jdk.example.test/java/21/" + f"jdk-21.0.13.1_linux-{name}_bin.tar.gz") self.assertEqual([call[-1] for call in calls], [url, url + ".sha256"]) for call in calls: From 8db53fc73f42547d75b8815a3d070f0f5cf88ff1 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Wed, 18 Nov 2026 13:30:23 +0100 Subject: [PATCH 3/5] refactor(docker): derive helper Java targets from the base image Keep java_version in the base Dockerfile as the single runtime setting. Read the selected base JDK release metadata in the native helper builder and use one Gradle property for Java source/target compatibility and the Kotlin JVM target. Verified full Java 21 and Java 25 image builds, matching helper bytecode versions, ZooKeeper startup and live helper operations; all 35 build-script tests pass. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit 9a20f307bcdda5ad1d792856cb041a7e25aa99af) --- docker/Dockerfile | 10 ++- docker/zookeeper-image/README.md | 11 +++- docker/zu/build.gradle.kts | 8 ++- scripts/test_zu_runtime.py | 103 +++++++++++++++++++++++++++++++ 4 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 scripts/test_zu_runtime.py diff --git a/docker/Dockerfile b/docker/Dockerfile index c67cd7bdb..4bdd8a046 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,4 @@ +# syntax=docker/dockerfile:1 # check=skip=InvalidDefaultArgInFrom # # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. @@ -15,6 +16,8 @@ # and shared zookeeperVersion. For local builds use `make build-zk-image`. ARG BASE_IMAGE +FROM ${BASE_IMAGE} AS runtime + # zu.jar is architecture-independent bytecode, so build it once on the native # build platform instead of re-running Gradle under emulation for every target. FROM --platform=$BUILDPLATFORM gradle:jdk25 AS zu-builder @@ -22,9 +25,12 @@ FROM --platform=$BUILDPLATFORM gradle:jdk25 AS zu-builder RUN mkdir /zu COPY zu /zu WORKDIR /zu -RUN ./gradlew shadowJar --no-daemon +RUN --mount=type=bind,from=runtime,source=/usr/local,target=/runtime-java \ + java_version="$(sed -n 's/^JAVA_VERSION="\([0-9][0-9]*\).*/\1/p' /runtime-java/openjdk-*/release)" && \ + test -n "$java_version" && \ + ./gradlew -PjavaVersion="$java_version" shadowJar --no-daemon -FROM ${BASE_IMAGE} +FROM runtime COPY bin /usr/local/bin RUN chmod +x /usr/local/bin/* COPY --from=zu-builder /zu/build/libs/zu.jar /opt/libs/ diff --git a/docker/zookeeper-image/README.md b/docker/zookeeper-image/README.md index dd8a4d5eb..ea970475d 100644 --- a/docker/zookeeper-image/README.md +++ b/docker/zookeeper-image/README.md @@ -58,6 +58,11 @@ docker buildx build --load \ -t zookeeper:local docker ``` -The `zu` helper targets Java 21 bytecode, so the full image supports both -Java 21 and the default Java 25 runtime. The helper is built with JDK 25 -regardless of the runtime selected for the base image. +The full image reads the base image's JDK release metadata and uses its +Java major version for both Java and Kotlin compilation targets in `zu`. +Set `java_version` only when building the base image; no second version +argument is needed for the full image. + +The native Gradle builder uses JDK 25 independently of the selected runtime. +For direct Gradle builds, supply the target explicitly with +`./gradlew -PjavaVersion=21 shadowJar --no-daemon`. diff --git a/docker/zu/build.gradle.kts b/docker/zu/build.gradle.kts index eabe3783c..c75b48dee 100644 --- a/docker/zu/build.gradle.kts +++ b/docker/zu/build.gradle.kts @@ -11,9 +11,11 @@ repositories { mavenCentral() } +val javaVersion = JavaVersion.toVersion(providers.gradleProperty("javaVersion").get()) + java { - sourceCompatibility = JavaVersion.VERSION_21 - targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = javaVersion + targetCompatibility = javaVersion } dependencies { @@ -30,6 +32,6 @@ tasks.withType() { tasks.withType { compilerOptions { - jvmTarget.set(JvmTarget.JVM_21) + jvmTarget.set(JvmTarget.fromTarget(javaVersion.majorVersion)) } } diff --git a/scripts/test_zu_runtime.py b/scripts/test_zu_runtime.py new file mode 100644 index 000000000..3c481c6c3 --- /dev/null +++ b/scripts/test_zu_runtime.py @@ -0,0 +1,103 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class ZuJavaVersionTest(unittest.TestCase): + def run_build(self, release): + dockerfile = (ROOT / "docker/Dockerfile").read_text() + build = dockerfile.split("WORKDIR /zu\n", 1)[1].split( + "\nFROM ", 1 + )[0].strip().removeprefix("RUN ") + if build.startswith("--mount="): + build = build.split("\\\n", 1)[1] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + jdk = root / "runtime-java/openjdk-selected" + jdk.mkdir(parents=True) + (jdk / "release").write_text(release) + gradle = root / "gradlew" + gradle.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\n') + gradle.chmod(0o700) + return subprocess.run( + ["sh", "-ec", build.replace("\\\n", "").replace( + "/runtime-java", str(root / "runtime-java") + )], + cwd=root, text=True, capture_output=True, + ) + + def test_compilation_target_follows_base_image_jdk(self): + for release, major in (('JAVA_VERSION="21.0.13.1"\n', "21"), + ('JAVA_VERSION="25.0.5.1"\n', "25")): + with self.subTest(major=major): + result = self.run_build(release) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), [ + f"-PjavaVersion={major}", "shadowJar", "--no-daemon", + ]) + + def test_missing_java_version_stops_before_gradle(self): + result = self.run_build('IMPLEMENTOR="Oracle Corporation"\n') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + +class ZuRuntimeTest(unittest.TestCase): + def run_zu(self, env=None, java_body=None): + variables = { + key: value for key, value in os.environ.items() + if key not in ("ZK_CLIENT_HEAP", "CLIENT_JVMFLAGS", "ZK_OPTIONS") + } + variables.update(env or {}) + java_body = java_body or ( + """python3 -c 'import json, sys; print(json.dumps(sys.argv[1:]))' "$@";""" + ) + return subprocess.run( + ["bash", "-c", + 'source "$1"\n' + f"java() {{ {java_body} }}\n" + 'shift\n' + 'zkRunZu "$@"', + "test", str(ROOT / "docker/bin/zookeeperFunctions.sh"), + "sync", "localhost:2181", "/path with spaces"], + env=variables, text=True, capture_output=True, + ) + + def test_default_heap_and_quiet_logging_preserve_helper_arguments(self): + result = self.run_zu() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), [ + "-Xmx256m", + "-Dlogback.configurationFile=/conf/logback-quiet.xml", + "-jar", "/opt/libs/zu.jar", + "sync", "localhost:2181", "/path with spaces", + ]) + + def test_custom_heap_jvm_and_tls_options_are_forwarded(self): + result = self.run_zu({ + "ZK_CLIENT_HEAP": "384", + "CLIENT_JVMFLAGS": "-Xmx512m -Dcustom.option=value", + "ZK_OPTIONS": ( + "-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty " + "-Dzookeeper.client.secure=true" + ), + }) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), [ + "-Xmx384m", "-Xmx512m", "-Dcustom.option=value", + "-Dlogback.configurationFile=/conf/logback-quiet.xml", + "-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty", + "-Dzookeeper.client.secure=true", + "-jar", "/opt/libs/zu.jar", + "sync", "localhost:2181", "/path with spaces", + ]) + + def test_helper_failure_is_not_swallowed(self): + result = self.run_zu(java_body="return 7;") + self.assertEqual(result.returncode, 7, result.stderr) From ae00a7820b426ffd6caeb6a8dd7b4b0ca6f0dad8 Mon Sep 17 00:00:00 2001 From: Adi Muraru Date: Wed, 18 Nov 2026 13:45:34 +0100 Subject: [PATCH 4/5] fix(zu): restore bare Gradle builds using the central Java default Default javaVersion lazily from the base Dockerfile when no explicit Gradle property is supplied. This restores the CodeQL ./gradlew build command while retaining Java 25 as the shared runtime and compilation default. Docker builds continue to derive targets from the selected base image. Verified bare Gradle build, explicit Java 21 override, full Java 25 image/runtime smoke tests and 35 script regression tests. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> (cherry picked from commit b56b2771c1424981e8c22cf4764d9790a751cfa3) --- .github/workflows/zookeeper-images.yaml | 104 ++++++++++++++++++++++++ docker/zookeeper-image/README.md | 5 +- docker/zu/build.gradle.kts | 8 +- 3 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/zookeeper-images.yaml diff --git a/.github/workflows/zookeeper-images.yaml b/.github/workflows/zookeeper-images.yaml new file mode 100644 index 000000000..07b19e7cb --- /dev/null +++ b/.github/workflows/zookeeper-images.yaml @@ -0,0 +1,104 @@ +# Builds the Apache ZooKeeper base image (docker/zookeeper-image) and the +# operator-flavoured ZooKeeper image (docker/Dockerfile) on every change to +# them, so a ZooKeeper, JDK, zu or script change is validated before a release +# tag is pushed rather than after. +name: zookeeper-images + +on: + pull_request: + paths: + - 'docker/**' + - 'scripts/zookeeper-version.sh' + - '.github/workflows/zookeeper-images.yaml' + push: + branches: + - master + paths: + - 'docker/**' + - 'scripts/zookeeper-version.sh' + - '.github/workflows/zookeeper-images.yaml' + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + name: build (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + # Native runners per architecture: no QEMU emulation needed. + - arch: amd64 + runner: ubuntu-latest + - arch: arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Resolve ZooKeeper version + id: zk + run: | + zk_version="$(./scripts/zookeeper-version.sh)" + echo "zk_version=${zk_version}" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-buildx-action@v4 + + - name: Build Apache ZooKeeper base image + uses: docker/build-push-action@v7 + with: + context: docker/zookeeper-image + platforms: linux/${{ matrix.arch }} + build-args: | + ZK_VERSION=${{ steps.zk.outputs.zk_version }} + outputs: type=oci,dest=${{ runner.temp }}/zookeeper-apache,tar=false + cache-from: type=gha,scope=zookeeper-apache-${{ matrix.arch }} + cache-to: ${{ github.event_name == 'push' && format('type=gha,mode=max,scope=zookeeper-apache-{0}', matrix.arch) || '' }} + + # Same pairing as the release job: docker/Dockerfile is built FROM the + # base image just built (via BASE_IMAGE), here resolved from the local + # OCI layout through a named build context instead of a registry. + - name: Build ZooKeeper image on top of the freshly built base + uses: docker/build-push-action@v7 + with: + context: docker + platforms: linux/${{ matrix.arch }} + build-args: | + BASE_IMAGE=zookeeper-apache:ci + build-contexts: | + zookeeper-apache:ci=oci-layout://${{ runner.temp }}/zookeeper-apache + tags: zookeeper:ci + load: true + cache-from: type=gha,scope=zookeeper-${{ matrix.arch }} + cache-to: ${{ github.event_name == 'push' && format('type=gha,mode=max,scope=zookeeper-{0}', matrix.arch) || '' }} + + - name: Smoke test image + env: + ZK_VERSION: ${{ steps.zk.outputs.zk_version }} + run: | + docker run --rm --entrypoint bash -e ZK_VERSION zookeeper:ci -ec ' + java -version + java -XshowSettings:properties -version 2>&1 | grep -Eq "^[[:space:]]*java.specification.version = ${JAVA_VERSION}$" + zkServer.sh version 2>/dev/null | tee /dev/stderr | grep -q "${ZK_VERSION}" + test -s /opt/libs/zu.jar + set +e + zu_output="$(java -jar /opt/libs/zu.jar 2>&1)" + zu_status=$? + set -e + printf "%s\n" "$zu_output" + test "$zu_status" -eq 1 + [[ "$zu_output" == "Usage: zu "* ]] + for s in zookeeperStart.sh zookeeperReady.sh zookeeperLive.sh zookeeperTeardown.sh zookeeperMetrics.sh; do + test -x "/usr/local/bin/${s}" + done + command -v dig curl socat pgrep >/dev/null + ' diff --git a/docker/zookeeper-image/README.md b/docker/zookeeper-image/README.md index ea970475d..0e440757b 100644 --- a/docker/zookeeper-image/README.md +++ b/docker/zookeeper-image/README.md @@ -64,5 +64,6 @@ Set `java_version` only when building the base image; no second version argument is needed for the full image. The native Gradle builder uses JDK 25 independently of the selected runtime. -For direct Gradle builds, supply the target explicitly with -`./gradlew -PjavaVersion=21 shadowJar --no-daemon`. +Direct Gradle builds, including `./gradlew build --no-daemon`, read the +default from the same base Dockerfile. An explicit `-PjavaVersion=` +overrides that target when needed. diff --git a/docker/zu/build.gradle.kts b/docker/zu/build.gradle.kts index c75b48dee..7143b5ab5 100644 --- a/docker/zu/build.gradle.kts +++ b/docker/zu/build.gradle.kts @@ -11,7 +11,13 @@ repositories { mavenCentral() } -val javaVersion = JavaVersion.toVersion(providers.gradleProperty("javaVersion").get()) +val javaVersion = JavaVersion.toVersion( + providers.gradleProperty("javaVersion").orElse(providers.provider { + file("../zookeeper-image/Dockerfile").readLines() + .single { it.startsWith("ARG java_version=") } + .substringAfter("=") + }).get() +) java { sourceCompatibility = javaVersion From 49cc6fddcc4022bc608683ef3d81a625b4391e6b Mon Sep 17 00:00:00 2001 From: DeliaCraciun <94064078+DeliaCraciun@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:57:09 +0200 Subject: [PATCH 5/5] Oncall - Configure production ZooKeeper 3.8.4 JDK 21.0.13.1 hotfix Add production hotfix settings on top of all four separately cherry-picked commits from adobe/zookeeper-operator#113. Keep the donor download implementation, dynamic helper targets, original test names and test contents unchanged. Select approved Oracle JDK 21.0.13.1, rebuild the Apache base and pin the wrapper to its digest. Preserve ZooKeeper server 3.8.4, the deployed helper client dependency 3.9.3 and all existing service source code. Restore upstream build prerequisites missing from the old release: the shared version resolver and helper shell function, plus the Gradle/Kotlin/Shadow toolchain required by the retained JDK 25 native builder. Java and Kotlin bytecode targets still follow the selected Java 21 runtime. Configure trusted manual image validation with the same Artifactory override and add exact Oracle 21.0.13.1+4 process checks and live ZooKeeper/helper smoke tests for both architectures. Retain the old tag-push publishing trigger and document prerequisites; no tag or image is published by this commit. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .github/workflows/ci.yaml | 6 ++- .github/workflows/release.yml | 47 +++++++++++----- .github/workflows/zookeeper-images.yaml | 36 +++++++------ Makefile | 13 ++++- docker/bin/zookeeperFunctions.sh | 8 +++ docker/zookeeper-image/Dockerfile | 14 ++--- docker/zookeeper-image/README.md | 51 ++++++++++++++---- docker/zu/build.gradle.kts | 9 +++- .../gradle/wrapper/gradle-wrapper.properties | 2 +- scripts/test-zookeeper-image.sh | 53 +++++++++++++++++++ scripts/zookeeper-version.sh | 9 ++++ 11 files changed, 195 insertions(+), 53 deletions(-) create mode 100644 scripts/test-zookeeper-image.sh create mode 100755 scripts/zookeeper-version.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b044b073..4a9350f90 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.25 - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: 1.25 id: go @@ -20,11 +20,13 @@ jobs: run: | sudo ln -sf `which go` `sudo which go` || true sudo go version - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 - name: get go version run: go version - name: Gofmt and License checks run: make check + - name: Build script regression tests + run: python3 -B -m unittest discover -s scripts -p 'test_*.py' - name: unit tests run: make test - name: Build Docker image (no push) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 222961289..73d42e3c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false env: # GitHub Container Registry @@ -26,7 +26,10 @@ jobs: outputs: tag: ${{ steps.vars.outputs.tag }} built_at: ${{ steps.date.outputs.built_at }} + zk_version: ${{ steps.zk.outputs.zk_version }} steps: + - name: Checkout code + uses: actions/checkout@v5 - name: Get tag name id: vars run: echo "tag=${GITHUB_REF:10}" >> $GITHUB_OUTPUT @@ -34,6 +37,11 @@ jobs: id: date run: | echo "built_at=$(date --rfc-3339=date)" >> $GITHUB_OUTPUT + - name: Resolve Apache ZooKeeper version + id: zk + run: | + zk_version="$(./scripts/zookeeper-version.sh)" + echo "zk_version=$zk_version" >> "$GITHUB_OUTPUT" operator-test: runs-on: ubuntu-latest @@ -43,7 +51,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Set up Go 1.25 - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: 1.25 id: go @@ -60,14 +68,22 @@ jobs: build-zookeeper-apache: runs-on: ubuntu-latest - needs: prepare - if: ${{ startsWith(github.ref, 'refs/tags/zk') }} + needs: [prepare, operator-test] + outputs: + digest: ${{ steps.build-zk-apache.outputs.digest }} permissions: contents: read packages: write steps: - name: Checkout code uses: actions/checkout@v5 + - name: Check emergency JDK download configuration + env: + JAVA_DOWNLOAD_BASE_URL: ${{ secrets.JAVA_DOWNLOAD_BASE_URL }} + JAVA_DOWNLOAD_APIKEY: ${{ secrets.JAVA_DOWNLOAD_APIKEY }} + run: | + : "${JAVA_DOWNLOAD_BASE_URL:?Set the approved JDK mirror repository secret}" + : "${JAVA_DOWNLOAD_APIKEY:?Set the JDK mirror credential repository secret}" - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -91,21 +107,22 @@ jobs: ${{ env.DOCKERHUB_ZK_IMAGE }} ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}/zookeeper tags: | - type=raw,value=3.8.4-apache-${{ needs.prepare.outputs.tag }} + type=raw,value=${{ needs.prepare.outputs.zk_version }}-apache-${{ needs.prepare.outputs.tag }} labels: | org.opencontainers.image.description=Apache Zookeeper ${{ needs.prepare.outputs.zk_version }} # Rebuilt on every release with Debian security updates and the selected JDK. - name: Build and push zookeeper Apache image + id: build-zk-apache uses: docker/build-push-action@v6 with: context: docker/zookeeper-image - # Enable these together when selecting an alternate JDK release: - # secrets: | - # JAVA_DOWNLOAD_APIKEY=${{ secrets.JAVA_DOWNLOAD_APIKEY }} - # Add these overrides under build-args: - # java_version=25 - # java_download_base_url=${{ secrets.JAVA_DOWNLOAD_BASE_URL }} - # java_download_version=25.0.1.1 + secrets: | + JAVA_DOWNLOAD_APIKEY=${{ secrets.JAVA_DOWNLOAD_APIKEY }} + build-args: | + ZK_VERSION=${{ needs.prepare.outputs.zk_version }} + java_version=21 + java_download_version=21.0.13.1 + java_download_base_url=${{ secrets.JAVA_DOWNLOAD_BASE_URL }} platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta-zk-apache.outputs.tags }} labels: ${{ steps.meta-zk-apache.outputs.labels }} @@ -114,7 +131,7 @@ jobs: build-zookeeper: runs-on: ubuntu-latest - needs: [prepare, operator-test] + needs: [prepare, operator-test, build-zookeeper-apache] if: ${{ !startsWith(github.ref, 'refs/tags/zk') }} permissions: contents: read @@ -145,7 +162,7 @@ jobs: ${{ env.DOCKERHUB_ZK_IMAGE }} ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}/zookeeper tags: | - type=raw,value=3.8.4-${{ needs.prepare.outputs.tag }} + type=raw,value=${{ needs.prepare.outputs.zk_version }}-${{ needs.prepare.outputs.tag }} labels: | org.opencontainers.image.description=Apache Zookeeper image - name: Build and push zookeeper image @@ -153,6 +170,8 @@ jobs: with: context: docker platforms: linux/amd64,linux/arm64 + build-args: | + BASE_IMAGE=${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}/zookeeper@${{ needs.build-zookeeper-apache.outputs.digest }} tags: ${{ steps.meta-zk.outputs.tags }} labels: ${{ steps.meta-zk.outputs.labels }} annotations: ${{ steps.meta-zk.outputs.annotations }} diff --git a/.github/workflows/zookeeper-images.yaml b/.github/workflows/zookeeper-images.yaml index 07b19e7cb..57494863f 100644 --- a/.github/workflows/zookeeper-images.yaml +++ b/.github/workflows/zookeeper-images.yaml @@ -1,22 +1,7 @@ -# Builds the Apache ZooKeeper base image (docker/zookeeper-image) and the -# operator-flavoured ZooKeeper image (docker/Dockerfile) on every change to -# them, so a ZooKeeper, JDK, zu or script change is validated before a release -# tag is pushed rather than after. +# Validate authenticated emergency builds only on explicitly reviewed refs. name: zookeeper-images on: - pull_request: - paths: - - 'docker/**' - - 'scripts/zookeeper-version.sh' - - '.github/workflows/zookeeper-images.yaml' - push: - branches: - - master - paths: - - 'docker/**' - - 'scripts/zookeeper-version.sh' - - '.github/workflows/zookeeper-images.yaml' workflow_dispatch: permissions: @@ -52,14 +37,26 @@ jobs: echo "zk_version=${zk_version}" >> "$GITHUB_OUTPUT" - uses: docker/setup-buildx-action@v4 + - name: Check emergency JDK download configuration + env: + JAVA_DOWNLOAD_BASE_URL: ${{ secrets.JAVA_DOWNLOAD_BASE_URL }} + JAVA_DOWNLOAD_APIKEY: ${{ secrets.JAVA_DOWNLOAD_APIKEY }} + run: | + : "${JAVA_DOWNLOAD_BASE_URL:?Set the approved JDK mirror repository secret}" + : "${JAVA_DOWNLOAD_APIKEY:?Set the JDK mirror credential repository secret}" - name: Build Apache ZooKeeper base image uses: docker/build-push-action@v7 with: context: docker/zookeeper-image platforms: linux/${{ matrix.arch }} + secrets: | + JAVA_DOWNLOAD_APIKEY=${{ secrets.JAVA_DOWNLOAD_APIKEY }} build-args: | ZK_VERSION=${{ steps.zk.outputs.zk_version }} + java_version=21 + java_download_version=21.0.13.1 + java_download_base_url=${{ secrets.JAVA_DOWNLOAD_BASE_URL }} outputs: type=oci,dest=${{ runner.temp }}/zookeeper-apache,tar=false cache-from: type=gha,scope=zookeeper-apache-${{ matrix.arch }} cache-to: ${{ github.event_name == 'push' && format('type=gha,mode=max,scope=zookeeper-apache-{0}', matrix.arch) || '' }} @@ -96,9 +93,14 @@ jobs: set -e printf "%s\n" "$zu_output" test "$zu_status" -eq 1 - [[ "$zu_output" == "Usage: zu "* ]] + grep -q "^Usage: zu " <<<"$zu_output" for s in zookeeperStart.sh zookeeperReady.sh zookeeperLive.sh zookeeperTeardown.sh zookeeperMetrics.sh; do test -x "/usr/local/bin/${s}" done command -v dig curl socat pgrep >/dev/null ' + - name: Verify patched JVM in the running ZooKeeper process + run: | + docker run --rm --entrypoint bash \ + -v "${PWD}/scripts/test-zookeeper-image.sh:/test-zookeeper-image.sh:ro" \ + zookeeper:ci /test-zookeeper-image.sh diff --git a/Makefile b/Makefile index eb3aad7ef..d8ba1c078 100644 --- a/Makefile +++ b/Makefile @@ -151,9 +151,18 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform - docker buildx rm zookeeper-builder rm Dockerfile.cross -build-zk-image: +JAVA_DOWNLOAD_BASE_URL ?= https://download.oracle.com/java +JAVA_DOWNLOAD_VERSION ?= 21 - docker build --build-arg VERSION=$(VERSION) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) --build-arg GIT_SHA=$(GIT_SHA) -t $(APP_REPO):$(VERSION) ./docker +build-zk-image: + zk_version="$$(./scripts/zookeeper-version.sh)" && \ + docker build --build-arg ZK_VERSION="$$zk_version" \ + --build-arg java_download_base_url="$(JAVA_DOWNLOAD_BASE_URL)" \ + --build-arg java_download_version="$(JAVA_DOWNLOAD_VERSION)" \ + --secret id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY \ + -t $(APP_REPO):$(VERSION)-apache ./docker/zookeeper-image + docker build --build-arg VERSION=$(VERSION) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) --build-arg GIT_SHA=$(GIT_SHA) \ + --build-arg BASE_IMAGE=$(APP_REPO):$(VERSION)-apache -t $(APP_REPO):$(VERSION) ./docker docker tag $(APP_REPO):$(VERSION) $(APP_REPO):latest build-zk-image-swarm: diff --git a/docker/bin/zookeeperFunctions.sh b/docker/bin/zookeeperFunctions.sh index 8c7f0baf9..663e606e0 100644 --- a/docker/bin/zookeeperFunctions.sh +++ b/docker/bin/zookeeperFunctions.sh @@ -12,6 +12,14 @@ set -ex +function zkRunZu() { + # JVM option variables intentionally contain multiple arguments. + # shellcheck disable=SC2086 + java -Xmx"${ZK_CLIENT_HEAP:-256}m" ${CLIENT_JVMFLAGS:-} \ + -Dlogback.configurationFile=/conf/logback-quiet.xml ${ZK_OPTIONS:-} \ + -jar /opt/libs/zu.jar "$@" +} + function zkConfig() { if [ -n "$1" ]; then FQDN="$1" diff --git a/docker/zookeeper-image/Dockerfile b/docker/zookeeper-image/Dockerfile index 5fb10f335..13513fba9 100644 --- a/docker/zookeeper-image/Dockerfile +++ b/docker/zookeeper-image/Dockerfile @@ -3,7 +3,7 @@ # backported from https://github.com/docker-library/openjdk/blob/4c2b1a57869e24c5fcfbea6a17e558c710d4f6bd/21/jdk/slim-bullseye/Dockerfile FROM debian:trixie-slim -ARG java_version=25 +ARG java_version=21 ARG java_download_version=${java_version} ARG java_download_base_url=https://download.oracle.com/java @@ -125,14 +125,15 @@ RUN set -eux; \ dirmngr \ gosu \ gnupg \ - netcat \ + netcat-openbsd \ wget; \ rm -rf /var/lib/apt/lists/*; \ # Verify that gosu binary works gosu nobody true -ARG SHORT_DISTRO_NAME=zookeeper-3.8.4 -ARG DISTRO_NAME=apache-zookeeper-3.8.4-bin +ARG ZK_VERSION=3.8.4 +ARG SHORT_DISTRO_NAME=zookeeper-${ZK_VERSION} +ARG DISTRO_NAME=apache-zookeeper-${ZK_VERSION}-bin # Download Apache Zookeeper, verify its PGP signature, untar and clean up RUN set -eux; \ @@ -142,6 +143,7 @@ RUN set -eux; \ local success=; \ local distUrl=; \ for distUrl in \ + https://dlcdn.apache.org/ \ https://downloads.apache.org/ \ https://www-us.apache.org/dist/ \ https://www.apache.org/dist/ \ @@ -166,7 +168,7 @@ RUN set -eux; \ chown -R zookeeper:zookeeper "/$DISTRO_NAME" COPY opt/zkEnv.sh $DISTRO_NAME/bin/zkEnv.sh -WORKDIR $DISTRO_NAME +WORKDIR /$DISTRO_NAME VOLUME ["$ZOO_DATA_DIR", "$ZOO_DATA_LOG_DIR", "$ZOO_LOG_DIR"] EXPOSE 2181 2888 3888 8080 @@ -182,7 +184,7 @@ LABEL org.opencontainers.image.url="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.documentation="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.vendor="Adobe" LABEL org.opencontainers.image.licenses="Apache-2.0" -LABEL org.opencontainers.image.version="3.8.4" +LABEL org.opencontainers.image.version="${ZK_VERSION}" COPY docker-entrypoint.sh / ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/docker/zookeeper-image/README.md b/docker/zookeeper-image/README.md index 0e440757b..4048784bb 100644 --- a/docker/zookeeper-image/README.md +++ b/docker/zookeeper-image/README.md @@ -8,13 +8,13 @@ docker buildx build --load \ -t zookeeper-apache:local docker/zookeeper-image ``` -The image uses Debian Trixie and defaults to Java 25. Oracle's public JDK +The image uses Debian Trixie and this hotfix defaults to Java 21. Oracle's public JDK download URL is the default, without authentication. ## Alternate JDK downloads Override `java_download_base_url` and `java_download_version` to use another -source. The Dockerfile appends `java_version` (default: `25`) and `latest` +source. The Dockerfile appends `java_version` (default: `21`) and `latest` for Oracle, or only `java_version` for alternate sources. Do not include the Java-major directory in the base URL. Alternate sources must provide Oracle-style archive filenames and matching `.sha256` files for @@ -40,13 +40,12 @@ For authenticated downloads, pass an optional API key as a BuildKit secret: The key is sent as an `Authorization: Bearer` header, is not stored in the image, and is not forwarded to a different host on redirects. -Release builds also default to Oracle. To use an alternate source, add the -commented overrides in `.github/workflows/release.yml` to the Apache ZooKeeper -build step's `build-args`, configure the GitHub Actions secret -`JAVA_DOWNLOAD_BASE_URL`, and select a matching `java_download_version`. -For authenticated downloads, also enable the commented `secrets` input and -configure `JAVA_DOWNLOAD_APIKEY`. PR image builds continue to use Oracle -without secrets. +This hotfix enables the authenticated override in `.github/workflows/release.yml` +and selects Oracle JDK `21.0.13.1`. Configure repository secrets +`JAVA_DOWNLOAD_BASE_URL` and `JAVA_DOWNLOAD_APIKEY` before publishing. The base +URL must be the approved raw Image Factory repository, ending in +`generic-ifup-resources-release/java/oracle`, without a version directory. +The credential is passed only as a BuildKit secret. ## Full ZooKeeper image @@ -67,3 +66,37 @@ The native Gradle builder uses JDK 25 independently of the selected runtime. Direct Gradle builds, including `./gradlew build --no-daemon`, read the default from the same base Dockerfile. An explicit `-PjavaVersion=` overrides that target when needed. + +## Production hotfix + +The branch starts at production tag `0.2.15-adobe-20260225`. The four commits +from [PR #113](https://github.com/adobe/zookeeper-operator/pull/113) are retained +as separate cherry-picks with their original file names. Emergency overrides +and old-release compatibility changes are a separate commit. + +ZooKeeper remains `3.8.4`; the existing `zu` client dependency remains `3.9.3`. +The shared server version resolver and helper shell function are prerequisites +used by the unchanged upstream tests. The native builder retains JDK 25 from +upstream, with its matching Gradle/Kotlin/Shadow toolchain; it does not ship in +the runtime image. The compilation target follows the selected Java 21 base. + +Oracle identifies `21.0.13.1+4` as the fixed JDK 21 build for +[CVE-2026-114938](https://www.oracle.com/security-alerts/cspunov2026.html#AppendixJAVA); +see the [JDK release notes](https://www.oracle.com/java/technologies/javase/21all-relnotes.html#R21_0_13_1). +Confirm runner access and Oracle redistribution approval before publication. + +Run `python3 -B -m unittest discover -s scripts -p 'test_*.py'` for all upstream +build-script tests. Run `zookeeper-images` manually on the reviewed hotfix ref +to build both architectures with the same approved JDK override, without +publication. The additional smoke test verifies the exact Oracle build in the +running ZooKeeper process, server readiness and live `zu` access. + +This old release line publishes on **tag push**, not on GitHub Release +publication. Use a new Helm-compatible tag on the reviewed hotfix commit; +never move an existing tag. A regular tag (not starting with `zk`) rebuilds +the Apache base and pins the wrapper to its digest. It also publishes the +operator image and Helm chart, which need not be deployed for this runtime fix. + +The final server image is `adobe/zookeeper:3.8.4-`. +Verify the published digest and JVM before updating `pipeline-config`. +Publishing does not roll out the image to existing clusters. diff --git a/docker/zu/build.gradle.kts b/docker/zu/build.gradle.kts index 7143b5ab5..dc75d7fc2 100644 --- a/docker/zu/build.gradle.kts +++ b/docker/zu/build.gradle.kts @@ -3,8 +3,8 @@ import org.jetbrains.kotlin.gradle.tasks.KotlinCompile import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { - kotlin("jvm") version "1.9.21" - id("com.github.johnrengelman.shadow") version "8.1.1" + kotlin("jvm") version "2.4.21" + id("com.gradleup.shadow") version "9.6.1" } repositories { @@ -24,6 +24,8 @@ java { targetCompatibility = javaVersion } +val zookeeperVersion = "3.8.4" + dependencies { implementation(kotlin("stdlib")) implementation("org.apache.zookeeper:zookeeper:3.9.3") @@ -31,6 +33,9 @@ dependencies { tasks.withType() { archiveClassifier.set("") + filesMatching("META-INF/*.kotlin_module") { + duplicatesStrategy = DuplicatesStrategy.INCLUDE + } manifest { attributes["Main-Class"] = "io.pravega.zookeeper.MainKt" } diff --git a/docker/zu/gradle/wrapper/gradle-wrapper.properties b/docker/zu/gradle/wrapper/gradle-wrapper.properties index a59520664..16e57d81a 100644 --- a/docker/zu/gradle/wrapper/gradle-wrapper.properties +++ b/docker/zu/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.8.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/scripts/test-zookeeper-image.sh b/scripts/test-zookeeper-image.sh new file mode 100644 index 000000000..fc314fab7 --- /dev/null +++ b/scripts/test-zookeeper-image.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +java -version +properties="$(java -XshowSettings:properties -version 2>&1)" +grep -Eq '^[[:space:]]*java.version = 21\.0\.13\.1$' <<<"$properties" +grep -Eq '^[[:space:]]*java.runtime.version = 21\.0\.13\.1\+4([-[:space:]]|$)' <<<"$properties" +grep -Eq '^[[:space:]]*java.vendor = Oracle Corporation$' <<<"$properties" +zkServer.sh version 2>/dev/null | tee /dev/stderr | grep -q '3\.8\.4' + +set +e +zu_output="$(java -jar /opt/libs/zu.jar 2>&1)" +zu_status=$? +set -e +printf '%s\n' "$zu_output" +test "$zu_status" -eq 1 +grep -q '^Usage: zu ' <<<"$zu_output" + +for script in zookeeperStart.sh zookeeperReady.sh zookeeperLive.sh zookeeperTeardown.sh zookeeperMetrics.sh; do + test -x "/usr/local/bin/$script" +done +command -v dig curl socat pgrep nc >/dev/null + +cat > "$ZOOCFGDIR/zoo.cfg" < "$ZOO_DATA_DIR/myid" +zkServer.sh start +trap 'zkServer.sh stop' EXIT +ready=false +for attempt in $(seq 1 60); do + if [[ "$(printf ruok | nc -w 1 127.0.0.1 2181)" == imok ]]; then + ready=true + break + fi + sleep 1 +done +if [[ "$ready" != true ]]; then + echo "ZooKeeper did not become ready" >&2 + exit 1 +fi +zk_pid="$(pgrep -f 'org.apache.zookeeper.server.quorum.QuorumPeerMain')" +timeout 30 jcmd "$zk_pid" VM.version | tee /dev/stderr | grep -Eq '21\.0\.13\.1\+4' +timeout 30 jcmd "$zk_pid" VM.system_properties | grep -Fx "java.home=$JAVA_HOME" +timeout 30 java -jar /opt/libs/zu.jar get-all 127.0.0.1:2181 diff --git a/scripts/zookeeper-version.sh b/scripts/zookeeper-version.sh new file mode 100755 index 000000000..46c7073f3 --- /dev/null +++ b/scripts/zookeeper-version.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +version="$(sed -n 's/^val zookeeperVersion = "\([^"]*\)"$/\1/p' "$(dirname "$0")/../docker/zu/build.gradle.kts")" +if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Expected one stable zookeeperVersion in docker/zu/build.gradle.kts" >&2 + exit 1 +fi +printf '%s\n' "$version"