diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6b044b073..4a9350f90 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Go 1.25 - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: 1.25 id: go @@ -20,11 +20,13 @@ jobs: run: | sudo ln -sf `which go` `sudo which go` || true sudo go version - - uses: actions/checkout@v2 + - uses: actions/checkout@v5 - name: get go version run: go version - name: Gofmt and License checks run: make check + - name: Build script regression tests + run: python3 -B -m unittest discover -s scripts -p 'test_*.py' - name: unit tests run: make test - name: Build Docker image (no push) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index c6712efde..73d42e3c6 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -7,7 +7,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true + cancel-in-progress: false env: # GitHub Container Registry @@ -26,7 +26,10 @@ jobs: outputs: tag: ${{ steps.vars.outputs.tag }} built_at: ${{ steps.date.outputs.built_at }} + zk_version: ${{ steps.zk.outputs.zk_version }} steps: + - name: Checkout code + uses: actions/checkout@v5 - name: Get tag name id: vars run: echo "tag=${GITHUB_REF:10}" >> $GITHUB_OUTPUT @@ -34,6 +37,11 @@ jobs: id: date run: | echo "built_at=$(date --rfc-3339=date)" >> $GITHUB_OUTPUT + - name: Resolve Apache ZooKeeper version + id: zk + run: | + zk_version="$(./scripts/zookeeper-version.sh)" + echo "zk_version=$zk_version" >> "$GITHUB_OUTPUT" operator-test: runs-on: ubuntu-latest @@ -43,7 +51,7 @@ jobs: - name: Checkout code uses: actions/checkout@v5 - name: Set up Go 1.25 - uses: actions/setup-go@v2 + uses: actions/setup-go@v6 with: go-version: 1.25 id: go @@ -60,14 +68,22 @@ jobs: build-zookeeper-apache: runs-on: ubuntu-latest - needs: prepare - if: ${{ startsWith(github.ref, 'refs/tags/zk') }} + needs: [prepare, operator-test] + outputs: + digest: ${{ steps.build-zk-apache.outputs.digest }} permissions: contents: read packages: write steps: - name: Checkout code uses: actions/checkout@v5 + - name: Check emergency JDK download configuration + env: + JAVA_DOWNLOAD_BASE_URL: ${{ secrets.JAVA_DOWNLOAD_BASE_URL }} + JAVA_DOWNLOAD_APIKEY: ${{ secrets.JAVA_DOWNLOAD_APIKEY }} + run: | + : "${JAVA_DOWNLOAD_BASE_URL:?Set the approved JDK mirror repository secret}" + : "${JAVA_DOWNLOAD_APIKEY:?Set the JDK mirror credential repository secret}" - name: Set up QEMU uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx @@ -91,13 +107,22 @@ jobs: ${{ env.DOCKERHUB_ZK_IMAGE }} ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}/zookeeper tags: | - type=raw,value=3.8.4-apache-${{ needs.prepare.outputs.tag }} + type=raw,value=${{ needs.prepare.outputs.zk_version }}-apache-${{ needs.prepare.outputs.tag }} labels: | - org.opencontainers.image.description=Apache Zookeeper 3.8.4 with OpenJDK 21 + org.opencontainers.image.description=Apache Zookeeper ${{ needs.prepare.outputs.zk_version }} + # Rebuilt on every release with Debian security updates and the selected JDK. - name: Build and push zookeeper Apache image + id: build-zk-apache uses: docker/build-push-action@v6 with: context: docker/zookeeper-image + secrets: | + JAVA_DOWNLOAD_APIKEY=${{ secrets.JAVA_DOWNLOAD_APIKEY }} + build-args: | + ZK_VERSION=${{ needs.prepare.outputs.zk_version }} + java_version=21 + java_download_version=21.0.13.1 + java_download_base_url=${{ secrets.JAVA_DOWNLOAD_BASE_URL }} platforms: linux/amd64,linux/arm64 tags: ${{ steps.meta-zk-apache.outputs.tags }} labels: ${{ steps.meta-zk-apache.outputs.labels }} @@ -106,7 +131,7 @@ jobs: build-zookeeper: runs-on: ubuntu-latest - needs: [prepare, operator-test] + needs: [prepare, operator-test, build-zookeeper-apache] if: ${{ !startsWith(github.ref, 'refs/tags/zk') }} permissions: contents: read @@ -137,14 +162,16 @@ jobs: ${{ env.DOCKERHUB_ZK_IMAGE }} ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}/zookeeper tags: | - type=raw,value=3.8.4-${{ needs.prepare.outputs.tag }} + type=raw,value=${{ needs.prepare.outputs.zk_version }}-${{ needs.prepare.outputs.tag }} labels: | - org.opencontainers.image.description=Apache Zookeeper image build with java-21 + org.opencontainers.image.description=Apache Zookeeper image - name: Build and push zookeeper image uses: docker/build-push-action@v6 with: context: docker platforms: linux/amd64,linux/arm64 + build-args: | + BASE_IMAGE=${{ env.GHCR_REGISTRY }}/${{ env.GHCR_IMAGE_NAME }}/zookeeper@${{ needs.build-zookeeper-apache.outputs.digest }} tags: ${{ steps.meta-zk.outputs.tags }} labels: ${{ steps.meta-zk.outputs.labels }} annotations: ${{ steps.meta-zk.outputs.annotations }} diff --git a/.github/workflows/zookeeper-images.yaml b/.github/workflows/zookeeper-images.yaml new file mode 100644 index 000000000..57494863f --- /dev/null +++ b/.github/workflows/zookeeper-images.yaml @@ -0,0 +1,106 @@ +# Validate authenticated emergency builds only on explicitly reviewed refs. +name: zookeeper-images + +on: + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + build: + name: build (${{ matrix.arch }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 45 + strategy: + fail-fast: false + matrix: + include: + # Native runners per architecture: no QEMU emulation needed. + - arch: amd64 + runner: ubuntu-latest + - arch: arm64 + runner: ubuntu-24.04-arm + steps: + - uses: actions/checkout@v7 + with: + persist-credentials: false + + - name: Resolve ZooKeeper version + id: zk + run: | + zk_version="$(./scripts/zookeeper-version.sh)" + echo "zk_version=${zk_version}" >> "$GITHUB_OUTPUT" + + - uses: docker/setup-buildx-action@v4 + - name: Check emergency JDK download configuration + env: + JAVA_DOWNLOAD_BASE_URL: ${{ secrets.JAVA_DOWNLOAD_BASE_URL }} + JAVA_DOWNLOAD_APIKEY: ${{ secrets.JAVA_DOWNLOAD_APIKEY }} + run: | + : "${JAVA_DOWNLOAD_BASE_URL:?Set the approved JDK mirror repository secret}" + : "${JAVA_DOWNLOAD_APIKEY:?Set the JDK mirror credential repository secret}" + + - name: Build Apache ZooKeeper base image + uses: docker/build-push-action@v7 + with: + context: docker/zookeeper-image + platforms: linux/${{ matrix.arch }} + secrets: | + JAVA_DOWNLOAD_APIKEY=${{ secrets.JAVA_DOWNLOAD_APIKEY }} + build-args: | + ZK_VERSION=${{ steps.zk.outputs.zk_version }} + java_version=21 + java_download_version=21.0.13.1 + java_download_base_url=${{ secrets.JAVA_DOWNLOAD_BASE_URL }} + outputs: type=oci,dest=${{ runner.temp }}/zookeeper-apache,tar=false + cache-from: type=gha,scope=zookeeper-apache-${{ matrix.arch }} + cache-to: ${{ github.event_name == 'push' && format('type=gha,mode=max,scope=zookeeper-apache-{0}', matrix.arch) || '' }} + + # Same pairing as the release job: docker/Dockerfile is built FROM the + # base image just built (via BASE_IMAGE), here resolved from the local + # OCI layout through a named build context instead of a registry. + - name: Build ZooKeeper image on top of the freshly built base + uses: docker/build-push-action@v7 + with: + context: docker + platforms: linux/${{ matrix.arch }} + build-args: | + BASE_IMAGE=zookeeper-apache:ci + build-contexts: | + zookeeper-apache:ci=oci-layout://${{ runner.temp }}/zookeeper-apache + tags: zookeeper:ci + load: true + cache-from: type=gha,scope=zookeeper-${{ matrix.arch }} + cache-to: ${{ github.event_name == 'push' && format('type=gha,mode=max,scope=zookeeper-{0}', matrix.arch) || '' }} + + - name: Smoke test image + env: + ZK_VERSION: ${{ steps.zk.outputs.zk_version }} + run: | + docker run --rm --entrypoint bash -e ZK_VERSION zookeeper:ci -ec ' + java -version + java -XshowSettings:properties -version 2>&1 | grep -Eq "^[[:space:]]*java.specification.version = ${JAVA_VERSION}$" + zkServer.sh version 2>/dev/null | tee /dev/stderr | grep -q "${ZK_VERSION}" + test -s /opt/libs/zu.jar + set +e + zu_output="$(java -jar /opt/libs/zu.jar 2>&1)" + zu_status=$? + set -e + printf "%s\n" "$zu_output" + test "$zu_status" -eq 1 + grep -q "^Usage: zu " <<<"$zu_output" + for s in zookeeperStart.sh zookeeperReady.sh zookeeperLive.sh zookeeperTeardown.sh zookeeperMetrics.sh; do + test -x "/usr/local/bin/${s}" + done + command -v dig curl socat pgrep >/dev/null + ' + - name: Verify patched JVM in the running ZooKeeper process + run: | + docker run --rm --entrypoint bash \ + -v "${PWD}/scripts/test-zookeeper-image.sh:/test-zookeeper-image.sh:ro" \ + zookeeper:ci /test-zookeeper-image.sh diff --git a/Makefile b/Makefile index eb3aad7ef..d8ba1c078 100644 --- a/Makefile +++ b/Makefile @@ -151,9 +151,18 @@ docker-buildx: ## Build and push docker image for the manager for cross-platform - docker buildx rm zookeeper-builder rm Dockerfile.cross -build-zk-image: +JAVA_DOWNLOAD_BASE_URL ?= https://download.oracle.com/java +JAVA_DOWNLOAD_VERSION ?= 21 - docker build --build-arg VERSION=$(VERSION) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) --build-arg GIT_SHA=$(GIT_SHA) -t $(APP_REPO):$(VERSION) ./docker +build-zk-image: + zk_version="$$(./scripts/zookeeper-version.sh)" && \ + docker build --build-arg ZK_VERSION="$$zk_version" \ + --build-arg java_download_base_url="$(JAVA_DOWNLOAD_BASE_URL)" \ + --build-arg java_download_version="$(JAVA_DOWNLOAD_VERSION)" \ + --secret id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY \ + -t $(APP_REPO):$(VERSION)-apache ./docker/zookeeper-image + docker build --build-arg VERSION=$(VERSION) --build-arg DOCKER_REGISTRY=$(DOCKER_REGISTRY) --build-arg GIT_SHA=$(GIT_SHA) \ + --build-arg BASE_IMAGE=$(APP_REPO):$(VERSION)-apache -t $(APP_REPO):$(VERSION) ./docker docker tag $(APP_REPO):$(VERSION) $(APP_REPO):latest build-zk-image-swarm: diff --git a/docker/Dockerfile b/docker/Dockerfile index 769601218..4bdd8a046 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,3 +1,5 @@ +# syntax=docker/dockerfile:1 +# check=skip=InvalidDefaultArgInFrom # # Copyright (c) 2018 Dell Inc., or its subsidiaries. All Rights Reserved. # @@ -8,26 +10,37 @@ # http://www.apache.org/licenses/LICENSE-2.0 # -FROM gradle:jdk21 +# Base Apache ZooKeeper + OpenJDK image built from docker/zookeeper-image. +# It is not pinned here: the build-zookeeper release job builds that image +# first and passes its digest in, so both always come from the same commit +# and shared zookeeperVersion. For local builds use `make build-zk-image`. +ARG BASE_IMAGE + +FROM ${BASE_IMAGE} AS runtime + +# zu.jar is architecture-independent bytecode, so build it once on the native +# build platform instead of re-running Gradle under emulation for every target. +FROM --platform=$BUILDPLATFORM gradle:jdk25 AS zu-builder RUN mkdir /zu COPY zu /zu WORKDIR /zu -RUN ./gradlew shadowJar --no-daemon +RUN --mount=type=bind,from=runtime,source=/usr/local,target=/runtime-java \ + java_version="$(sed -n 's/^JAVA_VERSION="\([0-9][0-9]*\).*/\1/p' /runtime-java/openjdk-*/release)" && \ + test -n "$java_version" && \ + ./gradlew -PjavaVersion="$java_version" shadowJar --no-daemon -# use forked base zookeeper 3.8.4 docker image -# that runs on openjdk-21 -FROM adobe/zookeeper:3.8.4-apache-zk-20250914 +FROM runtime COPY bin /usr/local/bin RUN chmod +x /usr/local/bin/* -COPY --from=0 /zu/build/libs/zu.jar /opt/libs/ +COPY --from=zu-builder /zu/build/libs/zu.jar /opt/libs/ RUN apt-get -q update && \ apt-get install --no-install-recommends -y dnsutils curl procps socat # OCI standard labels LABEL org.opencontainers.image.title="Apache Zookeeper" -LABEL org.opencontainers.image.description="Apache Zookeeper image build with java-21" +LABEL org.opencontainers.image.description="Apache Zookeeper image build with java-${JAVA_VERSION}" LABEL org.opencontainers.image.source="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.url="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.documentation="https://github.com/adobe/zookeeper-operator" diff --git a/docker/bin/zookeeperFunctions.sh b/docker/bin/zookeeperFunctions.sh index 8c7f0baf9..663e606e0 100644 --- a/docker/bin/zookeeperFunctions.sh +++ b/docker/bin/zookeeperFunctions.sh @@ -12,6 +12,14 @@ set -ex +function zkRunZu() { + # JVM option variables intentionally contain multiple arguments. + # shellcheck disable=SC2086 + java -Xmx"${ZK_CLIENT_HEAP:-256}m" ${CLIENT_JVMFLAGS:-} \ + -Dlogback.configurationFile=/conf/logback-quiet.xml ${ZK_OPTIONS:-} \ + -jar /opt/libs/zu.jar "$@" +} + function zkConfig() { if [ -n "$1" ]; then FQDN="$1" diff --git a/docker/zookeeper-image/Dockerfile b/docker/zookeeper-image/Dockerfile index 5c098232a..13513fba9 100644 --- a/docker/zookeeper-image/Dockerfile +++ b/docker/zookeeper-image/Dockerfile @@ -1,5 +1,11 @@ +# syntax=docker/dockerfile:1 + # backported from https://github.com/docker-library/openjdk/blob/4c2b1a57869e24c5fcfbea6a17e558c710d4f6bd/21/jdk/slim-bullseye/Dockerfile -FROM debian:bullseye-slim +FROM debian:trixie-slim + +ARG java_version=21 +ARG java_download_version=${java_version} +ARG java_download_base_url=https://download.oracle.com/java RUN set -eux; \ apt-get update; \ @@ -8,24 +14,30 @@ RUN set -eux; \ ; \ rm -rf /var/lib/apt/lists/* -ENV JAVA_HOME /usr/local/openjdk-21 -ENV PATH $JAVA_HOME/bin:$PATH +ENV JAVA_HOME=/usr/local/openjdk-${java_version} +ENV PATH=$JAVA_HOME/bin:$PATH # Default to UTF-8 file.encoding -ENV LANG C.UTF-8 -ENV JAVA_VERSION 21 +ENV LANG=C.UTF-8 +ENV JAVA_VERSION=${java_version} -RUN set -eux; \ +RUN --mount=type=secret,id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY \ + set -eux; \ + \ + case "$java_download_base_url" in \ + https://download.oracle.com/*) java_download_base_url="${java_download_base_url}/${java_version}/latest" ;; \ + *) java_download_base_url="${java_download_base_url}/${java_version}" ;; \ + esac; \ \ arch="$(dpkg --print-architecture)"; \ case "$arch" in \ 'amd64') \ - downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz'; \ - downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-x64_bin.tar.gz.sha256'; \ + downloadUrl="${java_download_base_url}/jdk-${java_download_version}_linux-x64_bin.tar.gz"; \ + downloadSha256="${downloadUrl}.sha256"; \ ;; \ 'arm64') \ - downloadUrl='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz'; \ - downloadSha256='https://download.oracle.com/java/21/latest/jdk-21_linux-aarch64_bin.tar.gz.sha256'; \ + downloadUrl="${java_download_base_url}/jdk-${java_download_version}_linux-aarch64_bin.tar.gz"; \ + downloadSha256="${downloadUrl}.sha256"; \ ;; \ *) echo >&2 "error: unsupported architecture: '$arch'"; exit 1 ;; \ esac; \ @@ -33,12 +45,16 @@ RUN set -eux; \ savedAptMark="$(apt-mark showmanual)"; \ apt-get update; \ apt-get install -y --no-install-recommends \ - wget \ + curl \ ; \ rm -rf /var/lib/apt/lists/*; \ \ - wget --progress=dot:giga -O openjdk.tgz "$downloadUrl"; \ - SHA256=$(wget -qO- $downloadSha256) ; \ + set --; \ + if [ -n "${JAVA_DOWNLOAD_APIKEY:+set}" ]; then \ + set -- --variable '%JAVA_DOWNLOAD_APIKEY' --expand-header 'Authorization: Bearer {{JAVA_DOWNLOAD_APIKEY}}'; \ + fi; \ + curl --fail --location "$@" --output openjdk.tgz "$downloadUrl"; \ + SHA256="$(curl --fail --silent --show-error --location "$@" "$downloadSha256")"; \ echo "$SHA256 *openjdk.tgz" | sha256sum --strict --check -; \ \ mkdir -p "$JAVA_HOME"; \ @@ -109,14 +125,15 @@ RUN set -eux; \ dirmngr \ gosu \ gnupg \ - netcat \ + netcat-openbsd \ wget; \ rm -rf /var/lib/apt/lists/*; \ # Verify that gosu binary works gosu nobody true -ARG SHORT_DISTRO_NAME=zookeeper-3.8.4 -ARG DISTRO_NAME=apache-zookeeper-3.8.4-bin +ARG ZK_VERSION=3.8.4 +ARG SHORT_DISTRO_NAME=zookeeper-${ZK_VERSION} +ARG DISTRO_NAME=apache-zookeeper-${ZK_VERSION}-bin # Download Apache Zookeeper, verify its PGP signature, untar and clean up RUN set -eux; \ @@ -126,6 +143,7 @@ RUN set -eux; \ local success=; \ local distUrl=; \ for distUrl in \ + https://dlcdn.apache.org/ \ https://downloads.apache.org/ \ https://www-us.apache.org/dist/ \ https://www.apache.org/dist/ \ @@ -150,7 +168,7 @@ RUN set -eux; \ chown -R zookeeper:zookeeper "/$DISTRO_NAME" COPY opt/zkEnv.sh $DISTRO_NAME/bin/zkEnv.sh -WORKDIR $DISTRO_NAME +WORKDIR /$DISTRO_NAME VOLUME ["$ZOO_DATA_DIR", "$ZOO_DATA_LOG_DIR", "$ZOO_LOG_DIR"] EXPOSE 2181 2888 3888 8080 @@ -160,13 +178,13 @@ ENV PATH=$PATH:/$DISTRO_NAME/bin \ # OCI standard labels LABEL org.opencontainers.image.title="Apache Zookeeper" -LABEL org.opencontainers.image.description="Apache Zookeeper 3.8.4 with OpenJDK 21" +LABEL org.opencontainers.image.description="Apache Zookeeper ${ZK_VERSION} with OpenJDK ${java_version}" LABEL org.opencontainers.image.source="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.url="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.documentation="https://github.com/adobe/zookeeper-operator" LABEL org.opencontainers.image.vendor="Adobe" LABEL org.opencontainers.image.licenses="Apache-2.0" -LABEL org.opencontainers.image.version="3.8.4" +LABEL org.opencontainers.image.version="${ZK_VERSION}" COPY docker-entrypoint.sh / ENTRYPOINT ["/docker-entrypoint.sh"] diff --git a/docker/zookeeper-image/README.md b/docker/zookeeper-image/README.md new file mode 100644 index 000000000..4048784bb --- /dev/null +++ b/docker/zookeeper-image/README.md @@ -0,0 +1,102 @@ +# Apache ZooKeeper base image + +Build from the repository root: + +```bash +docker buildx build --load \ + --build-arg ZK_VERSION="$(./scripts/zookeeper-version.sh)" \ + -t zookeeper-apache:local docker/zookeeper-image +``` + +The image uses Debian Trixie and this hotfix defaults to Java 21. Oracle's public JDK +download URL is the default, without authentication. + +## Alternate JDK downloads + +Override `java_download_base_url` and `java_download_version` to use another +source. The Dockerfile appends `java_version` (default: `21`) and `latest` +for Oracle, or only `java_version` for alternate sources. Do not include +the Java-major directory in the base URL. Alternate sources must +provide Oracle-style archive filenames and matching `.sha256` files for +each architecture (`x64` and `aarch64`). + +For example, add these options to the build command: + +```bash +--build-arg java_download_base_url=https://jdk.example.com/java \ +--build-arg java_version=21 \ +--build-arg java_download_version=21.0.13.1 +``` + +This requests `/java/21/jdk-21.0.13.1_linux-_bin.tar.gz` +and its `.sha256` file. Both downloads must succeed and the checksum must match. + +For authenticated downloads, pass an optional API key as a BuildKit secret: + +```bash +--secret id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY +``` + +The key is sent as an `Authorization: Bearer` header, is not stored in the +image, and is not forwarded to a different host on redirects. + +This hotfix enables the authenticated override in `.github/workflows/release.yml` +and selects Oracle JDK `21.0.13.1`. Configure repository secrets +`JAVA_DOWNLOAD_BASE_URL` and `JAVA_DOWNLOAD_APIKEY` before publishing. The base +URL must be the approved raw Image Factory repository, ending in +`generic-ifup-resources-release/java/oracle`, without a version directory. +The credential is passed only as a BuildKit secret. + +## Full ZooKeeper image + +After building the base image above, build the operator-flavoured image: + +```bash +docker buildx build --load \ + --build-arg BASE_IMAGE=zookeeper-apache:local \ + -t zookeeper:local docker +``` + +The full image reads the base image's JDK release metadata and uses its +Java major version for both Java and Kotlin compilation targets in `zu`. +Set `java_version` only when building the base image; no second version +argument is needed for the full image. + +The native Gradle builder uses JDK 25 independently of the selected runtime. +Direct Gradle builds, including `./gradlew build --no-daemon`, read the +default from the same base Dockerfile. An explicit `-PjavaVersion=` +overrides that target when needed. + +## Production hotfix + +The branch starts at production tag `0.2.15-adobe-20260225`. The four commits +from [PR #113](https://github.com/adobe/zookeeper-operator/pull/113) are retained +as separate cherry-picks with their original file names. Emergency overrides +and old-release compatibility changes are a separate commit. + +ZooKeeper remains `3.8.4`; the existing `zu` client dependency remains `3.9.3`. +The shared server version resolver and helper shell function are prerequisites +used by the unchanged upstream tests. The native builder retains JDK 25 from +upstream, with its matching Gradle/Kotlin/Shadow toolchain; it does not ship in +the runtime image. The compilation target follows the selected Java 21 base. + +Oracle identifies `21.0.13.1+4` as the fixed JDK 21 build for +[CVE-2026-114938](https://www.oracle.com/security-alerts/cspunov2026.html#AppendixJAVA); +see the [JDK release notes](https://www.oracle.com/java/technologies/javase/21all-relnotes.html#R21_0_13_1). +Confirm runner access and Oracle redistribution approval before publication. + +Run `python3 -B -m unittest discover -s scripts -p 'test_*.py'` for all upstream +build-script tests. Run `zookeeper-images` manually on the reviewed hotfix ref +to build both architectures with the same approved JDK override, without +publication. The additional smoke test verifies the exact Oracle build in the +running ZooKeeper process, server readiness and live `zu` access. + +This old release line publishes on **tag push**, not on GitHub Release +publication. Use a new Helm-compatible tag on the reviewed hotfix commit; +never move an existing tag. A regular tag (not starting with `zk`) rebuilds +the Apache base and pins the wrapper to its digest. It also publishes the +operator image and Helm chart, which need not be deployed for this runtime fix. + +The final server image is `adobe/zookeeper:3.8.4-`. +Verify the published digest and JVM before updating `pipeline-config`. +Publishing does not roll out the image to existing clusters. diff --git a/docker/zu/build.gradle.kts b/docker/zu/build.gradle.kts index 784580a60..dc75d7fc2 100644 --- a/docker/zu/build.gradle.kts +++ b/docker/zu/build.gradle.kts @@ -1,20 +1,31 @@ import com.github.jengelman.gradle.plugins.shadow.tasks.ShadowJar import org.jetbrains.kotlin.gradle.tasks.KotlinCompile +import org.jetbrains.kotlin.gradle.dsl.JvmTarget plugins { - kotlin("jvm") version "1.9.21" - id("com.github.johnrengelman.shadow") version "8.1.1" + kotlin("jvm") version "2.4.21" + id("com.gradleup.shadow") version "9.6.1" } repositories { mavenCentral() } +val javaVersion = JavaVersion.toVersion( + providers.gradleProperty("javaVersion").orElse(providers.provider { + file("../zookeeper-image/Dockerfile").readLines() + .single { it.startsWith("ARG java_version=") } + .substringAfter("=") + }).get() +) + java { - sourceCompatibility = JavaVersion.VERSION_21 - targetCompatibility = JavaVersion.VERSION_21 + sourceCompatibility = javaVersion + targetCompatibility = javaVersion } +val zookeeperVersion = "3.8.4" + dependencies { implementation(kotlin("stdlib")) implementation("org.apache.zookeeper:zookeeper:3.9.3") @@ -22,13 +33,16 @@ dependencies { tasks.withType() { archiveClassifier.set("") + filesMatching("META-INF/*.kotlin_module") { + duplicatesStrategy = DuplicatesStrategy.INCLUDE + } manifest { attributes["Main-Class"] = "io.pravega.zookeeper.MainKt" } } tasks.withType { - kotlinOptions { - jvmTarget = "21" + compilerOptions { + jvmTarget.set(JvmTarget.fromTarget(javaVersion.majorVersion)) } } diff --git a/docker/zu/gradle/wrapper/gradle-wrapper.properties b/docker/zu/gradle/wrapper/gradle-wrapper.properties index a59520664..16e57d81a 100644 --- a/docker/zu/gradle/wrapper/gradle-wrapper.properties +++ b/docker/zu/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.5-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-9.8.2-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/scripts/test-zookeeper-image.sh b/scripts/test-zookeeper-image.sh new file mode 100644 index 000000000..fc314fab7 --- /dev/null +++ b/scripts/test-zookeeper-image.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +set -euo pipefail + +java -version +properties="$(java -XshowSettings:properties -version 2>&1)" +grep -Eq '^[[:space:]]*java.version = 21\.0\.13\.1$' <<<"$properties" +grep -Eq '^[[:space:]]*java.runtime.version = 21\.0\.13\.1\+4([-[:space:]]|$)' <<<"$properties" +grep -Eq '^[[:space:]]*java.vendor = Oracle Corporation$' <<<"$properties" +zkServer.sh version 2>/dev/null | tee /dev/stderr | grep -q '3\.8\.4' + +set +e +zu_output="$(java -jar /opt/libs/zu.jar 2>&1)" +zu_status=$? +set -e +printf '%s\n' "$zu_output" +test "$zu_status" -eq 1 +grep -q '^Usage: zu ' <<<"$zu_output" + +for script in zookeeperStart.sh zookeeperReady.sh zookeeperLive.sh zookeeperTeardown.sh zookeeperMetrics.sh; do + test -x "/usr/local/bin/$script" +done +command -v dig curl socat pgrep nc >/dev/null + +cat > "$ZOOCFGDIR/zoo.cfg" < "$ZOO_DATA_DIR/myid" +zkServer.sh start +trap 'zkServer.sh stop' EXIT +ready=false +for attempt in $(seq 1 60); do + if [[ "$(printf ruok | nc -w 1 127.0.0.1 2181)" == imok ]]; then + ready=true + break + fi + sleep 1 +done +if [[ "$ready" != true ]]; then + echo "ZooKeeper did not become ready" >&2 + exit 1 +fi +zk_pid="$(pgrep -f 'org.apache.zookeeper.server.quorum.QuorumPeerMain')" +timeout 30 jcmd "$zk_pid" VM.version | tee /dev/stderr | grep -Eq '21\.0\.13\.1\+4' +timeout 30 jcmd "$zk_pid" VM.system_properties | grep -Fx "java.home=$JAVA_HOME" +timeout 30 java -jar /opt/libs/zu.jar get-all 127.0.0.1:2181 diff --git a/scripts/test_zookeeper_version.py b/scripts/test_zookeeper_version.py new file mode 100644 index 000000000..dc483ae78 --- /dev/null +++ b/scripts/test_zookeeper_version.py @@ -0,0 +1,221 @@ +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class JavaDownloadTest(unittest.TestCase): + def run_download(self, arch="amd64", base="https://download.oracle.com/java", + major="25", version="25", api_key=None, corrupt=False): + dockerfile = (ROOT / "docker/zookeeper-image/Dockerfile").read_text() + download = dockerfile.split("ENV JAVA_VERSION=", 1)[1].split( + "RUN ", 1 + )[1].split('mkdir -p "$JAVA_HOME"', 1)[0] + if download.startswith("--mount="): + download = download.split("\\\n", 1)[1] + download = download.replace("\\\n", "").rstrip() + "\n" + variables = {key: value for key, value in os.environ.items() + if key != "JAVA_DOWNLOAD_APIKEY"} + variables.update( + ARCH=arch, java_version=major, java_download_version=version, + java_download_base_url=base, + DOWNLOAD_SHA=hashlib.sha256( + b"wrong" if corrupt else b"artifact" + ).hexdigest(), + ) + if api_key is not None: + variables["JAVA_DOWNLOAD_APIKEY"] = api_key + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + ["bash", "-ec", + 'dpkg() { printf "%s\\n" "$ARCH"; }\n' + 'apt-mark() { :; }\n' + 'apt-get() { :; }\n' + 'rm() { :; }\n' + 'wget() { return 99; }\n' + 'curl() {\n' + ' python3 -c \'import json, sys; ' + 'open("downloads.jsonl", "a").write(' + 'json.dumps(sys.argv[1:]) + "\\n")\' "$@"\n' + ' for url; do :; done\n' + ' case "$url" in\n' + ' *.sha256) printf "%s\\n" "$DOWNLOAD_SHA" ;;\n' + ' *) printf artifact > openjdk.tgz ;;\n' + ' esac\n' + '}\n' + download], + cwd=directory, env=variables, text=True, capture_output=True, + ) + log = Path(directory) / "downloads.jsonl" + calls = [json.loads(line) for line in log.read_text().splitlines() + ] if log.exists() else [] + return result, calls + + def test_oracle_defaults_preserve_both_architecture_urls(self): + for arch, name in (("amd64", "x64"), ("arm64", "aarch64")): + with self.subTest(arch=arch): + result, calls = self.run_download(arch=arch) + self.assertEqual(result.returncode, 0, result.stderr) + url = ("https://download.oracle.com/java/25/latest/" + f"jdk-25_linux-{name}_bin.tar.gz") + self.assertEqual([call[-1] for call in calls], + [url, url + ".sha256"]) + self.assertNotIn("--expand-header", calls[0]) + + def test_alternate_source_uses_patch_version_and_optional_authentication(self): + for arch, name in (("amd64", "x64"), ("arm64", "aarch64")): + for key in (None, "synthetic-download-key"): + with self.subTest(arch=arch, authenticated=key is not None): + result, calls = self.run_download( + arch=arch, base="https://jdk.example.test/java", + major="21", version="21.0.13.1", api_key=key, + ) + self.assertEqual(result.returncode, 0, result.stderr) + url = ("https://jdk.example.test/java/21/" + f"jdk-21.0.13.1_linux-{name}_bin.tar.gz") + self.assertEqual([call[-1] for call in calls], + [url, url + ".sha256"]) + for call in calls: + self.assertIn("--fail", call) + self.assertIn("--location", call) + self.assertEqual("--expand-header" in call, key is not None) + if key is not None: + self.assertIn("%JAVA_DOWNLOAD_APIKEY", call) + self.assertIn("Authorization: Bearer {{JAVA_DOWNLOAD_APIKEY}}", + call) + self.assertNotIn(key, " ".join(call)) + self.assertNotIn(key, result.stderr) + + def test_checksum_mismatch_stops_the_build(self): + result, _ = self.run_download(corrupt=True) + self.assertNotEqual(result.returncode, 0) + self.assertIn("FAILED", result.stdout) + + def test_unsupported_architecture_stops_before_downloading(self): + result, calls = self.run_download(arch="s390x") + self.assertNotEqual(result.returncode, 0) + self.assertIn("unsupported architecture", result.stderr) + self.assertEqual(calls, []) + + +class ZookeeperDownloadTest(unittest.TestCase): + def run_download(self, available_url, empty=False): + dockerfile = (ROOT / "docker/zookeeper-image/Dockerfile").read_text() + helper = "ddist() {" + dockerfile.split("ddist() {", 1)[1].split( + " ddist ", 1 + )[0] + helper = helper.replace("\\\n", "") + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + ["sh", "-ec", + 'wget() {\n' + ' while [ "$#" -gt 0 ]; do\n' + ' case "$1" in\n' + ' -O) shift; output="$1" ;;\n' + ' https://*) url="$1" ;;\n' + ' esac\n' + ' shift\n' + ' done\n' + ' [ "$url" = "$AVAILABLE_URL" ] || return 4\n' + ' : > "$output"\n' + ' [ "$EMPTY" = 1 ] || printf artifact > "$output"\n' + '}\n' + helper + '\nddist artifact.tgz zookeeper/release.tgz'], + cwd=directory, + env={**os.environ, "AVAILABLE_URL": available_url, + "EMPTY": "1" if empty else "0"}, + text=True, capture_output=True, + ) + target = Path(directory) / "artifact.tgz" + content = target.read_text() if target.exists() else None + return result, content + + def test_downloads_from_cdn_when_origin_is_unavailable(self): + result, content = self.run_download( + "https://dlcdn.apache.org/zookeeper/release.tgz" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(content, "artifact") + + def test_falls_back_to_archive_for_older_releases(self): + result, content = self.run_download( + "https://archive.apache.org/dist/zookeeper/release.tgz" + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(content, "artifact") + + def test_fails_when_all_sources_are_unavailable_or_empty(self): + for empty in (False, True): + with self.subTest(empty=empty): + result, _ = self.run_download( + "https://archive.apache.org/dist/zookeeper/release.tgz" + if empty else "", empty=empty, + ) + self.assertNotEqual(result.returncode, 0) + + +class ZookeeperVersionTest(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + self.addCleanup(self.temp.cleanup) + self.root = Path(self.temp.name) + for name in ("Makefile", "docker/zu/build.gradle.kts", + "scripts/zookeeper-version.sh"): + target = self.root / name + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(ROOT / name, target) + self.buildfile = self.root / "docker/zu/build.gradle.kts" + self.bin = self.root / "bin" + self.bin.mkdir() + docker = self.bin / "docker" + docker.write_text( + "#!/usr/bin/env python3\n" + "import json, sys\n" + "with open('docker-commands.log', 'a') as log:\n" + " log.write(json.dumps(sys.argv[1:]) + '\\n')\n" + ) + docker.chmod(0o755) + + def run_make(self): + return subprocess.run( + ["make", "build-zk-image", "VERSION=test", "GIT_SHA=test"], + cwd=self.root, + env={**os.environ, "PATH": f"{self.bin}:{os.environ['PATH']}"}, + text=True, + capture_output=True, + ) + + def test_reads_shared_version_outside_repository_cwd(self): + self.buildfile.write_text('val zookeeperVersion = "9.8.7"\n') + result = subprocess.run( + ["bash", str(self.root / "scripts/zookeeper-version.sh")], + cwd=self.bin, text=True, capture_output=True, + ) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout, "9.8.7\n") + + def test_make_passes_updated_version_to_server_build(self): + self.buildfile.write_text('val zookeeperVersion = "9.8.7"\n') + result = self.run_make() + self.assertEqual(result.returncode, 0, result.stderr) + commands = [ + json.loads(line) + for line in (self.root / "docker-commands.log").read_text().splitlines() + ] + self.assertIn("ZK_VERSION=9.8.7", commands[0]) + + def test_invalid_version_stops_before_docker_build(self): + for value in ("", 'val zookeeperVersion = ""\n', + 'val zookeeperVersion = "latest"\n', + 'val zookeeperVersion = "3.10.0-rc1"\n', + 'val zookeeperVersion = "3.9.6"\nval zookeeperVersion = "3.10.0"\n'): + with self.subTest(value=value): + self.buildfile.write_text(value) + result = self.run_make() + self.assertNotEqual(result.returncode, 0) + self.assertFalse((self.root / "docker-commands.log").exists()) diff --git a/scripts/test_zu_runtime.py b/scripts/test_zu_runtime.py new file mode 100644 index 000000000..3c481c6c3 --- /dev/null +++ b/scripts/test_zu_runtime.py @@ -0,0 +1,103 @@ +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +ROOT = Path(__file__).resolve().parents[1] + + +class ZuJavaVersionTest(unittest.TestCase): + def run_build(self, release): + dockerfile = (ROOT / "docker/Dockerfile").read_text() + build = dockerfile.split("WORKDIR /zu\n", 1)[1].split( + "\nFROM ", 1 + )[0].strip().removeprefix("RUN ") + if build.startswith("--mount="): + build = build.split("\\\n", 1)[1] + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + jdk = root / "runtime-java/openjdk-selected" + jdk.mkdir(parents=True) + (jdk / "release").write_text(release) + gradle = root / "gradlew" + gradle.write_text('#!/bin/sh\nprintf "%s\\n" "$@"\n') + gradle.chmod(0o700) + return subprocess.run( + ["sh", "-ec", build.replace("\\\n", "").replace( + "/runtime-java", str(root / "runtime-java") + )], + cwd=root, text=True, capture_output=True, + ) + + def test_compilation_target_follows_base_image_jdk(self): + for release, major in (('JAVA_VERSION="21.0.13.1"\n', "21"), + ('JAVA_VERSION="25.0.5.1"\n', "25")): + with self.subTest(major=major): + result = self.run_build(release) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(result.stdout.splitlines(), [ + f"-PjavaVersion={major}", "shadowJar", "--no-daemon", + ]) + + def test_missing_java_version_stops_before_gradle(self): + result = self.run_build('IMPLEMENTOR="Oracle Corporation"\n') + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, "") + + +class ZuRuntimeTest(unittest.TestCase): + def run_zu(self, env=None, java_body=None): + variables = { + key: value for key, value in os.environ.items() + if key not in ("ZK_CLIENT_HEAP", "CLIENT_JVMFLAGS", "ZK_OPTIONS") + } + variables.update(env or {}) + java_body = java_body or ( + """python3 -c 'import json, sys; print(json.dumps(sys.argv[1:]))' "$@";""" + ) + return subprocess.run( + ["bash", "-c", + 'source "$1"\n' + f"java() {{ {java_body} }}\n" + 'shift\n' + 'zkRunZu "$@"', + "test", str(ROOT / "docker/bin/zookeeperFunctions.sh"), + "sync", "localhost:2181", "/path with spaces"], + env=variables, text=True, capture_output=True, + ) + + def test_default_heap_and_quiet_logging_preserve_helper_arguments(self): + result = self.run_zu() + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), [ + "-Xmx256m", + "-Dlogback.configurationFile=/conf/logback-quiet.xml", + "-jar", "/opt/libs/zu.jar", + "sync", "localhost:2181", "/path with spaces", + ]) + + def test_custom_heap_jvm_and_tls_options_are_forwarded(self): + result = self.run_zu({ + "ZK_CLIENT_HEAP": "384", + "CLIENT_JVMFLAGS": "-Xmx512m -Dcustom.option=value", + "ZK_OPTIONS": ( + "-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty " + "-Dzookeeper.client.secure=true" + ), + }) + self.assertEqual(result.returncode, 0, result.stderr) + self.assertEqual(json.loads(result.stdout), [ + "-Xmx384m", "-Xmx512m", "-Dcustom.option=value", + "-Dlogback.configurationFile=/conf/logback-quiet.xml", + "-Dzookeeper.clientCnxnSocket=org.apache.zookeeper.ClientCnxnSocketNetty", + "-Dzookeeper.client.secure=true", + "-jar", "/opt/libs/zu.jar", + "sync", "localhost:2181", "/path with spaces", + ]) + + def test_helper_failure_is_not_swallowed(self): + result = self.run_zu(java_body="return 7;") + self.assertEqual(result.returncode, 7, result.stderr) diff --git a/scripts/zookeeper-version.sh b/scripts/zookeeper-version.sh new file mode 100755 index 000000000..46c7073f3 --- /dev/null +++ b/scripts/zookeeper-version.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +version="$(sed -n 's/^val zookeeperVersion = "\([^"]*\)"$/\1/p' "$(dirname "$0")/../docker/zu/build.gradle.kts")" +if ! [[ "$version" =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]]; then + echo "Expected one stable zookeeperVersion in docker/zu/build.gradle.kts" >&2 + exit 1 +fi +printf '%s\n' "$version"