diff --git a/.github/workflows/publish-kafka-docker.yml b/.github/workflows/publish-kafka-docker.yml index 5405fdb57..264ca9fa6 100644 --- a/.github/workflows/publish-kafka-docker.yml +++ b/.github/workflows/publish-kafka-docker.yml @@ -53,6 +53,28 @@ jobs: run: | KAFKA_VERSION=$(echo "${{ needs.prepare.outputs.tag }}" | sed 's/kafka-//') echo "kafka_version=$KAFKA_VERSION" >> $GITHUB_OUTPUT + - name: Extract Java version from Dockerfile + id: java_version + run: | + JAVA_VERSION=$(sed -nE 's/^ARG java_version=([0-9]+).*$/\1/p' docker/kafka/Dockerfile) + echo "java_version=$JAVA_VERSION" >> $GITHUB_OUTPUT + - name: Validate tagged Kafka distribution + env: + IMAGE_VERSION: ${{ steps.kafka_version.outputs.kafka_version }} + run: | + set -euo pipefail + if [[ ! "$IMAGE_VERSION" =~ ^([0-9]+\.[0-9]+)-([0-9]+\.[0-9]+\.[0-9]+)(-rc[0-9]+)?(-jdk[0-9]+\.[0-9]+\.[0-9]+)?$ ]]; then + echo "::error::Expected kafka--[-rc][-jdk] release tag" + exit 1 + fi + TAG_SCALA="${BASH_REMATCH[1]}" + TAG_KAFKA="${BASH_REMATCH[2]}" + DIST_SCALA=$(sed -nE 's/^ARG scala_version=([^[:space:]]+).*$/\1/p' docker/kafka/Dockerfile) + DIST_KAFKA=$(sed -nE 's/^ARG KAFKA_VERSION=([^[:space:]]+).*$/\1/p' docker/kafka/Dockerfile) + if [[ "$TAG_SCALA" != "$DIST_SCALA" || "$TAG_KAFKA" != "$DIST_KAFKA" ]]; then + echo "::error::Tag $IMAGE_VERSION does not match Dockerfile distribution $DIST_SCALA-$DIST_KAFKA" + exit 1 + fi - name: Check if Kafka image already exists id: image_exists run: | @@ -91,7 +113,7 @@ jobs: tags: | type=match,pattern=kafka-(.*),group=1 labels: | - org.opencontainers.image.description=Apache Kafka with OpenJDK 21 + org.opencontainers.image.description=Apache Kafka with Oracle JDK ${{ steps.java_version.outputs.java_version }} - name: Build and push kafka image if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false' uses: docker/build-push-action@v7 @@ -128,14 +150,15 @@ jobs: id: jmx_exporter_version run: | JMX_VERSION=$(grep "ARG JMX_EXPORTER_VERSION=" docker/jmx_exporter/Dockerfile | cut -d'=' -f2 | cut -d' ' -f1) - # Strip a leading "v" so the published image tag is always version-only, - # regardless of whether the upstream release tag (and thus the Dockerfile ARG) has one. + # Normalize the upstream version before adding the JDK rebuild suffix. JMX_VERSION="${JMX_VERSION#v}" + JAVA_VERSION=$(sed -nE 's/^ARG java_version=([0-9]+).*$/\1/p' docker/kafka/Dockerfile) echo "jmx_version=$JMX_VERSION" >> $GITHUB_OUTPUT + echo "jmx_image_tag=${JMX_VERSION}-jdk${JAVA_VERSION}" >> $GITHUB_OUTPUT - name: Check if JMX exporter image already exists id: image_exists run: | - IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}:${{ steps.jmx_exporter_version.outputs.jmx_version }}" + IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}:${{ steps.jmx_exporter_version.outputs.jmx_image_tag }}" echo "Checking if image exists: $IMAGE_NAME" # Try to inspect the manifest using docker @@ -161,7 +184,7 @@ jobs: images: | ${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }} tags: | - type=raw,value=${{ steps.jmx_exporter_version.outputs.jmx_version }} + type=raw,value=${{ steps.jmx_exporter_version.outputs.jmx_image_tag }} labels: | org.opencontainers.image.description=Prometheus JMX Exporter for Kafka monitoring - name: Build and push JMX exporter image diff --git a/docker/jmx_exporter/Dockerfile b/docker/jmx_exporter/Dockerfile index f77cff91f..b4080ea6a 100644 --- a/docker/jmx_exporter/Dockerfile +++ b/docker/jmx_exporter/Dockerfile @@ -1,10 +1,11 @@ ARG JMX_EXPORTER_VERSION=1.7.0 # renovate: datasource=github-releases depName=prometheus/jmx_exporter - -FROM maven:3-amazoncorretto-21@sha256:f64fd4f0363c918004816d83d2589495b622a8d2fc5c5602c135cb4124c7ea12 AS build +ARG BUILDPLATFORM +# The Java agent JAR is architecture-independent; compile and test without QEMU. +FROM --platform=$BUILDPLATFORM maven:3-amazoncorretto-25@sha256:d96c6ff2e7a3258e6f0c447340d2284a3404917bb8c5f01c8660cc4066bd382c AS build ARG JMX_EXPORTER_VERSION -# Install wget to download the release tarball -RUN yum install -y wget && yum clean all +# The Java 25 builder does not include tar or gzip. +RUN yum install -y wget tar gzip && yum clean all # Download and extract the JMX exporter release RUN wget https://github.com/prometheus/jmx_exporter/archive/refs/tags/${JMX_EXPORTER_VERSION}.tar.gz \ @@ -14,6 +15,7 @@ RUN wget https://github.com/prometheus/jmx_exporter/archive/refs/tags/${JMX_EXPO rm /tmp/jmx_exporter.tar.gz WORKDIR /src +# Keep the agent compatible with Java 21 consumers while building with JDK 25. RUN mvn -B -Dmaven.javadoc.skip=true -Dskip.javadoc=true \ -Dmaven.compiler.source=21 -Dmaven.compiler.target=21 -Dmaven.compiler.release=21 \ clean package -pl jmx_prometheus_javaagent -am && \ diff --git a/docker/kafka/Dockerfile b/docker/kafka/Dockerfile index 3e73cdaca..ff97ec707 100644 --- a/docker/kafka/Dockerfile +++ b/docker/kafka/Dockerfile @@ -1,12 +1,9 @@ # syntax=docker/dockerfile:1 ARG scala_version=2.13 -# Renovate is pinned to the 3.x line (see the apache/kafka packageRule in renovate.json). -# Kafka 4.x removes ZooKeeper support entirely (KRaft-only) and needs an operator-side -# migration; when this repo is ready for that, drop the allowedVersions constraint too. -# Also bump java_version to 25 (next LTS after 21) as part of that migration. -ARG KAFKA_VERSION=3.9.2 # renovate: datasource=github-tags depName=apache/kafka -ARG java_version=21 +#Kafka 4 is kRaft only +ARG KAFKA_VERSION=4.2.2 # renovate: datasource=github-tags depName=apache/kafka +ARG java_version=25 ARG java_download_version=${java_version} ARG java_download_base_url=https://download.oracle.com/java @@ -25,9 +22,12 @@ WORKDIR /var/tmp # Download, verify, extract and clean up Kafka in single layer RUN apk add --no-cache gnupg wget && \ - wget -q $kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro && \ - wget -q $kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro_asc && \ - wget -q $kafka_distro_base_url/KEYS && \ + wget --timeout=30 --tries=3 --progress=dot:mega \ + "$kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro" && \ + wget --timeout=30 --tries=3 --progress=dot:mega \ + "$kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro_asc" && \ + wget --timeout=30 --tries=3 --progress=dot:mega \ + "$kafka_distro_base_url/KEYS" && \ gpg --import KEYS && \ gpg --verify $kafka_distro_asc $kafka_distro && \ tar -xzf $kafka_distro && \ @@ -38,7 +38,7 @@ RUN apk add --no-cache gnupg wget && \ # backported from https://github.com/docker-library/openjdk/blob/master/18/jdk/slim-bullseye/Dockerfile -FROM debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4 +FROM debian:stable-slim@sha256:13e0a2e78aa0554ad5ff66e4e260c6dbcdff807eb1f39d43b9fb0288aae88b04 ARG scala_version ARG KAFKA_VERSION @@ -63,6 +63,9 @@ ENV PATH=$JAVA_HOME/bin:$PATH ENV LANG=C.UTF-8 ENV JAVA_VERSION=${java_version} +# Apply Java 25 defaults before generating the CDS archive. +ENV JDK_JAVA_OPTIONS="--enable-native-access=ALL-UNNAMED --illegal-native-access=deny -XX:+UseCompactObjectHeaders" + RUN --mount=type=secret,id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY \ set -eux; \ \ @@ -155,7 +158,9 @@ RUN apt-get update && \ mkdir -p ${KAFKA_HOME}/libs/extensions COPY --from=kafka_dist /var/tmp/kafka_$scala_version-$KAFKA_VERSION ${KAFKA_HOME} -COPY opt/kafka/config/log4j.properties ${KAFKA_HOME}/config/log4j.properties +COPY opt/kafka/config/log4j2.properties ${KAFKA_HOME}/config/log4j2.properties + +ENV KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=/opt/kafka/config/log4j2.properties" # Optimize Kafka installation: remove unnecessary files and set permissions RUN chmod a+x ${KAFKA_HOME}/bin/*.sh && \ @@ -167,7 +172,7 @@ RUN chmod a+x ${KAFKA_HOME}/bin/*.sh && \ LABEL org.opencontainers.image.title="Apache Kafka" LABEL org.opencontainers.image.description="Apache Kafka with Oracle JDK ${java_version}" LABEL org.opencontainers.image.vendor="Adobe" -LABEL org.opencontainers.image.licenses="Apache-2.0" +LABEL org.opencontainers.image.licenses="Apache-2.0 AND LicenseRef-Oracle-NFTC" LABEL org.opencontainers.image.source="https://github.com/adobe/koperator" LABEL org.opencontainers.image.documentation="https://github.com/adobe/koperator/blob/main/docker/kafka/README.md" LABEL org.opencontainers.image.version="${VERSION}" diff --git a/docker/kafka/README.md b/docker/kafka/README.md index c5a9ea981..2443c9032 100644 --- a/docker/kafka/README.md +++ b/docker/kafka/README.md @@ -4,34 +4,47 @@ A new `kafka-*` tag created in this repo triggers the image build and push to [adobe/kafka](https://hub.docker.com/r/adobe/kafka/tags?page=1&ordering=last_updated) docker hub repo. -Tags should be `kafka--` e.g `kafak-2.13-2.6.2` +Tags should be `kafka--`, for example +`kafka-2.13-4.2.2`. Release-candidate tags may append `-rc`, +followed by an optional `-jdk` suffix. The Scala and Kafka versions +must match the Dockerfile defaults. # Upstream base This is based on [wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) with the following additions: -1. Use `openjdk 21` in order to support container based resource monitoring -2. Use custom `log4j.properties` to get all kafka logs to stdout only +1. Use Oracle JDK 25 LTS with container based resource monitoring +2. Use custom `log4j2.properties` to get all Kafka logs to stdout only -## Alternate JDK downloads +## Java 25 runtime defaults -The Dockerfile defaults to Oracle's public download URL without authentication. -Override `java_download_base_url` and `java_download_version` to use another source. -The Dockerfile appends `java_version` and `latest` for Oracle, or only -`java_version` for alternate sources. Neither version belongs in the base URL. -`java_download_version` selects the archive filename, not a directory. Alternate -sources must provide the Oracle-style archive filenames and matching `.sha256` -files for each architecture. +The image sets `JDK_JAVA_OPTIONS` to: -For authenticated downloads, pass an optional API key as a BuildKit secret: - -```bash ---secret id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY +```text +--enable-native-access=ALL-UNNAMED --illegal-native-access=deny -XX:+UseCompactObjectHeaders ``` -The key is sent as an `Authorization: Bearer` header, is not stored in the image, -and is not forwarded to a different host on redirects. -Release builds also default to Oracle. To use an alternate source, add the -commented overrides to the release workflow's `build-args`, configure the GitHub -Actions secret `JAVA_DOWNLOAD_BASE_URL`, and select a matching `java_download_version`. -For authenticated downloads, also enable the commented `secrets` input and -configure `JAVA_DOWNLOAD_APIKEY`. +These options permit Kafka's classpath-based native compression libraries, reject +native access that has not been granted, and enable compact object headers. The +CDS archive is generated with the same options. + +The defaults apply to Java launches in this image, including Kafka's storage and +command-line tools, without replacing `KAFKA_OPTS` used for the JMX agent. They +are not added to the operator's shared JVM defaults: Java 21 images reject the +native-access deny option and compact object headers. + +## Java 25 rollout + +This branch targets Kafka 4.2.2 on Oracle JDK 25 LTS. Kafka 4 is KRaft-only; +do not use this image with ZooKeeper brokers or unfinished migrations. + +Operator defaults, samples, benchmark manifests, and the vendored Cruise Control +Compose configuration still reference the published Java 21 Kafka image. After +validating Kafka startup, SASL/TLS authentication, and the JMX exporter on Java 25, +publish a new `kafka---jdk` release tag. +Then update those image references and refresh any pinned image digests together. +Do not point them at an unpublished tag or reuse a Java 21 image digest. + +The rebuilt JMX exporter uses JDK 25 to compile Java 21 bytecode, preserving +compatibility with Cruise Control's Java 21 runtime. Its architecture-independent +JAR is built and tested on the native build platform, without QEMU. Publish and +validate a new image tag before changing deployed JMX image references. diff --git a/docker/kafka/opt/kafka/config/log4j2.properties b/docker/kafka/opt/kafka/config/log4j2.properties new file mode 100644 index 000000000..8ab03f40c --- /dev/null +++ b/docker/kafka/opt/kafka/config/log4j2.properties @@ -0,0 +1,13 @@ +status=warn +name=KafkaConsole +appender.console.type=Console +appender.console.name=STDOUT +appender.console.target=SYSTEM_OUT +appender.console.layout.type=PatternLayout +appender.console.layout.pattern=[%d] %p %m (%c)%n +rootLogger.level=info +rootLogger.appenderRef.stdout.ref=STDOUT +logger.requests.name=kafka.request.logger +logger.requests.level=warn +logger.authorizer.name=kafka.authorizer.logger +logger.authorizer.level=info diff --git a/docs/examples/springboot-kafka-avro/Dockerfile b/docs/examples/springboot-kafka-avro/Dockerfile index b84ff78ef..1227b46de 100644 --- a/docs/examples/springboot-kafka-avro/Dockerfile +++ b/docs/examples/springboot-kafka-avro/Dockerfile @@ -1,4 +1,4 @@ -FROM maven:3.10-eclipse-temurin-21-alpine@sha256:db7b12415a992fec6d3d2a1f4a32c9225a1d98c71cbfd2379f8c4a12775f6f34 as build +FROM maven:3.10-eclipse-temurin-25-alpine@sha256:5d74a0f788b3cca0d36a4a3f358eadc8b0dd6f2f4f2dfcd0091b0cd5934d3002 as build # Set working directory WORKDIR /usr/src/myapp @@ -12,7 +12,7 @@ COPY src ./src RUN mvn clean package -DskipTests # The final image should have minimal layers -FROM eclipse-temurin:27-jre-alpine@sha256:0faf24c1d1e778bbc0700fc71c53b081c9432f1e51e6e41c86c63840e8b89ac1 +FROM eclipse-temurin:25-jre-alpine@sha256:b7528962e65d43064d8a8b67ba175367896bc948363ea2d3fcd7f10d34e1f048 RUN apk add --no-cache curl COPY --from=build /usr/src/myapp/target/kafka-avro-0.0.1-SNAPSHOT.jar app.jar ENTRYPOINT ["java", "-jar", "app.jar"] \ No newline at end of file diff --git a/docs/examples/springboot-kafka-avro/pom.xml b/docs/examples/springboot-kafka-avro/pom.xml index b3f118e9e..cb6baf5d4 100644 --- a/docs/examples/springboot-kafka-avro/pom.xml +++ b/docs/examples/springboot-kafka-avro/pom.xml @@ -15,7 +15,7 @@ Demo project for Spring Boot - 21 + 25 @@ -62,6 +62,19 @@ + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.projectlombok + lombok + ${lombok.version} + + + + org.springframework.boot spring-boot-maven-plugin diff --git a/pkg/resources/kafka/pod.go b/pkg/resources/kafka/pod.go index 33ddaec3a..527a85261 100644 --- a/pkg/resources/kafka/pod.go +++ b/pkg/resources/kafka/pod.go @@ -656,9 +656,11 @@ func generateEnvConfig(brokerConfig *v1beta1.BrokerConfig, defaultEnvVars []core } if brokerConfig.Log4jConfig != "" { - envs["KAFKA_LOG4J_OPTS"] = corev1.EnvVar{ - Name: "KAFKA_LOG4J_OPTS", - Value: "-Dlog4j.configuration=file:/config/log4j.properties", + if _, configured := envs["KAFKA_LOG4J_OPTS"]; !configured { + envs["KAFKA_LOG4J_OPTS"] = corev1.EnvVar{ + Name: "KAFKA_LOG4J_OPTS", + Value: "-Dlog4j.configuration=file:/config/log4j.properties", + } } } diff --git a/pkg/resources/kafka/pod_test.go b/pkg/resources/kafka/pod_test.go index 0f43e0539..9946d5c45 100644 --- a/pkg/resources/kafka/pod_test.go +++ b/pkg/resources/kafka/pod_test.go @@ -188,6 +188,52 @@ func Test_generateEnvConfig(t *testing.T) { } } +func TestGenerateEnvConfigLogging(t *testing.T) { + for _, test := range []struct { + name string + config string + envs []corev1.EnvVar + expected corev1.EnvVar + expectFound bool + }{ + { + name: "legacy logging default", + config: "log4j.rootLogger=INFO, stdout", + expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j.configuration=file:/config/log4j.properties"}, + expectFound: true, + }, + { + name: "explicit Log4j 2 option", + config: "rootLogger.level=info", + envs: []corev1.EnvVar{{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j2.configurationFile=/config/log4j.properties"}}, + expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j2.configurationFile=/config/log4j.properties"}, + expectFound: true, + }, + { + name: "explicit option from secret", + config: "rootLogger.level=info", + envs: []corev1.EnvVar{{Name: "KAFKA_LOG4J_OPTS", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "logging"}, Key: "opts"}}}}, + expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "logging"}, Key: "opts"}}}, + expectFound: true, + }, + { + name: "image logging default without custom config", + }, + } { + t.Run(test.name, func(t *testing.T) { + result := generateEnvConfig(&v1beta1.BrokerConfig{Log4jConfig: test.config, Envs: test.envs}, nil) + found := false + for _, env := range result { + if env.Name == "KAFKA_LOG4J_OPTS" { + found = true + assert.DeepEqual(t, env, test.expected) + } + } + assert.Equal(t, found, test.expectFound) + }) + } +} + func TestAddDynamicKRaftQuorumFormatEnv(t *testing.T) { brokers := []v1beta1.Broker{ {Id: 0, BrokerConfig: &v1beta1.BrokerConfig{Roles: []string{"controller"}}}, diff --git a/renovate.json b/renovate.json index d769e453b..6ab7dad2d 100644 --- a/renovate.json +++ b/renovate.json @@ -167,12 +167,24 @@ "patch" ] }, + { + "matchDatasources": ["docker"], + "matchPackageNames": ["maven"], + "allowedVersions": "/^3(?:\\.\\d+)*-(?:amazoncorretto-25|eclipse-temurin-25-alpine)$/", + "description": "Keep Maven builders on Java 25 LTS while allowing Maven 3 updates and image digest refreshes." + }, + { + "matchDatasources": ["docker"], + "matchPackageNames": ["eclipse-temurin"], + "allowedVersions": "/^25-jre-alpine$/", + "description": "Keep the Spring example runtime on Java 25 LTS; continue refreshing its image digest." + }, { "matchPackageNames": [ "apache/kafka" ], - "allowedVersions": "<4.0.0", - "description": "Kafka 4.x drops ZooKeeper support (KRaft-only) and needs an operator-side migration. Keep Renovate on the 3.x line until docker/kafka/Dockerfile is ready for that jump, then remove this rule." + "allowedVersions": ">=4.2.2 <4.3.0", + "description": "Keep distribution updates on the 4.2 line; later minor/major versions require separate validation." }, { "matchCurrentValue": "/^\\d+\\.\\d+\\.\\d+-ado?be-\\d+$/", @@ -195,7 +207,7 @@ ], "matchCurrentValue": "/^\\d+\\.\\d+-\\d+\\.\\d+\\.\\d+(-jdk\\d+\\.\\d+\\.\\d+)?$/", "versioning": "regex:^(?\\d+\\.\\d+)-(?\\d+)\\.(?\\d+)\\.(?\\d+)(?:-jdk(?\\d+)\\.\\d+\\.(?\\d+))?$", - "description": "Tag format is -[-jdk], e.g. 2.13-3.9.1 or 2.13-3.9.2-jdk21.0.11 (the latter shape shipped starting with the 3.9.2 release tag). Verified live against ghcr.io: default docker versioning parses currentVersion as just '2.13' and silently drops everything after the first hyphen, so Renovate can never see a real 3.9.1->3.9.2 bump (only digest-pin updates fire, via the docker pinDigests rule below) - this affects every '# renovate:' annotated occurrence of this image. Pinning the scala prefix as 'compatibility' (never auto-changed) and parsing the kafka release as major.minor.patch fixes real version comparison. The optional '-jdk..' suffix maps jdk major to 'build' and jdk update to 'revision' (Renovate's regex versioning caps out at 5 numeric levels: major/minor/patch/build/revision, each required to be purely numeric, so the always-0 jdk interim segment is matched but not captured); tags without the suffix default build/revision to 0 and still compare correctly against suffixed ones since a differing kafka patch alone already resolves ordering. This does NOT gate jdk-major jumps (e.g. jdk21.x -> jdk25.x) the way 'compatibility' gates scala - that's covered by the same operator-side migration as the apache/kafka 4.x bump below (see docker/kafka/Dockerfile's java_version comment); revisit if jdk25 lands before that migration." + "description": "Tag format is -[-jdk], e.g. 2.13-3.9.2-jdk21.0.11 or 2.13-4.2.2. Pinning the scala prefix as compatibility and parsing the Kafka release as major.minor.patch permits real Kafka version comparisons. The optional JDK suffix maps its major version to build and update to revision. The Kafka 4.2.2 candidate uses Oracle Java 25; keep published Java 21 defaults until replacement images are published and validated." }, { "groupName": "kafka docker images (major)",