Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
33 changes: 28 additions & 5 deletions .github/workflows/publish-kafka-docker.yml
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,28 @@ jobs:
run: |
KAFKA_VERSION=$(echo "${{ needs.prepare.outputs.tag }}" | sed 's/kafka-//')
echo "kafka_version=$KAFKA_VERSION" >> $GITHUB_OUTPUT
- name: Extract Java version from Dockerfile
id: java_version
run: |
JAVA_VERSION=$(sed -nE 's/^ARG java_version=([0-9]+).*$/\1/p' docker/kafka/Dockerfile)
echo "java_version=$JAVA_VERSION" >> $GITHUB_OUTPUT
- name: Validate tagged Kafka distribution
env:
IMAGE_VERSION: ${{ steps.kafka_version.outputs.kafka_version }}
run: |
set -euo pipefail
if [[ ! "$IMAGE_VERSION" =~ ^([0-9]+\.[0-9]+)-([0-9]+\.[0-9]+\.[0-9]+)(-rc[0-9]+)?(-jdk[0-9]+\.[0-9]+\.[0-9]+)?$ ]]; then
echo "::error::Expected kafka-<scala>-<kafka>[-rc<number>][-jdk<version>] release tag"
exit 1
fi
TAG_SCALA="${BASH_REMATCH[1]}"
TAG_KAFKA="${BASH_REMATCH[2]}"
DIST_SCALA=$(sed -nE 's/^ARG scala_version=([^[:space:]]+).*$/\1/p' docker/kafka/Dockerfile)
DIST_KAFKA=$(sed -nE 's/^ARG KAFKA_VERSION=([^[:space:]]+).*$/\1/p' docker/kafka/Dockerfile)
if [[ "$TAG_SCALA" != "$DIST_SCALA" || "$TAG_KAFKA" != "$DIST_KAFKA" ]]; then
echo "::error::Tag $IMAGE_VERSION does not match Dockerfile distribution $DIST_SCALA-$DIST_KAFKA"
exit 1
fi
- name: Check if Kafka image already exists
id: image_exists
run: |
Expand Down Expand Up @@ -91,7 +113,7 @@ jobs:
tags: |
type=match,pattern=kafka-(.*),group=1
labels: |
org.opencontainers.image.description=Apache Kafka with OpenJDK 21
org.opencontainers.image.description=Apache Kafka with Oracle JDK ${{ steps.java_version.outputs.java_version }}
- name: Build and push kafka image
if: startsWith(github.ref, 'refs/tags/') && steps.image_exists.outputs.exists == 'false'
uses: docker/build-push-action@v7
Expand Down Expand Up @@ -128,14 +150,15 @@ jobs:
id: jmx_exporter_version
run: |
JMX_VERSION=$(grep "ARG JMX_EXPORTER_VERSION=" docker/jmx_exporter/Dockerfile | cut -d'=' -f2 | cut -d' ' -f1)
# Strip a leading "v" so the published image tag is always version-only,
# regardless of whether the upstream release tag (and thus the Dockerfile ARG) has one.
# Normalize the upstream version before adding the JDK rebuild suffix.
JMX_VERSION="${JMX_VERSION#v}"
JAVA_VERSION=$(sed -nE 's/^ARG java_version=([0-9]+).*$/\1/p' docker/kafka/Dockerfile)
echo "jmx_version=$JMX_VERSION" >> $GITHUB_OUTPUT
echo "jmx_image_tag=${JMX_VERSION}-jdk${JAVA_VERSION}" >> $GITHUB_OUTPUT
- name: Check if JMX exporter image already exists
id: image_exists
run: |
IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}:${{ steps.jmx_exporter_version.outputs.jmx_version }}"
IMAGE_NAME="${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}:${{ steps.jmx_exporter_version.outputs.jmx_image_tag }}"
echo "Checking if image exists: $IMAGE_NAME"

# Try to inspect the manifest using docker
Expand All @@ -161,7 +184,7 @@ jobs:
images: |
${{ env.GHCR_REGISTRY }}/${{ env.GHCR_JMX_IMAGE_NAME }}
tags: |
type=raw,value=${{ steps.jmx_exporter_version.outputs.jmx_version }}
type=raw,value=${{ steps.jmx_exporter_version.outputs.jmx_image_tag }}
labels: |
org.opencontainers.image.description=Prometheus JMX Exporter for Kafka monitoring
- name: Build and push JMX exporter image
Expand Down
10 changes: 6 additions & 4 deletions docker/jmx_exporter/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
ARG JMX_EXPORTER_VERSION=1.7.0 # renovate: datasource=github-releases depName=prometheus/jmx_exporter

FROM maven:3-amazoncorretto-21@sha256:f64fd4f0363c918004816d83d2589495b622a8d2fc5c5602c135cb4124c7ea12 AS build
ARG BUILDPLATFORM
# The Java agent JAR is architecture-independent; compile and test without QEMU.
FROM --platform=$BUILDPLATFORM maven:3-amazoncorretto-25@sha256:d96c6ff2e7a3258e6f0c447340d2284a3404917bb8c5f01c8660cc4066bd382c AS build
ARG JMX_EXPORTER_VERSION

# Install wget to download the release tarball
RUN yum install -y wget && yum clean all
# The Java 25 builder does not include tar or gzip.
RUN yum install -y wget tar gzip && yum clean all

# Download and extract the JMX exporter release
RUN wget https://github.com/prometheus/jmx_exporter/archive/refs/tags/${JMX_EXPORTER_VERSION}.tar.gz \
Expand All @@ -14,6 +15,7 @@ RUN wget https://github.com/prometheus/jmx_exporter/archive/refs/tags/${JMX_EXPO
rm /tmp/jmx_exporter.tar.gz

WORKDIR /src
# Keep the agent compatible with Java 21 consumers while building with JDK 25.
RUN mvn -B -Dmaven.javadoc.skip=true -Dskip.javadoc=true \
-Dmaven.compiler.source=21 -Dmaven.compiler.target=21 -Dmaven.compiler.release=21 \
clean package -pl jmx_prometheus_javaagent -am && \
Expand Down
29 changes: 17 additions & 12 deletions docker/kafka/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,12 +1,9 @@
# syntax=docker/dockerfile:1

ARG scala_version=2.13
# Renovate is pinned to the 3.x line (see the apache/kafka packageRule in renovate.json).
# Kafka 4.x removes ZooKeeper support entirely (KRaft-only) and needs an operator-side
# migration; when this repo is ready for that, drop the allowedVersions constraint too.
# Also bump java_version to 25 (next LTS after 21) as part of that migration.
ARG KAFKA_VERSION=3.9.2 # renovate: datasource=github-tags depName=apache/kafka
ARG java_version=21
#Kafka 4 is kRaft only
ARG KAFKA_VERSION=4.2.2 # renovate: datasource=github-tags depName=apache/kafka
ARG java_version=25
ARG java_download_version=${java_version}
ARG java_download_base_url=https://download.oracle.com/java

Expand All @@ -25,9 +22,12 @@ WORKDIR /var/tmp

# Download, verify, extract and clean up Kafka in single layer
RUN apk add --no-cache gnupg wget && \
wget -q $kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro && \
wget -q $kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro_asc && \
wget -q $kafka_distro_base_url/KEYS && \
wget --timeout=30 --tries=3 --progress=dot:mega \
"$kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro" && \
wget --timeout=30 --tries=3 --progress=dot:mega \
"$kafka_distro_base_url/$KAFKA_VERSION/$kafka_distro_asc" && \
wget --timeout=30 --tries=3 --progress=dot:mega \
"$kafka_distro_base_url/KEYS" && \
gpg --import KEYS && \
gpg --verify $kafka_distro_asc $kafka_distro && \
tar -xzf $kafka_distro && \
Expand All @@ -38,7 +38,7 @@ RUN apk add --no-cache gnupg wget && \


# backported from https://github.com/docker-library/openjdk/blob/master/18/jdk/slim-bullseye/Dockerfile
FROM debian:trixie-slim@sha256:f6b08b09a05725f9431502d3b5714f5c4576e82c1a84fa1debe0c24a7e0c3ec4
FROM debian:stable-slim@sha256:13e0a2e78aa0554ad5ff66e4e260c6dbcdff807eb1f39d43b9fb0288aae88b04

ARG scala_version
ARG KAFKA_VERSION
Expand All @@ -63,6 +63,9 @@ ENV PATH=$JAVA_HOME/bin:$PATH
ENV LANG=C.UTF-8
ENV JAVA_VERSION=${java_version}

# Apply Java 25 defaults before generating the CDS archive.
ENV JDK_JAVA_OPTIONS="--enable-native-access=ALL-UNNAMED --illegal-native-access=deny -XX:+UseCompactObjectHeaders"

RUN --mount=type=secret,id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY \
set -eux; \
\
Expand Down Expand Up @@ -155,7 +158,9 @@ RUN apt-get update && \
mkdir -p ${KAFKA_HOME}/libs/extensions

COPY --from=kafka_dist /var/tmp/kafka_$scala_version-$KAFKA_VERSION ${KAFKA_HOME}
COPY opt/kafka/config/log4j.properties ${KAFKA_HOME}/config/log4j.properties
COPY opt/kafka/config/log4j2.properties ${KAFKA_HOME}/config/log4j2.properties

ENV KAFKA_LOG4J_OPTS="-Dlog4j2.configurationFile=/opt/kafka/config/log4j2.properties"

# Optimize Kafka installation: remove unnecessary files and set permissions
RUN chmod a+x ${KAFKA_HOME}/bin/*.sh && \
Expand All @@ -167,7 +172,7 @@ RUN chmod a+x ${KAFKA_HOME}/bin/*.sh && \
LABEL org.opencontainers.image.title="Apache Kafka"
LABEL org.opencontainers.image.description="Apache Kafka with Oracle JDK ${java_version}"
LABEL org.opencontainers.image.vendor="Adobe"
LABEL org.opencontainers.image.licenses="Apache-2.0"
LABEL org.opencontainers.image.licenses="Apache-2.0 AND LicenseRef-Oracle-NFTC"
LABEL org.opencontainers.image.source="https://github.com/adobe/koperator"
LABEL org.opencontainers.image.documentation="https://github.com/adobe/koperator/blob/main/docker/kafka/README.md"
LABEL org.opencontainers.image.version="${VERSION}"
Expand Down
57 changes: 35 additions & 22 deletions docker/kafka/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,34 +4,47 @@

A new `kafka-*` tag created in this repo triggers the image build and push to [adobe/kafka](https://hub.docker.com/r/adobe/kafka/tags?page=1&ordering=last_updated) docker hub repo.

Tags should be `kafka-<scala_version>-<kafka_version>` e.g `kafak-2.13-2.6.2`
Tags should be `kafka-<scala_version>-<kafka_version>`, for example
`kafka-2.13-4.2.2`. Release-candidate tags may append `-rc<number>`,
followed by an optional `-jdk<version>` suffix. The Scala and Kafka versions
must match the Dockerfile defaults.

# Upstream base

This is based on [wurstmeister/kafka-docker](https://github.com/wurstmeister/kafka-docker) with the following additions:
1. Use `openjdk 21` in order to support container based resource monitoring
2. Use custom `log4j.properties` to get all kafka logs to stdout only
1. Use Oracle JDK 25 LTS with container based resource monitoring
2. Use custom `log4j2.properties` to get all Kafka logs to stdout only

## Alternate JDK downloads
## Java 25 runtime defaults

The Dockerfile defaults to Oracle's public download URL without authentication.
Override `java_download_base_url` and `java_download_version` to use another source.
The Dockerfile appends `java_version` and `latest` for Oracle, or only
`java_version` for alternate sources. Neither version belongs in the base URL.
`java_download_version` selects the archive filename, not a directory. Alternate
sources must provide the Oracle-style archive filenames and matching `.sha256`
files for each architecture.
The image sets `JDK_JAVA_OPTIONS` to:

For authenticated downloads, pass an optional API key as a BuildKit secret:

```bash
--secret id=JAVA_DOWNLOAD_APIKEY,env=JAVA_DOWNLOAD_APIKEY
```text
--enable-native-access=ALL-UNNAMED --illegal-native-access=deny -XX:+UseCompactObjectHeaders
```

The key is sent as an `Authorization: Bearer` header, is not stored in the image,
and is not forwarded to a different host on redirects.
Release builds also default to Oracle. To use an alternate source, add the
commented overrides to the release workflow's `build-args`, configure the GitHub
Actions secret `JAVA_DOWNLOAD_BASE_URL`, and select a matching `java_download_version`.
For authenticated downloads, also enable the commented `secrets` input and
configure `JAVA_DOWNLOAD_APIKEY`.
These options permit Kafka's classpath-based native compression libraries, reject
native access that has not been granted, and enable compact object headers. The
CDS archive is generated with the same options.

The defaults apply to Java launches in this image, including Kafka's storage and
command-line tools, without replacing `KAFKA_OPTS` used for the JMX agent. They
are not added to the operator's shared JVM defaults: Java 21 images reject the
native-access deny option and compact object headers.

## Java 25 rollout

This branch targets Kafka 4.2.2 on Oracle JDK 25 LTS. Kafka 4 is KRaft-only;
do not use this image with ZooKeeper brokers or unfinished migrations.

Operator defaults, samples, benchmark manifests, and the vendored Cruise Control
Compose configuration still reference the published Java 21 Kafka image. After
validating Kafka startup, SASL/TLS authentication, and the JMX exporter on Java 25,
publish a new `kafka-<scala_version>-<kafka_version>-jdk<jdk_version>` release tag.
Then update those image references and refresh any pinned image digests together.
Do not point them at an unpublished tag or reuse a Java 21 image digest.

The rebuilt JMX exporter uses JDK 25 to compile Java 21 bytecode, preserving
compatibility with Cruise Control's Java 21 runtime. Its architecture-independent
JAR is built and tested on the native build platform, without QEMU. Publish and
validate a new image tag before changing deployed JMX image references.
13 changes: 13 additions & 0 deletions docker/kafka/opt/kafka/config/log4j2.properties
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
status=warn
name=KafkaConsole
appender.console.type=Console
appender.console.name=STDOUT
appender.console.target=SYSTEM_OUT
appender.console.layout.type=PatternLayout
appender.console.layout.pattern=[%d] %p %m (%c)%n
rootLogger.level=info
rootLogger.appenderRef.stdout.ref=STDOUT
logger.requests.name=kafka.request.logger
logger.requests.level=warn
logger.authorizer.name=kafka.authorizer.logger
logger.authorizer.level=info
4 changes: 2 additions & 2 deletions docs/examples/springboot-kafka-avro/Dockerfile
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
FROM maven:3.10-eclipse-temurin-21-alpine@sha256:db7b12415a992fec6d3d2a1f4a32c9225a1d98c71cbfd2379f8c4a12775f6f34 as build
FROM maven:3.10-eclipse-temurin-25-alpine@sha256:5d74a0f788b3cca0d36a4a3f358eadc8b0dd6f2f4f2dfcd0091b0cd5934d3002 as build

# Set working directory
WORKDIR /usr/src/myapp
Expand All @@ -12,7 +12,7 @@ COPY src ./src
RUN mvn clean package -DskipTests

# The final image should have minimal layers
FROM eclipse-temurin:27-jre-alpine@sha256:0faf24c1d1e778bbc0700fc71c53b081c9432f1e51e6e41c86c63840e8b89ac1
FROM eclipse-temurin:25-jre-alpine@sha256:b7528962e65d43064d8a8b67ba175367896bc948363ea2d3fcd7f10d34e1f048
RUN apk add --no-cache curl
COPY --from=build /usr/src/myapp/target/kafka-avro-0.0.1-SNAPSHOT.jar app.jar
ENTRYPOINT ["java", "-jar", "app.jar"]
15 changes: 14 additions & 1 deletion docs/examples/springboot-kafka-avro/pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
<description>Demo project for Spring Boot</description>

<properties>
<java.version>21</java.version>
<java.version>25</java.version>
</properties>

<dependencies>
Expand Down Expand Up @@ -62,6 +62,19 @@

<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<configuration>
<annotationProcessorPaths>
<path>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>${lombok.version}</version>
</path>
</annotationProcessorPaths>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
Expand Down
8 changes: 5 additions & 3 deletions pkg/resources/kafka/pod.go
Original file line number Diff line number Diff line change
Expand Up @@ -656,9 +656,11 @@ func generateEnvConfig(brokerConfig *v1beta1.BrokerConfig, defaultEnvVars []core
}

if brokerConfig.Log4jConfig != "" {
envs["KAFKA_LOG4J_OPTS"] = corev1.EnvVar{
Name: "KAFKA_LOG4J_OPTS",
Value: "-Dlog4j.configuration=file:/config/log4j.properties",
if _, configured := envs["KAFKA_LOG4J_OPTS"]; !configured {
envs["KAFKA_LOG4J_OPTS"] = corev1.EnvVar{
Name: "KAFKA_LOG4J_OPTS",
Value: "-Dlog4j.configuration=file:/config/log4j.properties",
Comment thread
cawright-rh marked this conversation as resolved.
Resolved
}
}
}

Expand Down
46 changes: 46 additions & 0 deletions pkg/resources/kafka/pod_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -188,6 +188,52 @@ func Test_generateEnvConfig(t *testing.T) {
}
}

func TestGenerateEnvConfigLogging(t *testing.T) {
for _, test := range []struct {
name string
config string
envs []corev1.EnvVar
expected corev1.EnvVar
expectFound bool
}{
{
name: "legacy logging default",
config: "log4j.rootLogger=INFO, stdout",
expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j.configuration=file:/config/log4j.properties"},
expectFound: true,
},
{
name: "explicit Log4j 2 option",
config: "rootLogger.level=info",
envs: []corev1.EnvVar{{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j2.configurationFile=/config/log4j.properties"}},
expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", Value: "-Dlog4j2.configurationFile=/config/log4j.properties"},
expectFound: true,
},
{
name: "explicit option from secret",
config: "rootLogger.level=info",
envs: []corev1.EnvVar{{Name: "KAFKA_LOG4J_OPTS", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "logging"}, Key: "opts"}}}},
expected: corev1.EnvVar{Name: "KAFKA_LOG4J_OPTS", ValueFrom: &corev1.EnvVarSource{SecretKeyRef: &corev1.SecretKeySelector{LocalObjectReference: corev1.LocalObjectReference{Name: "logging"}, Key: "opts"}}},
expectFound: true,
},
{
name: "image logging default without custom config",
},
} {
t.Run(test.name, func(t *testing.T) {
result := generateEnvConfig(&v1beta1.BrokerConfig{Log4jConfig: test.config, Envs: test.envs}, nil)
found := false
for _, env := range result {
if env.Name == "KAFKA_LOG4J_OPTS" {
found = true
assert.DeepEqual(t, env, test.expected)
}
}
assert.Equal(t, found, test.expectFound)
})
}
}

func TestAddDynamicKRaftQuorumFormatEnv(t *testing.T) {
brokers := []v1beta1.Broker{
{Id: 0, BrokerConfig: &v1beta1.BrokerConfig{Roles: []string{"controller"}}},
Expand Down
Loading