Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .github/actions/kind-create/action.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,9 +6,8 @@ inputs: # adding these parameters to make this reusable later on
required: true
default: "e2e-kind"
kind_k8s_version:
description: 'The k8s version to use'
description: 'The Kubernetes target from config/kubernetes-versions.env, prefixed with v'
required: true
default: 'v1.37.0' # renovate: datasource=docker depName=kindest/node
kind_config:
description: 'Path to the kind config to use'
required: true
Expand All @@ -30,6 +29,7 @@ runs:
cluster_name: ${{ inputs.kind_cluster_name }}
version: v0.33.0 # renovate: datasource=github-releases depName=kubernetes-sigs/kind
node_image: kindest/node:${{ inputs.kind_k8s_version }}
kubectl_version: ${{ inputs.kind_k8s_version }}
config: ${{ inputs.kind_config }}

- name: Verify all nodes are healthy, recreating the cluster otherwise
Expand Down
67 changes: 63 additions & 4 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,18 @@ concurrency:
cancel-in-progress: true

jobs:
kubernetes-versions:
name: Read Kubernetes targets
runs-on: ubuntu-26.04
outputs:
matrix: ${{ steps.versions.outputs.matrix }}
steps:
- uses: actions/checkout@v7
- id: versions
run: |
source config/kubernetes-versions.env
echo "matrix=$(jq -cn --arg min "$KUBERNETES_MIN_VERSION" --arg max "$KUBERNETES_MAX_VERSION" '[$min, $max] | unique')" >> "$GITHUB_OUTPUT"

check-integrity:
name: Check integrity
runs-on: ubuntu-26.04
Expand All @@ -26,6 +38,26 @@ jobs:
working-directory: charts/kafka-operator
fail-on-diff: true

- name: Setup Helm
uses: azure/setup-helm@v5
with:
version: '4.3.1' # renovate: datasource=github-releases depName=helm/helm

- name: Check supported Kubernetes chart versions
run: |
source config/kubernetes-versions.env
for version in "$KUBERNETES_MIN_VERSION" "$KUBERNETES_MAX_VERSION"; do
helm lint charts/kafka-operator --strict --kube-version "$version"
helm template kafka-operator charts/kafka-operator --kube-version "$version" > /dev/null
done
IFS=. read -r major minor _ <<< "$KUBERNETES_MIN_VERSION"
unsupported="${major}.$((minor - 1)).99"
if output=$(helm template kafka-operator charts/kafka-operator --kube-version "$unsupported" 2>&1); then
echo "::error::Chart must reject unsupported Kubernetes ${unsupported}"
exit 1
fi
grep -F 'kubeVersion' <<< "$output"

- name: Set up Go
uses: actions/setup-go@v7
with:
Expand All @@ -45,6 +77,9 @@ jobs:
popd > /dev/null
done

- name: Check managed Kubernetes versions
run: make check-kubernetes-versions

- name: Check generated file integrity
run: |
make generate manifests
Expand All @@ -57,8 +92,15 @@ jobs:
build:
name: Build
runs-on: ubuntu-26.04
needs: test
if: ${{ always() }}

steps:
- name: Require successful Kubernetes test matrix
env:
TEST_RESULT: ${{ needs.test.result }}
run: test "$TEST_RESULT" = success

- name: Checkout code
uses: actions/checkout@v7

Expand Down Expand Up @@ -122,12 +164,29 @@ jobs:
run: |
make lint

- name: Run tests
run: |
make test

- name: Save license cache
uses: actions/cache/save@v6
with:
path: .licensei.cache
key: ci-license-v1-${{ hashFiles('**/go.sum') }}

test:
name: Test (Kubernetes ${{ matrix.kubernetes-version }})
runs-on: ubuntu-26.04
needs: kubernetes-versions
strategy:
fail-fast: false
matrix:
kubernetes-version: ${{ fromJSON(needs.kubernetes-versions.outputs.matrix) }}

steps:
- name: Checkout code
uses: actions/checkout@v7

- name: Set up Go
uses: actions/setup-go@v7
with:
go-version-file: go.mod

- name: Run tests
run: make test ENVTEST_K8S_VERSION=${{ matrix.kubernetes-version }}
40 changes: 34 additions & 6 deletions .github/workflows/e2e-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,26 @@ env:
REPOSITORY: koperator_e2e_test

jobs:
kubernetes-versions:
name: Read Kubernetes targets
runs-on: ubuntu-26.04
outputs:
matrix: ${{ steps.versions.outputs.matrix }}
steps:
- uses: actions/checkout@v7
- id: versions
run: |
source config/kubernetes-versions.env
echo "matrix=$(jq -cn --arg min "$KUBERNETES_MIN_VERSION" --arg max "$KUBERNETES_MAX_VERSION" '[$min, $max] | unique')" >> "$GITHUB_OUTPUT"

build:
name: E2E (Kubernetes ${{ matrix.kubernetes-version }})
runs-on: ubuntu-26.04
needs: kubernetes-versions
strategy:
fail-fast: false
matrix:
kubernetes-version: ${{ fromJSON(needs.kubernetes-versions.outputs.matrix) }}

steps:
- name: Checkout code
Expand Down Expand Up @@ -65,20 +83,30 @@ jobs:
- name: Setup Kind cluster
id: setup-kind
uses: ./.github/actions/kind-create
with:
kind_k8s_version: v${{ matrix.kubernetes-version }}

- name: Load image into kind cluster
run: |
kind load docker-image $REPOSITORY:$GITHUB_SHA --name e2e-kind


- name: Download dependencies
timeout-minutes: 20
run: |
# Download dependencies for all modules
go mod download
cd api && go mod download && cd ..
cd properties && go mod download && cd ..
cd tests/e2e && go mod download && cd ../..

for module in . api properties tests/e2e; do
attempt=1
until timeout --kill-after=10s 180s go -C "$module" mod download; do
if [ "$attempt" -ge 5 ]; then
echo "::error::Dependency download for ${module} failed after ${attempt} attempts"
exit 1
fi
delay=$((5 << (attempt - 1)))
echo "::warning::Dependency download for ${module} failed (attempt ${attempt}/5); retrying in ${delay}s"
sleep "$delay"
attempt=$((attempt + 1))
done
done

- name: Run E2E tests
env:
Expand Down
55 changes: 35 additions & 20 deletions .github/workflows/update-go-deps.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
name: Update Go Dependencies
name: Update Dependencies

on:
schedule:
Expand All @@ -12,7 +12,7 @@ permissions:

jobs:
update-dependencies:
name: Update Go Dependencies
name: Update Dependencies
runs-on: ubuntu-26.04

steps:
Expand All @@ -30,6 +30,9 @@ jobs:
- name: Update Go dependencies
run: make update-go-deps

- name: Update supported Kubernetes targets
run: make update-kubernetes-versions

- name: Tidy dependencies
run: make tidy

Expand All @@ -43,33 +46,45 @@ jobs:
id: check_changes
run: |
if [ -n "$(git status --porcelain)" ]; then
echo "has_changes=true" >> $GITHUB_OUTPUT
echo "has_changes=true" >> "$GITHUB_OUTPUT"
else
echo "has_changes=false" >> $GITHUB_OUTPUT
echo "has_changes=false" >> "$GITHUB_OUTPUT"
fi

- name: Create Pull Request
if: steps.check_changes.outputs.has_changes == 'true'
uses: peter-evans/create-pull-request@v8
with:
token: ${{ secrets.PAT }}
commit-message: 'chore(deps): update Go dependencies'
title: 'chore(deps): update Go dependencies'
commit-message: 'chore(deps): update Go dependencies and Kubernetes targets'
title: 'chore(deps): update Go dependencies and Kubernetes targets'
body: |
## Automated Go Dependencies Update

This PR updates Go module dependencies to their latest versions.

### Changes made:
- Updated Go module dependencies via `make update-go-deps`
- Bumped `CONTROLLER_GEN_VERSION` in the Makefile to the latest release
- Regenerated manifests via `make generate manifests`
- Applied formatting and linting fixes via `make fmt lint-fix`

Please review the changes and ensure all tests pass before merging.

---
*Generated by https://github.com/adobe/koperator/blob/main/.github/workflows/update-go-deps.yml*
## Description

Update Go dependencies/tooling and the managed Kubernetes test targets.
Regenerate manifests and derived support documentation. References #386.

Kubernetes targets follow the oldest/newest upstream-supported minors.
Review any increase to the minimum supported version. CI validation is required.

## Type of Change
<!-- Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields. -->
- [ ] Bug Fix
- [ ] New Feature
- [ ] Breaking Change
- [ ] Refactor
- [ ] Documentation
- [x] Other (please describe): dependency and Kubernetes target updates

## Checklist
<!-- Place an '[x]' (no spaces) in all applicable fields. Please remove unrelated fields. -->
- [ ] I have read the [contributing guidelines](../CONTRIBUTING.md)
- [x] Existing issues have been referenced (where applicable)
- [ ] I have verified this change is not present in other open pull requests
- [ ] Functionality is documented
- [ ] All code style checks pass
- [ ] New code contribution is covered by automated tests
- [ ] All new and existing tests pass
branch: automated/update-go-deps
delete-branch: true
labels: |
Expand Down
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -48,3 +48,4 @@ manager_image_patch.yaml
**/go.work
**/go.work.sum
.claude/worktrees/**
/improvement-plan.md
13 changes: 7 additions & 6 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
- **Language**: Go
- **Framework**: Kubebuilder v2 with controller-runtime
- **Primary CRD**: `KafkaCluster` (v1beta1)
- **Minimum Kubernetes server version**: 1.35.0; CI targets: 1.35.0 and 1.37.0, managed in config/kubernetes-versions.env
- **Key Features**: Fine-grained broker management, Cruise Control integration, multiple ingress options (Envoy, Istio, Contour)

## Architecture
Expand All @@ -27,7 +28,7 @@ This is a standard Kubernetes operator using:
6. **AlertManagerForKafka** - Self-healing based on Prometheus alerts

### Multi-Module Structure
Uses Go workspaces with 5+ modules:
Uses multiple Go modules with local replacements:
- Main: `/go.mod`
- API: `/api/go.mod`
- Properties parser: `/properties/go.mod`
Expand Down Expand Up @@ -88,7 +89,7 @@ make run # Run operator locally (outside cluster)

### Important Make Variables
- `IMG` - Operator image name (default: `ghcr.io/adobe/koperator:latest`)
- `ENVTEST_K8S_VERSION` - Kubernetes version for envtest assets (default: 1.36.2)
- `ENVTEST_K8S_VERSION` - Defaults to KUBERNETES_MAX_VERSION from config/kubernetes-versions.env; use `make test ENVTEST_K8S_VERSION=1.35.0` for the lower CI boundary

## Development Workflows

Expand Down Expand Up @@ -421,11 +422,11 @@ GitHub Actions workflows (`.github/workflows/`):

## Key Dependencies

- **controller-runtime** v0.22.4 - Operator framework
- **k8s.io/*** v0.34.3 - Kubernetes client libraries
- **IBM/sarama** v1.46.3 - Kafka client
- **controller-runtime** v0.25.3 - Operator framework
- **k8s.io/*** v0.37.1 - Kubernetes client libraries
- **IBM/sarama** v1.61.1 - Kafka client
- **Ginkgo** v2 - BDD testing framework
- **cert-manager** v1.19.2 - Certificate management integration
- **cert-manager** v1.21.2 - Certificate management integration

## Troubleshooting

Expand Down
14 changes: 11 additions & 3 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,15 @@ SHELL = /bin/bash
.SHELLFLAGS := -eu -o pipefail -c

PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
include $(PROJECT_DIR)/config/kubernetes-versions.env
BIN_DIR := $(PROJECT_DIR)/bin
BOILERPLATE_DIR := $(PROJECT_DIR)/hack/boilerplate

# Image URL to use all building/pushing image targets
TAG ?= $(shell git describe --tags --dirty --always --abbrev=0 --match '[0-9].*[0-9].*[0-9]*' 2>/dev/null)
IMG ?= ghcr.io/adobe/kafka-operator:$(TAG)

# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
# Generate CRDs for the supported Kubernetes server range (no version conversion).
CRD_OPTIONS ?= "crd"

RELEASE_TYPE ?= p
Expand All @@ -21,8 +22,8 @@ REL_TAG = $(shell ./scripts/increment_version.sh -${RELEASE_TYPE} ${TAG})
GOLANGCI_VERSION = 2.15.0 # renovate: datasource=github-releases depName=golangci/golangci-lint
LICENSEI_VERSION = 0.9.0 # renovate: datasource=github-releases depName=goph/licensei
CONTROLLER_GEN_VERSION = v0.22.0 # kept in sync with kubernetes-sigs/controller-tools by `make update-go-deps`
ENVTEST_K8S_VERSION = 1.37.0 # renovate: datasource=github-releases depName=kubernetes-sigs/controller-tools extractVersion=^envtest-v(?<version>.+)$
SETUP_ENVTEST_VERSION := latest
ENVTEST_K8S_VERSION = $(KUBERNETES_MAX_VERSION)
SETUP_ENVTEST_VERSION = v0.25.3 # renovate: datasource=github-releases depName=kubernetes-sigs/controller-runtime
ADDLICENSE_VERSION = 1.2.0 # renovate: datasource=github-releases depName=google/addlicense
GOTEMPLATE_VERSION = 3.12.1 # renovate: datasource=github-releases depName=coveooss/gotemplate
MOCKGEN_VERSION = 0.6.0 # renovate: datasource=github-releases depName=uber-go/mock
Expand Down Expand Up @@ -335,6 +336,13 @@ define update-module-deps
go mod tidy
endef

.PHONY: update-kubernetes-versions check-kubernetes-versions
update-kubernetes-versions: ## Update supported Kubernetes pins and derived documentation.
go run ./scripts/update-kubernetes-versions -update

check-kubernetes-versions: ## Check committed Kubernetes pins and derived documentation offline.
go run ./scripts/update-kubernetes-versions -check

update-go-deps: ## Update Go modules dependencies.
@echo "Updating third_party modules first (deepest to shallowest)..."
@for gomod in $$(find ./third_party -name "go.mod" 2>/dev/null | awk '{print length, $$0}' | sort -rn | cut -d' ' -f2-); do \
Expand Down
6 changes: 5 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -62,9 +62,13 @@ If you want to learn more about our design motivations and the scenarios that dr

This quick start guide will walk you through the process of deploying Koperator on an existing Kubernetes cluster and provisioning a Kafka cluster using its custom resources.

### Kubernetes support

Koperator requires **Kubernetes 1.35.0 or newer** and tracks upstream-maintained minor releases. CI tests 1.35.0 and 1.37.0 using envtest and Kind. Pins live in [config/kubernetes-versions.env](config/kubernetes-versions.env); the weekly dependency updater selects the oldest/newest supported minors and the newest patches available for both test tools. Kubernetes client-library versions do not change this policy.

### Prerequisites

To complete this guide, you will need a Kubernetes cluster (with a suggested minimum of 6 vCPUs and 8 GB RAM). You can run the cluster locally using Kind or Minikube.
To complete this guide, you will need a Kubernetes 1.35.0+ cluster (with a suggested minimum of 6 vCPUs and 8 GB RAM). You can run the cluster locally using Kind or Minikube.

> The quick start will help you set up a functioning Kafka cluster on Kubernetes. However, it does not include guidance on the installation of Prometheus and cert-manager, which are necessary for some of the more advanced functionality.

Expand Down
1 change: 1 addition & 0 deletions charts/kafka-operator/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -5,3 +5,4 @@ description: kafka-operator manages Kafka deployments on Kubernetes
sources:
- https://github.com/adobe/koperator
appVersion: "0.32.0"
kubeVersion: ">=1.35.0-0"
Loading