Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
41 changes: 29 additions & 12 deletions controllers/cruisecontroloperation_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -243,24 +243,41 @@ func (r *CruiseControlOperationReconciler) Reconcile(ctx context.Context, reques
}
}

conflictRetryFunction := func() error {
if err = updateResult(log, cruseControlTaskResult, ccOperationExecution, true); err != nil {
return err
}
err = r.Status().Update(ctx, ccOperationExecution)
if apiErrors.IsConflict(err) {
err = r.Get(ctx, client.ObjectKey{Name: ccOperationExecution.GetName(), Namespace: ccOperationExecution.GetNamespace()}, ccOperationExecution)
}
return err
}
// Protection to avoid re-execution task when status update is failed
if err := util.RetryOnConflict(util.DefaultBackOffForConflict, conflictRetryFunction); err != nil {
// Retry persistence without resubmitting the external task.
if err := r.persistExecutionResult(ctx, log, ccOperationExecution, cruseControlTaskResult); err != nil {
return requeueWithError(log, "could not update the result of the Cruise Control user task execution to the CruiseControlOperation status", err)
}

return reconciled()
}

func (r *CruiseControlOperationReconciler) persistExecutionResult(
ctx context.Context,
log logr.Logger,
operation *banzaiv1alpha1.CruiseControlOperation,
result *scale.Result,
) error {
firstAttempt := true
return util.RetryOnConflict(util.DefaultBackOffForConflict, func() error {
latest := &banzaiv1alpha1.CruiseControlOperation{}
if firstAttempt {
latest = operation.DeepCopy()
} else if err := r.DirectClient.Get(ctx, client.ObjectKeyFromObject(operation), latest); err != nil {
return err
}
firstAttempt = false
if latest.UID != operation.UID || latest.CurrentTask() == nil || latest.CurrentTaskID() != operation.CurrentTaskID() {
return errors.New("CruiseControlOperation identity or current task changed after external task submission")
}
// Preserve the stop operation selected in memory during finalization.
latest.Status.CurrentTask.Operation = operation.CurrentTaskOperation()
if err := updateResult(log, result, latest, true); err != nil {
return err
}
return r.Status().Update(ctx, latest)
})
}

func (r *CruiseControlOperationReconciler) addFinalizer(ctx context.Context, currentCCOperation *banzaiv1alpha1.CruiseControlOperation) error {
// examine DeletionTimestamp to determine if object is under deletion
if currentCCOperation.DeletionTimestamp.IsZero() {
Expand Down
183 changes: 183 additions & 0 deletions controllers/cruisecontroloperation_controller_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,206 @@ package controllers

import (
"context"
"errors"
"testing"
"time"

"github.com/go-logr/logr"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"go.uber.org/mock/gomock"
apierrors "k8s.io/apimachinery/pkg/api/errors"
v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/client/interceptor"

"github.com/banzaicloud/koperator/api/v1alpha1"
"github.com/banzaicloud/koperator/api/v1beta1"
mocks "github.com/banzaicloud/koperator/controllers/tests/mocks"
"github.com/banzaicloud/koperator/pkg/scale"
)

func TestOperationResultPersistsAcrossStatusConflicts(t *testing.T) {
for _, tt := range []struct {
name string
retry bool
deleting bool
replaced bool
taskRemoved bool
taskChanged bool
exhausted bool
conflicts int
wantRetries int
wantFailed int
wantOp v1alpha1.CruiseControlTaskOperation
}{
{name: "first submission without conflict", wantOp: v1alpha1.OperationAddBroker},
{name: "retry without conflict preserves history once", retry: true, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker},
{name: "stop submission without conflict retains operation", deleting: true, wantOp: v1alpha1.OperationStopExecution},
{name: "first submission", conflicts: 1, wantOp: v1alpha1.OperationAddBroker},
{name: "retry preserves history once", retry: true, conflicts: 2, wantRetries: 3, wantFailed: 2, wantOp: v1alpha1.OperationAddBroker},
{name: "stop submission retains operation", deleting: true, conflicts: 1, wantOp: v1alpha1.OperationStopExecution},
{name: "replacement object rejects stale result", replaced: true, conflicts: 1},
{name: "removed task rejects stale result", taskRemoved: true, conflicts: 1},
{name: "newer task rejects stale result", taskChanged: true, conflicts: 1},
{name: "exhausted conflicts return error", exhausted: true, conflicts: 1000},
} {
t.Run(tt.name, func(t *testing.T) {
ctx := context.Background()
scheme := runtime.NewScheme()
if err := v1alpha1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
if err := v1beta1.AddToScheme(scheme); err != nil {
t.Fatal(err)
}
cluster := &v1beta1.KafkaCluster{ObjectMeta: v1.ObjectMeta{Name: "cluster", Namespace: "ns"}}
operation := &v1alpha1.CruiseControlOperation{
ObjectMeta: v1.ObjectMeta{
Name: "operation", Namespace: "ns", UID: "operation-uid",
Labels: map[string]string{v1beta1.KafkaCRLabelKey: "cluster"},
},
Spec: v1alpha1.CruiseControlOperationSpec{ErrorPolicy: v1alpha1.ErrorPolicyRetry},
}
operation.Status.ErrorPolicy = v1alpha1.ErrorPolicyRetry
operation.Status.CurrentTask = &v1alpha1.CruiseControlTask{Operation: v1alpha1.OperationAddBroker}
taskResult := &scale.Result{
TaskID: "submitted-task", State: v1beta1.CruiseControlTaskInExecution,
StartedAt: "Mon, 02 Jan 2006 15:04:05 GMT", ResponseStatusCode: 202,
RequestURL: "http://cruise-control/add_broker",
}
var previous []*scale.Result
if tt.retry {
operation.Status.CurrentTask.ID = "failed-task"
operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskCompletedWithError
operation.Status.CurrentTask.Finished = &v1.Time{Time: time.Date(2006, 1, 2, 15, 5, 0, 0, time.UTC)}
operation.Status.RetryCount = 2
operation.Status.FailedTasks = []v1alpha1.CruiseControlTask{{ID: "earlier-failed-task"}}
previous = []*scale.Result{{TaskID: "failed-task", State: v1beta1.CruiseControlTaskCompletedWithError}}
}
if tt.deleting {
operation.DeletionTimestamp = &v1.Time{Time: time.Now()}
operation.Finalizers = []string{ccOperationFinalizerGroup}
operation.Status.CurrentTask.ID = "running-task"
operation.Status.CurrentTask.State = v1beta1.CruiseControlTaskInExecution
previous = []*scale.Result{{TaskID: "running-task", State: v1beta1.CruiseControlTaskInExecution}}
}
scaler := mocks.NewMockCruiseControlScaler(gomock.NewController(t))
scaler.EXPECT().Status(gomock.Any()).Return(scale.StatusTaskResult{Status: &scale.CruiseControlStatus{
AnalyzerReady: true, MonitorReady: true, ExecutorReady: true,
}}, nil)
var taskIDs []any
if len(previous) != 0 {
taskIDs = []any{previous[0].TaskID}
}
scaler.EXPECT().UserTasks(gomock.Any(), taskIDs...).Return(previous, nil)
if tt.deleting {
scaler.EXPECT().StopExecution(gomock.Any()).Return(taskResult, nil).Times(1)
} else {
scaler.EXPECT().AddBrokersWithParams(gomock.Any(), operation.CurrentTaskParameters()).Return(taskResult, nil).Times(1)
}
var resultWrites int
kubeClient := fake.NewClientBuilder().WithScheme(scheme).
WithObjects(cluster, operation).
WithStatusSubresource(operation).
WithInterceptorFuncs(interceptor.Funcs{
SubResourceUpdate: func(ctx context.Context, c client.Client, subresource string, obj client.Object, opts ...client.SubResourceUpdateOption) error {
if op, ok := obj.(*v1alpha1.CruiseControlOperation); ok && subresource == "status" && op.CurrentTaskID() == "submitted-task" {
resultWrites++
if resultWrites <= tt.conflicts {
latest := &v1alpha1.CruiseControlOperation{}
if err := c.Get(ctx, client.ObjectKeyFromObject(obj), latest); err != nil {
return err
}
latest.Annotations = map[string]string{"other-writer": "preserved"}
if err := c.Update(ctx, latest); err != nil {
return err
}
if tt.replaced {
latest.Finalizers = nil
if err := c.Update(ctx, latest); err != nil {
return err
}
if err := c.Delete(ctx, latest); err != nil {
return err
}
latest.ResourceVersion = ""
latest.UID = "replacement-uid"
if err := c.Create(ctx, latest); err != nil {
return err
}
}
if tt.taskRemoved || tt.taskChanged {
if tt.taskRemoved {
latest.Status.CurrentTask = nil
} else {
latest.Status.CurrentTask.ID = "newer-task"
}
if err := c.Status().Update(ctx, latest); err != nil {
return err
}
}
return apierrors.NewConflict(schema.GroupResource{Group: "kafka.banzaicloud.io", Resource: "cruisecontroloperations"}, obj.GetName(), errors.New("injected concurrent update"))
}
}
return c.SubResource(subresource).Update(ctx, obj, opts...)
},
}).Build()
var directReads int
directClient := interceptor.NewClient(kubeClient, interceptor.Funcs{
Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
directReads++
assert.Positive(t, resultWrites, "direct GET must follow a conflicting result write")
return c.Get(ctx, key, obj, opts...)
},
})
reconciler := &CruiseControlOperationReconciler{
Client: kubeClient, DirectClient: directClient, Scheme: scheme,
ScaleFactory: func(context.Context, *v1beta1.KafkaCluster) (scale.CruiseControlScaler, error) {
return scaler, nil
},
}
_, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(operation)})
if err != nil && !tt.replaced && !tt.taskRemoved && !tt.taskChanged && !tt.exhausted {
t.Fatal(err)
}
persisted := &v1alpha1.CruiseControlOperation{}
if err := kubeClient.Get(ctx, client.ObjectKeyFromObject(operation), persisted); err != nil {
t.Fatal(err)
}
if tt.replaced || tt.taskRemoved || tt.taskChanged || tt.exhausted {
if err == nil {
t.Error("reported success without persisting the result to the original task")
}
if persisted.CurrentTaskID() == "submitted-task" {
t.Error("attached stale external result to a replaced operation or removed task")
}
return
}
assert.Equal(t, "submitted-task", persisted.CurrentTaskID())
assert.Equal(t, v1beta1.CruiseControlTaskInExecution, persisted.CurrentTaskState())
assert.Equal(t, tt.wantOp, persisted.CurrentTaskOperation())
task := persisted.CurrentTask()
require.NotNil(t, task)
assert.Equal(t, taskResult.RequestURL, task.HTTPRequest)
assert.Equal(t, &taskResult.ResponseStatusCode, task.HTTPResponseCode)
require.NotNil(t, task.Started)
assert.True(t, task.Started.Time.Equal(time.Date(2006, 1, 2, 15, 4, 5, 0, time.UTC)))
assert.Equal(t, tt.wantRetries, persisted.Status.RetryCount)
assert.Len(t, persisted.Status.FailedTasks, tt.wantFailed)
if tt.conflicts > 0 {
assert.Equal(t, "preserved", persisted.Annotations["other-writer"])
}
assert.Equal(t, tt.conflicts+1, resultWrites)
assert.Equal(t, tt.conflicts, directReads)
})
}
}

func createCCRetryExecutionOperation(createTime time.Time, id string, operation v1alpha1.CruiseControlTaskOperation) *v1alpha1.CruiseControlOperation {
return &v1alpha1.CruiseControlOperation{
ObjectMeta: v1.ObjectMeta{
Expand Down
Loading