Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions lib/builds/manager.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"context"
_ "embed"
"encoding/json"
"errors"
"fmt"
"log/slog"
"net"
Expand Down Expand Up @@ -958,20 +959,40 @@ func (m *manager) updateBuildComplete(id string, status string, digest *string,
m.notifyStatusChange(id, status)
}

// imageImportRetryInterval is the delay between retries when the build's
// image has not yet been imported by the registry. Var so tests can shorten it.
var imageImportRetryInterval = time.Second

// waitForImageReady blocks until the build's image reaches a terminal state.
// imageRef should be the short repo name (e.g., "builds/abc123" or "myapp")
// matching what triggerConversion stores in the image manager.
// This ensures that when a build reports "ready", the image is actually usable
// for instance creation (fixes KERNEL-863 race condition).
//
// The registry imports pushed images asynchronously, and under concurrent
// build bursts the import (OCI layout copy) can outlast the image manager's
// internal existence deadline. By the time this is called the builder has
// already pushed the image, so "not found" means "import still in flight",
// not "push failed" — keep retrying until the build context expires.
func (m *manager) waitForImageReady(ctx context.Context, imageRef string) error {
m.logger.Debug("waiting for image to be ready", "image_ref", imageRef)

if err := m.imageManager.WaitForReady(ctx, imageRef); err != nil {
return err
for {
err := m.imageManager.WaitForReady(ctx, imageRef)
if err == nil {
m.logger.Debug("image is ready", "image_ref", imageRef)
return nil
}
if !errors.Is(err, images.ErrNotFound) {
return err
}
m.logger.Info("build image not yet imported by registry, retrying", "image_ref", imageRef)

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Retry matches conversion not-found errors

Low Severity

waitForImageReady retries on any error wrapping images.ErrNotFound, but WaitForReady can return that sentinel from a real conversion failure via the status-event path (image conversion failed with %w). Those failures are then misclassified as “import still in flight,” causing a spurious retry and misleading log before the next attempt sees failed status and exits.

Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 24d9b52. Configure here.

select {
case <-ctx.Done():
return err
case <-time.After(imageImportRetryInterval):
}
}

m.logger.Debug("image is ready", "image_ref", imageRef)
return nil
}

// subscribeToStatus adds a subscriber channel for status updates on a build
Expand Down
6 changes: 6 additions & 0 deletions lib/builds/manager_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,12 @@ func (m *mockImageManager) WaitForReady(ctx context.Context, name string) error
status = img.Status
}
m.mu.RUnlock()
// Mirror the real manager: a missing image is reported as not found
// (the real implementation polls for 30s first; return immediately
// to keep tests fast).
if !ok {
return fmt.Errorf("get image: %w", images.ErrNotFound)
}
switch status {
case images.StatusReady:
return nil
Expand Down
57 changes: 57 additions & 0 deletions lib/builds/race_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,63 @@ func TestWaitForImageReady_WaitsForConversion(t *testing.T) {
assert.GreaterOrEqual(t, elapsed, 200*time.Millisecond, "should have waited for conversion")
}

// TestWaitForImageReady_RetriesWhileImportInFlight tests that waitForImageReady
// keeps waiting when the image doesn't exist yet. The registry imports pushed
// images asynchronously, so under concurrent build bursts the image record can
// appear well after the builder reports success — "not found" at this point
// means the import is still in flight, not that the push failed.
func TestWaitForImageReady_RetriesWhileImportInFlight(t *testing.T) {
mgr, _, _, imageMgr, tempDir := setupTestManagerWithImageMgr(t)
defer os.RemoveAll(tempDir)

prevInterval := imageImportRetryInterval
imageImportRetryInterval = 20 * time.Millisecond
defer func() { imageImportRetryInterval = prevInterval }()

ctx := context.Background()
buildID := "test-build-late-import"
imageRef := "builds/" + buildID

// Image does not exist at all yet. Simulate the registry's async import
// registering it (pending) and conversion completing shortly after.
go func() {
time.Sleep(100 * time.Millisecond)
imageMgr.mu.Lock()
imageMgr.images[imageRef] = &images.Image{
Name: imageRef,
Status: images.StatusPending,
}
imageMgr.mu.Unlock()
time.Sleep(100 * time.Millisecond)
imageMgr.SetImageStatus(imageRef, images.StatusReady)
}()

start := time.Now()
err := mgr.waitForImageReady(ctx, imageRef)
elapsed := time.Since(start)

require.NoError(t, err)
assert.GreaterOrEqual(t, elapsed, 200*time.Millisecond, "should have waited for the import to land")
}

// TestWaitForImageReady_NotFoundUntilContextExpires tests that a build whose
// image never gets imported still fails once the build context expires.
func TestWaitForImageReady_NotFoundUntilContextExpires(t *testing.T) {
mgr, _, _, _, tempDir := setupTestManagerWithImageMgr(t)
defer os.RemoveAll(tempDir)

prevInterval := imageImportRetryInterval
imageImportRetryInterval = 20 * time.Millisecond
defer func() { imageImportRetryInterval = prevInterval }()

ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
defer cancel()

err := mgr.waitForImageReady(ctx, "builds/test-build-never-imported")
require.Error(t, err)
assert.ErrorIs(t, err, images.ErrNotFound)
}

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Flaky never-imported timeout test

Medium Severity

TestWaitForImageReady_NotFoundUntilContextExpires expects images.ErrNotFound when the context expires, but that only happens if the mock returns not-found before noticing cancellation. If ctx is already done when the next WaitForReady starts, the mock returns context.DeadlineExceeded and the outer loop exits immediately, so the assertion flakes. The real WaitForReady also returns ctx.Err() during its existence poll, so this test does not match production timeout behavior.

Additional Locations (2)
Fix in Cursor Fix in Web

Reviewed by Cursor Bugbot for commit 24d9b52. Configure here.


// TestWaitForImageReady_Timeout tests that waitForImageReady times out if image never becomes ready
func TestWaitForImageReady_ContextCancelled(t *testing.T) {
mgr, _, _, imageMgr, tempDir := setupTestManagerWithImageMgr(t)
Expand Down
Loading