Give the first credential fetch 30s and keep AWS error causes

aws/aws.go

@@ -4,7 +4,6 @@ package aws
 
 import (
 	"context"
-	"fmt"
 	"strings"
 	"time"
 
@@ -15,6 +14,24 @@ import (
 	"github.com/cashapp/blip/v2"
 )
 
+// regionError is returned when AWS region auto-detection fails. It keeps the
+// IMDS error so callers can inspect it with errors.Is and errors.As.
+type regionError struct {
+	err error
+}
+
+func (e regionError) Error() string {
+	return "cannot auto-detect AWS region (EC2 IMDS query failed)"
+}
+
+func (e regionError) Unwrap() error {
+	return e.err
+}
+
+// imdsOptions configures the IMDS client that Region uses. Tests set it to use
+// a local IMDS server.
+var imdsOptions = imds.Options{}
+
 type ConfigFactory struct {
 	region string
 }
@@ -27,7 +44,7 @@ func (f *ConfigFactory) Make(ba blip.AWS, endpoint string) (aws.Config, error) {
 		ba.Region, err = Region(ctx)
 		if err != nil {
 			blip.Debug("cannot auto-detect AWS region: %s", err)
-			return aws.Config{}, fmt.Errorf("cannot auto-detect AWS region (EC2 IMDS query failed)")
+			return aws.Config{}, regionError{err: err}
 		}
 		if f.region == "" {
 			blip.Debug("set factory region to %s", ba.Region)
@@ -60,7 +77,7 @@ func (f *ConfigFactory) Make(ba blip.AWS, endpoint string) (aws.Config, error) {
 // If the region cannot be detect, it returns an empty string.
 func Region(ctx context.Context) (string, error) {
 	blip.Debug("auto-detect AWS region from IMDS")
-	client := imds.New(imds.Options{})
+	client := imds.New(imdsOptions)
 	ec2, err := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
 	if err != nil {
 		return "", err

aws/aws_test.go

@@ -0,0 +1,41 @@
+// Copyright 2026 Block, Inc.
+
+package aws
+
+import (
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/aws/aws-sdk-go-v2/feature/ec2/imds"
+
+	"github.com/cashapp/blip/v2"
+)
+
+func TestConfigFactoryWrapsRegionError(t *testing.T) {
+	// IMDS that fails every request, so region auto-detection fails
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not found", http.StatusNotFound)
+	}))
+	defer server.Close()
+	imdsOptions = imds.Options{Endpoint: server.URL}
+	defer func() { imdsOptions = imds.Options{} }()
+
+	f := &ConfigFactory{}
+	_, err := f.Make(blip.AWS{Region: "auto"}, "")
+	if err == nil {
+		t.Fatal("got nil error, expected region auto-detection error")
+	}
+	if got, expect := err.Error(), "cannot auto-detect AWS region (EC2 IMDS query failed)"; got != expect {
+		t.Errorf("got error %q, expected %q", got, expect)
+	}
+	// The IMDS HTTP response error, which has the status code
+	var respErr interface{ HTTPStatusCode() int }
+	if !errors.As(err, &respErr) {
+		t.Fatalf("errors.As(%v, HTTPStatusCode() int) = false, expected the IMDS error", err)
+	}
+	if respErr.HTTPStatusCode() != http.StatusNotFound {
+		t.Errorf("got IMDS status %d, expected %d", respErr.HTTPStatusCode(), http.StatusNotFound)
+	}
+}

aws/secret.go

@@ -33,7 +33,7 @@ func (s Secret) GetSecret(ctx context.Context) (map[string]interface{}, error) {
 
 	var v map[string]interface{}
 	if err := json.Unmarshal(payload, &v); err != nil {
-		return nil, fmt.Errorf("cannot decode secret string as map[string]interface{}: %s", err)
+		return nil, fmt.Errorf("cannot decode secret string as map[string]interface{}: %w", err)
 	}
 	if v == nil {
 		return nil, fmt.Errorf("secret value is 'null' literal")
@@ -49,7 +49,7 @@ func (s Secret) GetSecretPayload(ctx context.Context) ([]byte, error) {
 
 	sv, err := s.client.GetSecretValue(ctx, input)
 	if err != nil {
-		return nil, fmt.Errorf("Secrets Manager API error: %s", err)
+		return nil, fmt.Errorf("Secrets Manager API error: %w", err)
 	}
 	name := ""
 	if sv.Name != nil {

aws/secret_test.go

@@ -0,0 +1,104 @@
+// Copyright 2026 Block, Inc.
+
+package aws_test
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+	"github.com/aws/aws-sdk-go-v2/credentials"
+	"github.com/aws/aws-sdk-go-v2/service/secretsmanager/types"
+
+	blipAWS "github.com/cashapp/blip/v2/aws"
+)
+
+func testSecretsManager(t *testing.T, handler http.HandlerFunc) aws.Config {
+	t.Helper()
+	server := httptest.NewServer(handler)
+	t.Cleanup(server.Close)
+	return aws.Config{
+		Credentials: credentials.NewStaticCredentialsProvider("access-key", "secret-key", ""),
+		EndpointResolver: aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) {
+			return aws.Endpoint{URL: server.URL, SigningRegion: "us-east-1"}, nil
+		}),
+		HTTPClient: server.Client(),
+		Region:     "us-east-1",
+		Retryer: func() aws.Retryer {
+			return aws.NopRetryer{}
+		},
+	}
+}
+
+func TestGetSecretPayloadWrapsAPIError(t *testing.T) {
+	cfg := testSecretsManager(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		w.WriteHeader(http.StatusBadRequest)
+		w.Write([]byte(`{"__type":"ResourceNotFoundException","message":"no such secret"}`))
+	})
+
+	_, err := blipAWS.NewSecret("test-secret", cfg).GetSecretPayload(context.Background())
+	if err == nil {
+		t.Fatal("got nil error, expected API error")
+	}
+	if !strings.HasPrefix(err.Error(), "Secrets Manager API error: ") {
+		t.Errorf("got error %q, expected prefix 'Secrets Manager API error: '", err)
+	}
+	var notFound *types.ResourceNotFoundException
+	if !errors.As(err, &notFound) {
+		t.Errorf("errors.As(%v, *types.ResourceNotFoundException) = false, expected true", err)
+	}
+}
+
+func TestGetSecretPayloadWrapsCanceled(t *testing.T) {
+	// The first call might have to fetch AWS credentials (e.g. IRSA). If the
+	// caller's context expires first, the SDK credentials cache returns
+	// *aws.RequestCanceledError, which callers must be able to detect.
+	cfg := testSecretsManager(t, func(w http.ResponseWriter, r *http.Request) {
+		t.Error("Secrets Manager called without credentials")
+	})
+	cfg.Credentials = aws.NewCredentialsCache(aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
+		select {
+		case <-time.After(2 * time.Second):
+		case <-ctx.Done():
+		}
+		return aws.Credentials{}, errors.New("slow credentials")
+	}))
+
+	ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+	defer cancel()
+	_, err := blipAWS.NewSecret("test-secret", cfg).GetSecretPayload(ctx)
+	if err == nil {
+		t.Fatal("got nil error, expected request canceled error")
+	}
+	var canceled *aws.RequestCanceledError
+	if !errors.As(err, &canceled) {
+		t.Errorf("errors.As(%v, *aws.RequestCanceledError) = false, expected true", err)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("errors.Is(%v, context.DeadlineExceeded) = false, expected true", err)
+	}
+}
+
+func TestGetSecretWrapsDecodeError(t *testing.T) {
+	cfg := testSecretsManager(t, func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/x-amz-json-1.1")
+		json.NewEncoder(w).Encode(map[string]string{
+			"Name":         "test-secret",
+			"SecretString": "not-json",
+			"VersionId":    "test-version",
+		})
+	})
+
+	_, err := blipAWS.NewSecret("test-secret", cfg).GetSecret(context.Background())
+	var syntaxErr *json.SyntaxError
+	if !errors.As(err, &syntaxErr) {
+		t.Errorf("errors.As(%v, *json.SyntaxError) = false, expected true", err)
+	}
+}

blip.go

@@ -121,7 +121,7 @@ func DefaultPasswordSecretParser(_ context.Context, cfg ConfigMonitor, payload [
 
 	var secretPayload map[string]interface{}
 	if err := json.Unmarshal(payload, &secretPayload); err != nil {
-		return fmt.Errorf("cannot decode secret as JSON object: %s", err)
+		return fmt.Errorf("cannot decode secret as JSON object: %w", err)
 	}
 	if secretPayload == nil {
 		return fmt.Errorf("secret value is 'null' literal")

dbconn/credential_timeout_test.go

@@ -0,0 +1,125 @@
+// Copyright 2026 Block, Inc.
+
+package dbconn
+
+import (
+	"context"
+	"errors"
+	"testing"
+	"time"
+
+	"github.com/aws/aws-sdk-go-v2/aws"
+
+	"github.com/cashapp/blip/v2"
+)
+
+// slowCredentialsConfig returns an AWS config for a Secrets Manager stub
+// with AWS credentials that take delay to fetch, like the first IRSA
+// AssumeRoleWithWebIdentity call.
+func slowCredentialsConfig(t *testing.T, delay time.Duration) aws.Config {
+	t.Helper()
+	awscfg, cleanup := testPasswordSecretConfig(t, `{"password":"secret-pass"}`)
+	t.Cleanup(cleanup)
+	awscfg.Credentials = aws.NewCredentialsCache(aws.CredentialsProviderFunc(func(ctx context.Context) (aws.Credentials, error) {
+		select {
+		case <-time.After(delay):
+			return aws.Credentials{AccessKeyID: "access-key", SecretAccessKey: "secret-key", Source: "test"}, nil
+		case <-ctx.Done():
+			return aws.Credentials{}, ctx.Err()
+		}
+	}))
+	return awscfg
+}
+
+func passwordSecretMonitor() blip.ConfigMonitor {
+	return blip.ConfigMonitor{
+		MonitorId: "m1",
+		Hostname:  "127.0.0.1:33997",
+		Username:  "config-user",
+		AWS: blip.ConfigAWS{
+			PasswordSecret: "test-secret",
+			Region:         "us-east-1",
+		},
+	}
+}
+
+func TestMakeCredentialTimeoutDefault(t *testing.T) {
+	// The first credentials from AWS can take seconds (fetching AWS credentials
+	// via IRSA, AssumeRole, etc.), so Make must wait longer than the old 500ms.
+	awscfg := slowCredentialsConfig(t, 700*time.Millisecond)
+	f := NewConnFactory(testAWSConfigFactory{cfg: awscfg}, nil)
+
+	db, _, err := f.Make(passwordSecretMonitor())
+	if err != nil {
+		t.Fatalf("got error %s, expected nil", err)
+	}
+	db.Close()
+}
+
+func TestMakeCredentialTimeoutOption(t *testing.T) {
+	awscfg := slowCredentialsConfig(t, 2*time.Second)
+	f := NewConnFactory(testAWSConfigFactory{cfg: awscfg}, nil, WithCredentialTimeout(50*time.Millisecond))
+
+	t0 := time.Now()
+	db, _, err := f.Make(passwordSecretMonitor())
+	elapsed := time.Since(t0)
+	if err == nil {
+		db.Close()
+		t.Fatal("got nil error, expected credential timeout")
+	}
+	if elapsed > 5*time.Second {
+		t.Errorf("Make took %s, expected about 50ms", elapsed)
+	}
+
+	// The SDK error must survive wrapping so callers can tell a timeout or
+	// cancellation from, e.g., a bad secret
+	var canceled *aws.RequestCanceledError
+	if !errors.As(err, &canceled) {
+		t.Errorf("errors.As(%v, *aws.RequestCanceledError) = false, expected true", err)
+	}
+	if !errors.Is(err, context.DeadlineExceeded) {
+		t.Errorf("errors.Is(%v, context.DeadlineExceeded) = false, expected true", err)
+	}
+}
+
+func TestCredentialContext(t *testing.T) {
+	tests := []struct {
+		name           string
+		f              factory
+		expectDeadline time.Duration // 0: no deadline
+	}{
+		{"default", NewConnFactory(nil, nil), DefaultCredentialTimeout},
+		{"zero value", factory{}, DefaultCredentialTimeout},
+		{"option", NewConnFactory(nil, nil, WithCredentialTimeout(5*time.Second)), 5 * time.Second},
+		{"disabled", NewConnFactory(nil, nil, WithCredentialTimeout(0)), 0},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t0 := time.Now()
+			ctx, cancel := tt.f.credentialContext()
+			defer cancel()
+			deadline, ok := ctx.Deadline()
+			if tt.expectDeadline == 0 {
+				if ok {
+					t.Errorf("got deadline in %s, expected no deadline", deadline.Sub(t0))
+				}
+				return
+			}
+			if !ok {
+				t.Fatalf("got no deadline, expected deadline in %s", tt.expectDeadline)
+			}
+			if d := deadline.Sub(t0); d < tt.expectDeadline-time.Second || d > tt.expectDeadline+time.Second {
+				t.Errorf("got deadline in %s, expected about %s", d, tt.expectDeadline)
+			}
+		})
+	}
+}
+
+func TestWithCredentialTimeoutNegativePanics(t *testing.T) {
+	defer func() {
+		if r := recover(); r == nil {
+			t.Error("WithCredentialTimeout(-1s) didn't panic")
+		}
+	}()
+	WithCredentialTimeout(-time.Second)
+}