diff --git a/api/observability/v1/vocabulary.json b/api/observability/v1/vocabulary.json index 611035926..5228c6e0b 100644 --- a/api/observability/v1/vocabulary.json +++ b/api/observability/v1/vocabulary.json @@ -209,6 +209,7 @@ "server.certificate_rejected", "server.cluster_secret_invalid", "server.command_invalid", + "server.component_failed", "server.config_file_invalid", "server.config_invalid", "server.connection_failed", diff --git a/internal/daemon/runtime.go b/internal/daemon/runtime.go index f62b71eb5..48e7e74a7 100644 --- a/internal/daemon/runtime.go +++ b/internal/daemon/runtime.go @@ -225,6 +225,7 @@ func serveRuntime(ctx context.Context, cfg daemonconfig.Config, options runtimeO case <-ctx.Done(): return nil case err := <-d.componentDone: + reportComponentFailure(ctx, err) return err } } diff --git a/internal/daemon/runtime_test.go b/internal/daemon/runtime_test.go index 60f80063e..7653405ba 100644 --- a/internal/daemon/runtime_test.go +++ b/internal/daemon/runtime_test.go @@ -1,14 +1,19 @@ package daemon import ( + "bytes" "context" "errors" + "log" "net" "net/http" + "strings" "testing" "time" "github.com/aws/aws-sdk-go-v2/aws" + "github.com/jackc/pgx/v5/pgconn" + "github.com/tnldotdev/tnl/internal/failure" ) func TestRoute53CredentialReadiness(t *testing.T) { @@ -61,6 +66,63 @@ func TestDaemonSupervisesAndJoinsComponents(t *testing.T) { } } +func TestDaemonComponentFailureKeepsClassificationCauseAndSafeReport(t *testing.T) { + for _, specific := range []bool{false, true} { + t.Run(map[bool]string{false: "unclassified", true: "specific"}[specific], func(t *testing.T) { + cause := &pgconn.PgError{Code: "40001", Message: "private-test-secret"} + var original error = cause + want, retry := failure.ServerComponentFailed, failure.NoRetry + if specific { + want, retry = failure.DatabaseUnavailable, failure.RetryLater + original = failure.Wrap("renew database authority", want, cause) + } + var output bytes.Buffer + previous := log.Writer() + log.SetOutput(&output) + defer log.SetOutput(previous) + d := &daemon{ctx: t.Context(), componentDone: make(chan error, 1)} + calls := 0 + d.start("run ingress control", func() error { calls++; return original }) + err := <-d.componentDone + d.components.Wait() + reportComponentFailure(t.Context(), err) + reason, definition, ok := failure.Describe(err) + if !ok || reason != want || definition.Class != failure.Unavailable || definition.Retry != retry { + t.Fatal("component exit lost its classification or retry policy", reason, definition) + } + typed, ok := failure.Of(err) + if !ok || typed.Operation() != "run ingress control" || errors.Unwrap(err) != original || !errors.Is(err, cause) || calls != 1 { + t.Fatal("component exit lost its cause, operation, or no-restart boundary", err, calls) + } + for _, safe := range []string{output.String(), failure.SafeMessage(err, failure.ServerStartupFailed)} { + if strings.Contains(safe, "private-test-secret") { + t.Fatal("component reporting exposed a provider cause") + } + } + if !strings.Contains(output.String(), "run ingress control reason="+string(want)) || !strings.Contains(output.String(), "sqlstate=40001") { + t.Fatal("component report omitted its operation, reason, or approved SQLSTATE", output.String()) + } + }) + } +} + +func TestDaemonComponentCancellationDoesNotReportFailure(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + var output bytes.Buffer + previous := log.Writer() + log.SetOutput(&output) + defer log.SetOutput(previous) + d := &daemon{ctx: ctx, componentDone: make(chan error, 1)} + d.start("stop ingress control", func() error { return context.Canceled }) + err := <-d.componentDone + d.components.Wait() + reportComponentFailure(ctx, err) + if !errors.Is(err, context.Canceled) || output.Len() != 0 { + t.Fatal("canceled component lost its cause or emitted an operational failure", err, output.String()) + } +} + func TestConnectionListenerTransfersOwnership(t *testing.T) { listener := newConnectionListener(&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: 443}) server, client := net.Pipe() diff --git a/internal/daemon/supervision.go b/internal/daemon/supervision.go index bf442b4e8..6d4958d8c 100644 --- a/internal/daemon/supervision.go +++ b/internal/daemon/supervision.go @@ -1,7 +1,10 @@ package daemon import ( + "context" "fmt" + + "github.com/tnldotdev/tnl/internal/failure" ) func runtimeCapacity(name string, value int64) (int, error) { @@ -19,7 +22,11 @@ func (d *daemon) start(name string, run func() error) { d.components.Go(func() { err := run() if err != nil { - err = fmt.Errorf("%s: %w", name, err) + reason, known := failure.ReasonOf(err) + if !known { + reason = failure.ServerComponentFailed + } + err = failure.Wrap(failure.Operation(name), reason, err) } select { case d.componentDone <- err: @@ -28,6 +35,19 @@ func (d *daemon) start(name string, run func() error) { }) } +// report the selected exit before shutdown loses the component name. caller +// cancellation stays normal control flow, not an operational fault. +func reportComponentFailure(ctx context.Context, err error) { + if err == nil || ctx.Err() != nil { + return + } + operation := failure.Operation("run configured role") + if typed, ok := failure.Of(err); ok { + operation = typed.Operation() + } + logOperationalError(ctx, operation, failure.ServerComponentFailed, err) +} + // background tracks work that can complete without ending the process, such // as draining a relay after control updates its lease. func (d *daemon) background(run func()) { diff --git a/internal/failure/reasons.go b/internal/failure/reasons.go index 03d3b800c..8f5f5018f 100644 --- a/internal/failure/reasons.go +++ b/internal/failure/reasons.go @@ -172,6 +172,7 @@ const ( DirectDatabaseURLMissing Reason = "server.direct_database_url_missing" ServerMigrationFailed Reason = "server.migration_failed" ServerStartupFailed Reason = "server.startup_failed" + ServerComponentFailed Reason = "server.component_failed" ServerOutputUnavailable Reason = "server.output_unavailable" ServerWorkerFailed Reason = "server.worker_failed" ServerIngressFailed Reason = "server.ingress_failed" @@ -779,6 +780,10 @@ var definitions = map[Reason]Definition{ Class: Unavailable, Message: "tnld could not start the configured role", Action: "check role listeners, control reachability, and required credentials", Retry: RetryAfterChange, }, + ServerComponentFailed: { + Class: Unavailable, Message: "a required role component stopped", + Action: "check the named component and its diagnostics before restarting the process", Retry: NoRetry, + }, ServerOutputUnavailable: { Class: Unavailable, Message: "tnld could not write the command output", Action: "check the output destination and retry", Retry: RetryAfterChange,