From ba3ab8cb33ca66ce3a03fe268609e6bed33d1e1a Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Fri, 9 Oct 2026 12:15:08 -0700 Subject: [PATCH 01/44] feat: show exact spend threshold units and alert semantics --- cmd/openai/spend_thresholds_test.go | 482 ++++++++++++++++++ docs/spend-thresholds.md | 71 +++ pkg/transformers/project_spend_threshold.go | 164 ++++++ .../project_spend_threshold_test.go | 155 ++++++ pkg/transformers/select_transformer.go | 3 + scripts/demos/record-spend-thresholds.sh | 67 +++ scripts/demos/spend-thresholds/README.md | 62 +++ scripts/demos/spend-thresholds/scene.sh | 23 + scripts/demos/spend-thresholds/server.py | 93 ++++ scripts/demos/spend-thresholds/validate.py | 64 +++ 10 files changed, 1184 insertions(+) create mode 100644 cmd/openai/spend_thresholds_test.go create mode 100644 docs/spend-thresholds.md create mode 100644 pkg/transformers/project_spend_threshold.go create mode 100644 pkg/transformers/project_spend_threshold_test.go create mode 100755 scripts/demos/record-spend-thresholds.sh create mode 100644 scripts/demos/spend-thresholds/README.md create mode 100755 scripts/demos/spend-thresholds/scene.sh create mode 100755 scripts/demos/spend-thresholds/server.py create mode 100755 scripts/demos/spend-thresholds/validate.py diff --git a/cmd/openai/spend_thresholds_test.go b/cmd/openai/spend_thresholds_test.go new file mode 100644 index 00000000..36c03f78 --- /dev/null +++ b/cmd/openai/spend_thresholds_test.go @@ -0,0 +1,482 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" +) + +func runSpendThresholdCommand(t *testing.T, server *httptest.Server, args ...string) mainDispatchResult { + t.Helper() + return runMainDispatchWithEnv(t, "bash", []string{ + "OPENAI_BASE_URL=" + server.URL, + "OPENAI_API_KEY=sk-fake-spend-test", + "OPENAI_ADMIN_KEY=sk-fake-spend-admin", + "FORCE_COLOR=0", "NO_COLOR=1", "COLUMNS=28", + }, append([]string{"openai"}, args...)...) +} + +func spendThresholdArgs(project bool, resource, operation string) []string { + args := []string{"admin", "organization"} + if project { + args = append(args, "projects") + } + args = append(args, resource, operation) + if project { + args = append(args, "--project-id", "proj_synthetic") + } + if resource == "spend-alerts" && operation != "create" && operation != "list" { + args = append(args, "--alert-id", "alert_synthetic") + } + return args +} + +func spendThresholdPath(project bool, resource string) string { + path := "/organization" + if project { + path += "/projects/proj_synthetic" + } + return path + "/" + strings.ReplaceAll(resource, "-", "_") +} + +func spendThresholdBody(project bool, resource, fields string) string { + scope := "organization" + if project { + scope = "project" + } + object := scope + "." + strings.TrimSuffix(strings.ReplaceAll(resource, "-", "_"), "s") + return `{"object":"` + object + `",` + strings.TrimPrefix(fields, "{") +} + +func assertSpendThresholdJSON(t *testing.T, output string, records ...string) { + t.Helper() + decoder := json.NewDecoder(strings.NewReader(output)) + decoder.UseNumber() + for _, record := range records { + original := json.NewDecoder(strings.NewReader(record)) + original.UseNumber() + var actual, want any + if err := original.Decode(&want); err != nil { + t.Fatal(err) + } + if err := decoder.Decode(&actual); err != nil || !reflect.DeepEqual(actual, want) { + t.Fatalf("spend response changed: actual=%v want=%v error=%v; output=%q", actual, want, err, output) + } + } + if err := decoder.Decode(new(any)); err != io.EOF { + t.Fatalf("unexpected trailing spend response: %v; output=%q", err, output) + } +} + +func TestMainSpendThresholdRoutesAndRequests(t *testing.T) { + for _, project := range []bool{false, true} { + for _, operation := range []struct{ resource, method string }{ + {"spend-limit", "retrieve"}, {"spend-limit", "update"}, + {"spend-alerts", "create"}, {"spend-alerts", "retrieve"}, {"spend-alerts", "update"}, + } { + for _, format := range []string{"", "text"} { + t.Run(fmt.Sprintf("project=%t/%s/%s/%s", project, operation.resource, operation.method, format), func(t *testing.T) { + path := spendThresholdPath(project, operation.resource) + if operation.resource == "spend-alerts" && operation.method != "create" { + path += "/alert_synthetic" + } + mutation := operation.method != "retrieve" + bodies := make(chan string, 8) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method := http.MethodGet + if mutation { + method = http.MethodPost + } + if r.Method != method || r.URL.Path != path { + t.Errorf("unexpected request: %s %s, want %s %s", r.Method, r.URL.Path, method, path) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + bodies <- string(body) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(project, operation.resource, `{"id":"alert_synthetic","currency":"USD","interval":"month","threshold_amount":10000,"future":{"note":"preserved"}}`)) + })) + defer server.Close() + args := spendThresholdArgs(project, operation.resource, operation.method) + if mutation { + args = append(args, "--currency", "USD", "--interval", "month", "--threshold-amount", "9007199254740993") + if operation.resource == "spend-alerts" { + args = append(args, "--notification-channel", `{"type":"email","recipients":["synthetic@example.invalid"],"subject_prefix":null}`) + } + } + if format != "" { + args = append([]string{"--format", format}, args...) + } + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("spend command failed: %+v", got) + } + if len(bodies) != 1 { + t.Fatalf("received %d requests, want one", len(bodies)) + } + body := <-bodies + if mutation { + want := `{"currency":"USD","interval":"month","threshold_amount":9007199254740993}` + if operation.resource == "spend-alerts" { + want = `{"currency":"USD","interval":"month","threshold_amount":9007199254740993,"notification_channel":{"type":"email","recipients":["synthetic@example.invalid"],"subject_prefix":null}}` + } + assertSpendThresholdJSON(t, body, want) + } else if body != "" { + t.Errorf("retrieve sent a request body: %q", body) + } + for _, want := range []string{"Spend threshold: USD 100.00 per month", "Currency: USD", "Interval: month", "Note: preserved"} { + if !strings.Contains(got.stdout, want) { + t.Errorf("missing %q in %q", want, got.stdout) + } + } + want := "Enforcement: not reported in this response" + if operation.resource == "spend-alerts" { + want = "Alert behavior: Alerts notify; they are not spending caps." + } + if !strings.Contains(got.stdout, want) || strings.Contains(got.stdout, "Threshold amount:") { + t.Errorf("unclear spend threshold presentation: %q; want %q", got.stdout, want) + } + }) + } + } + } +} + +func TestMainSpendThresholdPrecisionAndUnknownFields(t *testing.T) { + for _, tc := range []struct { + name, body string + want, absent []string + }{ + {"zero", `{"threshold_amount":0,"currency":"USD","interval":"month"}`, []string{"Spend threshold: USD 0.00 per month"}, nil}, + {"negative", `{"threshold_amount":-1,"currency":"USD","interval":"month"}`, []string{"Spend threshold: USD -0.01 per month"}, nil}, + {"large", `{"threshold_amount":9223372036854775807,"currency":"USD","interval":"month"}`, []string{"Spend threshold: USD 92233720368547758.07 per month"}, nil}, + {"decimal", `{"threshold_amount":100.005,"currency":"USD","interval":"month"}`, []string{"100.005", "cents"}, []string{"USD 1.00"}}, + {"exponent", `{"threshold_amount":1e4,"currency":"USD","interval":"month"}`, []string{"1e4", "cents"}, []string{"USD 100.00"}}, + {"unknown units", `{"threshold_amount":123,"currency":"ZZZ","interval":"quarter"}`, []string{"123 cents", "ZZZ", "(interval: quarter)"}, []string{"USD", "per month"}}, + {"missing units", `{"threshold_amount":123}`, []string{"123 cents"}, []string{"USD", "per month"}}, + {"missing interval", `{"threshold_amount":123,"currency":"USD"}`, []string{"USD 1.23"}, []string{"per month"}}, + {"null enforcement", `{"threshold_amount":123,"currency":"USD","interval":"month","enforcement":null}`, []string{"Enforcement: not reported in this response"}, nil}, + {"reported enforcement", `{"threshold_amount":123,"currency":"USD","interval":"month","enforcement":{"mode":"reported_mode","future":9007199254740993}}`, []string{"Mode: reported_mode", "Future: 9007199254740993"}, []string{"not reported in this response"}}, + {"null threshold", `{"threshold_amount":null,"future":"preserved"}`, []string{"Threshold amount: (null)", "Future: preserved"}, []string{"Spend threshold:", "USD", "per month"}}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", tc.body)) + })) + defer server.Close() + got := runSpendThresholdCommand(t, server, spendThresholdArgs(false, "spend-limit", "retrieve")...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("threshold response failed: %+v", got) + } + for _, want := range tc.want { + if !strings.Contains(got.stdout, want) { + t.Errorf("missing %q in %q", want, got.stdout) + } + } + for _, absent := range tc.absent { + if strings.Contains(got.stdout, absent) { + t.Errorf("invented or changed %q in %q", absent, got.stdout) + } + } + }) + } +} + +func TestMainSpendThresholdShortAlias(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/organization/spend_limit" { + t.Errorf("unexpected alias request: %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + })) + defer server.Close() + for _, args := range [][]string{ + {"admin", "spend-limit", "retrieve"}, + {"admin:organization:spend-limit", "retrieve"}, + {"--format", "TEXT", "admin", "spend-limit", "retrieve"}, + } { + t.Run(strings.Join(args, "/"), func(t *testing.T) { + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "Spend threshold: USD 100.00 per month") || !strings.Contains(got.stdout, "Enforcement: not reported in this response") { + t.Fatalf("spend-limit alias failed: %+v", got) + } + }) + } +} + +func TestMainSpendThresholdReportedEnforcementStatus(t *testing.T) { + for _, status := range []string{"inactive", "enforcing", "future_status"} { + t.Run(status, func(t *testing.T) { + body := spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"`+status+`","future":9007199254740993}}`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + args := spendThresholdArgs(false, "spend-limit", "retrieve") + got := runSpendThresholdCommand(t, server, args...) + want := "Object: organization.spend_limit\nSpend threshold: USD 100.00 per month\nCurrency: USD\nInterval: month\nEnforcement:\n Status: " + status + "\n Future: 9007199254740993\n" + if got.code != 0 || got.stderr != "" || got.stdout != want { + t.Fatalf("reported enforcement status changed: %+v; want %q", got, want) + } + got = runSpendThresholdCommand(t, server, append([]string{"--transform", "enforcement"}, args...)...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("enforcement extraction failed: %+v", got) + } + assertSpendThresholdJSON(t, got.stdout, `{"status":"`+status+`","future":9007199254740993}`) + }) + } +} + +func TestMainSpendThresholdMachineFormatsAndExtraction(t *testing.T) { + for _, resource := range []string{"spend-limit", "spend-alerts"} { + body := spendThresholdBody(true, resource, `{"id":"alert_synthetic","currency":"USD","interval":"month","threshold_amount":9007199254740993,"future":{"exact":123456789012345678901234567890,"decimal":0.100000000000000001},"enforcement":null}`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + for _, format := range []string{"json", "JSON", "jsonl", "raw", "yaml", "pretty", "explore"} { + t.Run(resource+"/"+format, func(t *testing.T) { + args := append([]string{"--format", format}, spendThresholdArgs(true, resource, "retrieve")...) + got := runSpendThresholdCommand(t, server, args...) + wantStderr := "" + if format == "explore" { + wantStderr = "Warning: Output format 'explore' not supported for non-terminal output; falling back to 'json'\n" + } + if got.code != 0 || got.stderr != wantStderr { + t.Fatalf("explicit format failed: %+v", got) + } + if format == "yaml" || format == "pretty" { + for _, value := range []string{"threshold_amount: 9007199254740993", "exact: 123456789012345678901234567890", "decimal: 0.100000000000000001", "enforcement: null"} { + if !strings.Contains(got.stdout, value) { + t.Errorf("%s lost %q: %q", format, value, got.stdout) + } + } + } else { + assertSpendThresholdJSON(t, got.stdout, body) + } + if format == "raw" && got.stdout != body+"\n" { + t.Errorf("raw response bytes changed: %q", got.stdout) + } + if strings.Contains(got.stdout, "spend_threshold") || strings.Contains(got.stdout, "alert_behavior") { + t.Errorf("projection entered explicit output: %q", got.stdout) + } + }) + } + for _, tc := range []struct { + name string + flags []string + want string + }{ + {"raw-output", []string{"--raw-output"}, body}, + {"number extraction", []string{"--transform", "threshold_amount"}, "9007199254740993"}, + {"text extraction", []string{"--format", "text", "--transform", "threshold_amount"}, "9007199254740993"}, + {"raw extraction", []string{"--transform", "future.decimal", "--raw-output"}, "0.100000000000000001"}, + } { + t.Run(resource+"/"+tc.name, func(t *testing.T) { + got := runSpendThresholdCommand(t, server, append(tc.flags, spendThresholdArgs(true, resource, "retrieve")...)...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("explicit extraction failed: %+v", got) + } + assertSpendThresholdJSON(t, got.stdout, tc.want) + }) + } + } +} + +func TestMainSpendAlertsPagination(t *testing.T) { + for _, project := range []bool{false, true} { + first := spendThresholdBody(project, "spend-alerts", `{"id":"alert_first","currency":"USD","interval":"month","threshold_amount":10000,"future":9007199254740993}`) + second := spendThresholdBody(project, "spend-alerts", `{"id":"alert_second","currency":"USD","interval":"month","threshold_amount":0}`) + page := `{"object":"list","data":[` + first + `],"has_more":true,"first_id":"alert_first","last_id":"alert_first"}` + for _, tc := range []struct { + name string + flags []string + max string + count, calls int + failure, raw bool + }{ + {name: "default", count: 2, calls: 2}, + {name: "unlimited", max: "-1", count: 2, calls: 2}, + {name: "zero", max: "0", calls: 1}, + {name: "one", max: "1", count: 1, calls: 1}, + {name: "raw page", flags: []string{"--format", "raw"}, count: 1, calls: 1, raw: true}, + {name: "jsonl", flags: []string{"--format", "jsonl"}, count: 2, calls: 2}, + {name: "partial failure", count: 1, calls: 2, failure: true}, + {name: "structured partial failure", flags: []string{"--format", "jsonl"}, count: 1, calls: 2, failure: true}, + } { + t.Run(fmt.Sprintf("project=%t/%s", project, tc.name), func(t *testing.T) { + var mu sync.Mutex + var cursors []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cursor := r.URL.Query().Get("after") + mu.Lock() + cursors = append(cursors, cursor) + mu.Unlock() + if r.Method != http.MethodGet || r.URL.Path != spendThresholdPath(project, "spend-alerts") || r.URL.Query().Get("limit") != "1" { + t.Errorf("unexpected list request: %s %s", r.Method, r.URL) + } + w.Header().Set("Content-Type", "application/json") + switch cursor { + case "": + io.WriteString(w, page) + case "alert_first": + if tc.failure { + w.Header().Set("x-should-retry", "false") + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, `{"error":{"type":"invalid_request_error","message":"Synthetic page failure"}}`) + } else { + io.WriteString(w, `{"object":"list","data":[`+second+`],"has_more":false,"first_id":"alert_second","last_id":"alert_second"}`) + } + default: + http.Error(w, "unexpected cursor", http.StatusBadRequest) + } + })) + defer server.Close() + args := append(tc.flags, spendThresholdArgs(project, "spend-alerts", "list")...) + args = append(args, "--limit", "1") + if tc.max != "" { + args = append(args, "--max-items", tc.max) + } + got := runSpendThresholdCommand(t, server, args...) + mu.Lock() + actualCursors := slices.Clone(cursors) + mu.Unlock() + if want := []string{"", "alert_first"}[:tc.calls]; !slices.Equal(actualCursors, want) { + t.Fatalf("cursors=%q, want %q", actualCursors, want) + } + if tc.failure { + if got.code != 1 { + t.Fatalf("partial API failure disappeared: %+v", got) + } + if slices.Contains(tc.flags, "jsonl") { + payload := decodeMainStructuredError(t, "jsonl", got.stderr) + if payload["message"] != "Synthetic page failure" { + t.Errorf("structured page failure changed: %#v", payload) + } + } else if !strings.Contains(got.stderr, "HTTP 400: Bad Request.") || strings.Contains(got.stderr, "Synthetic page failure") { + t.Errorf("readable page failure framing changed: %q", got.stderr) + } + } else if got.code != 0 || got.stderr != "" { + t.Fatalf("spend alerts list failed: %+v", got) + } + if tc.raw { + if got.stdout != page+"\n" { + t.Fatalf("raw page changed: %q", got.stdout) + } + } else if slices.Contains(tc.flags, "jsonl") { + assertSpendThresholdJSON(t, got.stdout, []string{first, second}[:tc.count]...) + } else { + if strings.Count(got.stdout, "Spend threshold:") != tc.count || strings.Count(got.stdout, "Alerts notify; they are not spending caps.") != tc.count { + t.Fatalf("list lost threshold or alert clarification: %+v", got) + } + if tc.count > 0 && (!strings.Contains(got.stdout, "ID: alert_first") || !strings.Contains(got.stdout, "9007199254740993")) { + t.Errorf("partial result lost original metadata: %q", got.stdout) + } + if tc.count == 0 && got.stdout != "" { + t.Errorf("zero item list emitted output: %q", got.stdout) + } + } + }) + } + } +} + +func TestMainSpendThresholdMalformedAPIErrors(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"empty", ""}, + {"malformed", `{"error":{"message":"synthetic private spend detail"`}, + {"empty envelope", `{}`}, + {"empty error", `{"error":{}}`}, + } { + for _, format := range []string{"text", "json"} { + t.Run(tc.name+"/"+format, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("x-should-retry", "false") + w.WriteHeader(http.StatusBadGateway) + io.WriteString(w, tc.body) + })) + defer server.Close() + // Preserve the shared error contract, including an empty error object. + reference := runSpendThresholdCommand(t, server, "--format", format, "models", "retrieve", "model_synthetic") + args := append([]string{"--format", format}, spendThresholdArgs(false, "spend-limit", "retrieve")...) + got := runSpendThresholdCommand(t, server, args...) + if got != reference || got.code != 1 || got.stdout != "" || requests.Load() != 2 { + t.Fatalf("spend error framing changed: got=%+v reference=%+v requests=%d", got, reference, requests.Load()) + } + if format == "json" { + payload := decodeMainErrorObject(t, "json", got.stderr) + if len(payload) > 0 && (payload["status_code"] != json.Number("502") || payload["message"] != "HTTP 502 Bad Gateway: the server returned no usable JSON error details.") { + t.Errorf("fallback status or message changed: %#v", payload) + } + } else if !strings.Contains(got.stderr, "HTTP 502: Bad Gateway.") || !strings.Contains(got.stderr, "API error details: --format-error json.") { + t.Errorf("readable error lacks status or guidance: %q", got.stderr) + } + for _, absent := range []string{"synthetic private spend detail", "sk-fake-spend", "Spend threshold:", "spend_threshold"} { + if strings.Contains(got.stderr, absent) { + t.Errorf("error included %q: %q", absent, got.stderr) + } + } + }) + } + } +} + +func TestMainSpendThresholdControlsAndDeleteIdentity(t *testing.T) { + const body = `{"object":"organization.spend_limit","threshold_amount":123,"currency":"ZZZ\u001b[31m","interval":"quarter\r\u001b[2J","future":"keep\u0007value"}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + got := runSpendThresholdCommand(t, server, spendThresholdArgs(false, "spend-limit", "retrieve")...) + if got.code != 0 || got.stderr != "" || strings.ContainsAny(got.stdout, "\x1b\r\a") || !strings.Contains(got.stdout, "123 cents") || !strings.Contains(got.stdout, "keep") { + t.Fatalf("readable spend threshold emitted controls or lost content: %+v", got) + } + for _, resource := range []string{"spend-limit", "spend-alerts"} { + got := runSpendThresholdCommand(t, server, spendThresholdArgs(false, resource, "delete")...) + if got.code != 0 || got.stderr != "" || strings.Contains(got.stdout, "Spend threshold:") || strings.Contains(got.stdout, "Alert behavior:") || strings.Contains(got.stdout, "Enforcement:") || !strings.Contains(got.stdout, "Threshold amount: 123") { + t.Fatalf("delete response incorrectly projected: %+v", got) + } + } +} + +func TestMainSpendThresholdLargeUnknownField(t *testing.T) { + // Keep a complete unknown field larger than common line and buffer limits. + value := strings.Repeat("synthetic", 1<<20) + body := spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month","future":"`+value+`"}`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + args := spendThresholdArgs(false, "spend-limit", "retrieve") + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "Spend threshold: USD 100.00 per month") || !strings.Contains(got.stdout, "Future: "+value+"\n") { + t.Fatalf("large readable response lost data: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } + got = runSpendThresholdCommand(t, server, append([]string{"--format", "raw"}, args...)...) + if got.code != 0 || got.stderr != "" || got.stdout != body+"\n" { + t.Fatalf("large raw response changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } + got = runSpendThresholdCommand(t, server, append([]string{"--transform", "future", "--raw-output"}, args...)...) + if got.code != 0 || got.stderr != "" || got.stdout != value+"\n" { + t.Fatalf("large extracted field changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } +} diff --git a/docs/spend-thresholds.md b/docs/spend-thresholds.md new file mode 100644 index 00000000..69696ad3 --- /dev/null +++ b/docs/spend-thresholds.md @@ -0,0 +1,71 @@ +# Spend thresholds + +Spend-limit and spend-alert amounts use cents. Readable output shows integer USD amounts as dollars and cents. + +```sh +# Read the organization threshold. +openai admin organization spend-limit retrieve + +# Read a project threshold. +openai admin organization projects spend-limit retrieve --project-id proj_example + +# List notification thresholds. +openai admin organization spend-alerts list +``` + +A response containing `threshold_amount: 10000`, `currency: "USD"`, and `interval: "month"` displays: + +```text +Spend threshold: USD 100.00 per month +``` + +The CLI preserves reported enforcement fields. When enforcement is absent or null, it displays: + +```text +Enforcement: not reported in this response +``` + +Spend alerts notify recipients. They do not enforce spending caps or stop API requests. +Each readable alert includes this distinction, including alerts returned across multiple pages. +Notification channels, recipients, identifiers, and unfamiliar fields remain visible. + +## Preserved data and inputs + +The projection applies to default readable output and `--format text`. +Organization and project responses share these rules. +Successful create/update responses use the same presentation as retrieve/list responses. +Delete confirmations remain unchanged. + +```sh +# Preserve the original numeric cents and response fields. +openai admin organization spend-limit retrieve --format json + +# Extract the original amount. +openai admin organization spend-limit retrieve --transform threshold_amount --raw-output + +# Preserve existing pagination controls. +openai admin organization spend-alerts list --max-items -1 +``` + +Explicit JSON, JSONL, YAML, raw, pretty, and explore formats retain their existing data behavior. +Extraction and `--raw-output` bypass the projection. +The CLI does not change errors, exit status, or pagination. + +Unknown currencies retain the original amount in cents without assuming a currency conversion. +Unknown intervals remain literal values. Missing currency or interval receives a `not reported` label. +The CLI preserves exact large integers without floating-point conversion. +Unexpected decimal or exponent values retain their original numeric spelling in cents. + +Existing update/create flags still accept cents. This change does not convert inputs or change validation. +Limit inputs require at least one cent in the API schema; alert inputs allow zero. +Response edge-case handling does not establish new accepted mutation inputs. + +## Scope + +The bundled API contract establishes USD, monthly intervals, and reported limit enforcement states. +The CLI does not infer an enforcement state from the threshold amount. +This presentation change adds no billing action, policy API, usage-tier discovery, or Costs API conversion. +Costs API amounts follow a separate contract. + +See the [organization spend-limit reference](https://developers.openai.com/api/reference/resources/admin/subresources/organization/subresources/spend_limit/methods/retrieve) +and [spend-alert guidance](https://developers.openai.com/api/docs/guides/terraform/rate-limits-and-spend). diff --git a/pkg/transformers/project_spend_threshold.go b/pkg/transformers/project_spend_threshold.go new file mode 100644 index 00000000..e41f33d4 --- /dev/null +++ b/pkg/transformers/project_spend_threshold.go @@ -0,0 +1,164 @@ +package transformers + +import ( + "context" + "encoding/json" + "strings" + + "github.com/tidwall/gjson" +) + +// Spend thresholds use cents. Costs API amounts have a separate contract. +func selectSpendThresholdTransformer(route Route) Transformer { + var object string + alert := false + switch route { + case Route{"(resource) admin.organization.spend_limit > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.spend_limit > (method) update", OutputResponse}: + object = "organization.spend_limit" + case Route{"(resource) admin.organization.projects.spend_limit > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_limit > (method) update", OutputResponse}: + object = "project.spend_limit" + case Route{"(resource) admin.organization.spend_alerts > (method) create", OutputResponse}, + Route{"(resource) admin.organization.spend_alerts > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.spend_alerts > (method) update", OutputResponse}, + Route{"(resource) admin.organization.spend_alerts > (method) list", OutputPageItem}: + object, alert = "organization.spend_alert", true + case Route{"(resource) admin.organization.projects.spend_alerts > (method) create", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_alerts > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_alerts > (method) update", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_alerts > (method) list", OutputPageItem}: + object, alert = "project.spend_alert", true + default: + return nil + } + return func(ctx context.Context, value gjson.Result) (gjson.Result, error) { + return projectSpendThreshold(ctx, value, object, alert) + } +} + +func projectSpendThreshold(ctx context.Context, value gjson.Result, object string, alert bool) (gjson.Result, error) { + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + if !value.IsObject() || value.Get("object").String() != object { + return value, nil + } + amount := value.Get("threshold_amount") + summary := "" + var summaryJSON []byte + // Do not overwrite an unfamiliar response field with a derived field. + if amount.Type == gjson.Number && !value.Get("spend_threshold").Exists() { + var err error + summary, err = spendThresholdText(ctx, amount.Raw, value.Get("currency"), value.Get("interval")) + if err != nil { + return gjson.Result{}, err + } + summaryJSON, err = json.Marshal(summary) + if err != nil { + return gjson.Result{}, err + } + } + var out strings.Builder + out.WriteByte('{') + first := true + write := func(key, raw string) { + if !first { + out.WriteByte(',') + } + first = false + out.WriteString(key) + out.WriteByte(':') + out.WriteString(raw) + } + value.ForEach(func(key, field gjson.Result) bool { + if ctx.Err() != nil { + return false + } + switch { + case key.Str == "threshold_amount" && summary != "": + write(`"spend_threshold"`, string(summaryJSON)) + case key.Str == "enforcement" && !alert && field.Type == gjson.Null: + write(key.Raw, `"not reported in this response"`) + default: + // Preserve unknown fields and number spellings without decoding floats. + write(key.Raw, field.Raw) + } + return true + }) + if alert { + if !value.Get("alert_behavior").Exists() { + write(`"alert_behavior"`, `"Alerts notify; they are not spending caps."`) + } + } else if !value.Get("enforcement").Exists() { + write(`"enforcement"`, `"not reported in this response"`) + } + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + out.WriteByte('}') + return gjson.Parse(out.String()), nil +} + +func spendThresholdText(ctx context.Context, amount string, currency, interval gjson.Result) (string, error) { + text := amount + " cents" + if currency.Type == gjson.String && currency.Str == "USD" { + decimal, err := spendCentsDecimal(ctx, amount) + if err != nil { + return "", err + } + if decimal != "" { + text = "USD " + decimal + } else { + text += " (currency: USD)" + } + } else { + text += " (currency: " + spendReportedUnit(currency) + ")" + } + if interval.Type == gjson.String && interval.Str == "month" { + text += " per month" + } else { + text += " (interval: " + spendReportedUnit(interval) + ")" + } + return text, ctx.Err() +} + +func spendReportedUnit(value gjson.Result) string { + if value.Type == gjson.Null || value.Type == gjson.String && value.Str == "" { + return "not reported" + } + if value.Type == gjson.String { + return value.Str + } + return "see response field" +} + +// Insert a decimal point in integer cents without float conversion or bounds. +// Unexpected fractions and exponents retain their original spelling in cents. +func spendCentsDecimal(ctx context.Context, amount string) (string, error) { + sign := "" + if strings.HasPrefix(amount, "-") { + sign, amount = "-", amount[1:] + } + if amount == "" { + return "", nil + } + for i := 0; i < len(amount); i++ { + if i%32768 == 0 { + if err := ctx.Err(); err != nil { + return "", err + } + } + if amount[i] < '0' || amount[i] > '9' { + return "", nil + } + } + switch len(amount) { + case 1: + return sign + "0.0" + amount, nil + case 2: + return sign + "0." + amount, nil + default: + return sign + amount[:len(amount)-2] + "." + amount[len(amount)-2:], nil + } +} diff --git a/pkg/transformers/project_spend_threshold_test.go b/pkg/transformers/project_spend_threshold_test.go new file mode 100644 index 00000000..6aa1a4ee --- /dev/null +++ b/pkg/transformers/project_spend_threshold_test.go @@ -0,0 +1,155 @@ +package transformers + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/tidwall/gjson" +) + +func TestSpendThresholdExactUnits(t *testing.T) { + for _, tc := range []struct{ amount, currency, interval, want string }{ + {"10000", `"USD"`, `"month"`, "USD 100.00 per month"}, + {"0", `"USD"`, `"month"`, "USD 0.00 per month"}, + {"1", `"USD"`, `"month"`, "USD 0.01 per month"}, + {"99", `"USD"`, `"month"`, "USD 0.99 per month"}, + {"-1", `"USD"`, `"month"`, "USD -0.01 per month"}, + {"-10000", `"USD"`, `"month"`, "USD -100.00 per month"}, + {"900719925474099312345678901234567890", `"USD"`, `"month"`, "USD 9007199254740993123456789012345678.90 per month"}, + {"10000", `"EUR"`, `"month"`, "10000 cents (currency: EUR) per month"}, + {"10000", `"USD"`, `"fortnight"`, "USD 100.00 (interval: fortnight)"}, + {"10000", `null`, `null`, "10000 cents (currency: not reported) (interval: not reported)"}, + {"10000", `""`, `""`, "10000 cents (currency: not reported) (interval: not reported)"}, + {"10000", `{}`, `false`, "10000 cents (currency: see response field) (interval: see response field)"}, + {"0.1234567890123456789", `"USD"`, `"month"`, "0.1234567890123456789 cents (currency: USD) per month"}, + {"1e100", `"USD"`, `"month"`, "1e100 cents (currency: USD) per month"}, + } { + t.Run(tc.amount+tc.currency+tc.interval, func(t *testing.T) { + got, err := spendThresholdText(t.Context(), tc.amount, gjson.Parse(tc.currency), gjson.Parse(tc.interval)) + if err != nil || got != tc.want { + t.Fatalf("got %q, %v; want %q", got, err, tc.want) + } + }) + } +} + +func TestSpendThresholdPreservesResponseFields(t *testing.T) { + const raw = `{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"future","extra":1.234567890123456789},"future":{"integer":9007199254740993,"decimal":0.000000000000000001}}` + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil { + t.Fatal(err) + } + if value.Get("spend_threshold").Str != "USD 100.00 per month" || value.Get("threshold_amount").Exists() { + t.Fatalf("threshold projection: %s", value.Raw) + } + for _, field := range []string{"object", "currency", "interval", "enforcement", "future"} { + if got, want := value.Get(field).Raw, gjson.Get(raw, field).Raw; got != want { + t.Errorf("%s changed: %s; want %s", field, got, want) + } + } +} + +func TestSpendThresholdSummaryUsesJSONEscapes(t *testing.T) { + const raw = `{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD\u001b[31m","interval":"\u0007\u000b\u0000"}` + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil || !gjson.Valid(value.Raw) { + t.Fatalf("control characters broke JSON projection: %q, %v", value.Raw, err) + } + want := "10000 cents (currency: USD\x1b[31m) (interval: \a\v\x00)" + if value.Get("spend_threshold").Str != want { + t.Fatalf("controls changed before renderer: %q", value.Get("spend_threshold").Str) + } +} + +func TestSpendThresholdMissingFieldsAndCollisions(t *testing.T) { + for _, raw := range []string{ + `{"object":"organization.spend_limit"}`, + `{"object":"organization.spend_limit","threshold_amount":null,"enforcement":null}`, + `{"object":"organization.spend_limit","threshold_amount":"10000"}`, + `{"object":"organization.spend_limit","threshold_amount":10000,"spend_threshold":"future field"}`, + } { + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil || !gjson.Valid(value.Raw) { + t.Fatalf("invalid projection: %s, %v", value.Raw, err) + } + if value.Get("enforcement").Str != "not reported in this response" { + t.Fatalf("missing enforcement: %s", value.Raw) + } + if value.Get("threshold_amount").Raw != gjson.Get(raw, "threshold_amount").Raw { + t.Fatalf("unexpected amount rewritten: %s", value.Raw) + } + if value.Get("spend_threshold").Raw != gjson.Get(raw, "spend_threshold").Raw { + t.Fatalf("collision overwritten: %s", value.Raw) + } + } + const alert = `{"object":"organization.spend_alert","threshold_amount":0,"alert_behavior":{"future":"kept"}}` + value, err := projectSpendThreshold(t.Context(), gjson.Parse(alert), "organization.spend_alert", true) + if err != nil || value.Get("alert_behavior").Raw != `{"future":"kept"}` || value.Get("enforcement").Exists() { + t.Fatalf("alert collision or invented enforcement: %s, %v", value.Raw, err) + } +} + +func TestSpendThresholdRouteBoundaries(t *testing.T) { + for _, resource := range []string{"admin.organization", "admin.organization.projects"} { + for _, tc := range []struct { + resource, method string + kind OutputKind + match bool + }{ + {"spend_limit", "retrieve", OutputResponse, true}, + {"spend_limit", "update", OutputResponse, true}, + {"spend_limit", "delete", OutputResponse, false}, + {"spend_alerts", "create", OutputResponse, true}, + {"spend_alerts", "retrieve", OutputResponse, true}, + {"spend_alerts", "update", OutputResponse, true}, + {"spend_alerts", "list", OutputPageItem, true}, + {"spend_alerts", "list", OutputResponse, false}, + {"spend_alerts", "list", OutputStreamEvent, false}, + {"spend_alerts", "delete", OutputResponse, false}, + {"spend_limit", "retrieve", OutputUnspecified, false}, + {"costs", "list", OutputPageItem, false}, + } { + route := Route{"(resource) " + resource + "." + tc.resource + " > (method) " + tc.method, tc.kind} + if got := selectSpendThresholdTransformer(route); (got != nil) != tc.match { + t.Errorf("route %+v selected=%t, want %t", route, got != nil, tc.match) + } + } + } + for _, raw := range []string{`null`, `[]`, `{"object":"project.spend_limit","threshold_amount":10000}`, `{"threshold_amount":10000}`} { + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil || value.Raw != raw { + t.Fatalf("unrecognized object changed: %s, %v", value.Raw, err) + } + } +} + +func TestSpendThresholdCancellationAndLargeValues(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := projectSpendThreshold(ctx, gjson.Parse(`{}`), "organization.spend_limit", false); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + if _, err := spendCentsDecimal(ctx, strings.Repeat("9", 100000)); !errors.Is(err, context.Canceled) { + t.Fatalf("amount cancellation lost: %v", err) + } + // This exceeds machine integers without adding a presentation rejection limit. + amount := strings.Repeat("9", 100000) + got, err := spendCentsDecimal(t.Context(), amount) + if err != nil || got != amount[:len(amount)-2]+".99" { + t.Fatalf("large exact amount changed: length=%d, error=%v", len(got), err) + } +} + +func TestSpendThresholdCancelsDuringScan(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + controlled := &cancelSummaryContext{Context: ctx, cancel: cancel, after: 4} + raw := `{"object":"organization.spend_limit","threshold_amount":` + strings.Repeat("9", 200000) + `,"currency":"USD","interval":"month"}` + transform := Select(Route{"(resource) admin.organization.spend_limit > (method) retrieve", OutputResponse}) + got, err := transform(controlled, gjson.Parse(raw)) + if !errors.Is(err, context.Canceled) || got.Raw != "" { + t.Fatalf("cancellation during scan returned partial output: bytes=%d error=%v", len(got.Raw), err) + } +} diff --git a/pkg/transformers/select_transformer.go b/pkg/transformers/select_transformer.go index c1c2ab2f..67e0be9b 100644 --- a/pkg/transformers/select_transformer.go +++ b/pkg/transformers/select_transformer.go @@ -36,5 +36,8 @@ func Identity(_ context.Context, value gjson.Result) (gjson.Result, error) { // Select is the SDK-owned hook for command-specific default transformations. // Only known API fields are summarized; unfamiliar routes preserve their values. func Select(route Route) Transformer { + if transform := selectSpendThresholdTransformer(route); transform != nil { + return transform + } return selectReadableTransformer(route) } diff --git a/scripts/demos/record-spend-thresholds.sh b/scripts/demos/record-spend-thresholds.sh new file mode 100755 index 00000000..f014e90b --- /dev/null +++ b/scripts/demos/record-spend-thresholds.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -euo pipefail + +if [ "$#" -ne 6 ] || [ "$1" != --run-authorized-pty-slot ]; then + echo 'usage: record-spend-thresholds.sh --run-authorized-pty-slot BEFORE_BINARY AFTER_BINARY BEFORE_SHA AFTER_SHA OUTPUT_DIR' >&2 + exit 2 +fi +shift +record_source="$(cd "$(dirname "$0")" && pwd -P)" +record_repo="$(cd "$record_source/../.." && pwd -P)" +record_recipe="$record_source/spend-thresholds" +source "$record_source/capture_and_render.sh" +demo_prepare_capture "$record_repo" "$1" "$2" "$3" "$4" "$5" "$record_recipe/server.py" +demo_python="$(command -v python3)" +demo_start_api "$demo_output/requests.jsonl" "$demo_output/fixtures.json" +cp "$record_recipe/scene.sh" "$demo_runtime/scene.sh" +cp "$record_recipe/scene.sh" "$demo_output/scene.sh" +{ + echo 'feature: spend threshold units and notification semantics' + echo "before commit: $demo_before_sha" + echo "after commit: $demo_after_sha" + echo 'scope: Bash PTY terminal replay; synthetic loopback API; no native graphical terminal validation' + echo 'render: 100x42 and 40x56; Menlo 18px; asciinema theme; NO_COLOR=1' + echo 'data: synthetic USD amounts in integer cents; fake admin credentials; isolated HOME' + demo_capture_metadata + "$demo_python" --version + shasum -a 256 "$demo_python" "$record_source/record-spend-thresholds.sh" \ + "$record_recipe/scene.sh" "$record_recipe/validate.py" "$record_recipe/README.md" \ + "$demo_output/fixtures.json" +} > "$demo_output/metadata.txt" +if [ -n "${DEMO_SOURCE_MANIFEST:-}" ]; then + cp "$DEMO_SOURCE_MANIFEST" "$demo_output/source-manifest.txt" + shasum -a 256 "$demo_output/source-manifest.txt" >> "$demo_output/metadata.txt" +fi +demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ + --theme asciinema --fps-cap 15 --last-frame-duration 2) +for record_width in 100 40; do + demo_window_size="${record_width}x42" + if [ "$record_width" -eq 40 ]; then demo_window_size=40x56; fi + record_scenes=() + for record_case in limit alerts json missing; do + for record_phase in before after; do + record_scene="$record_phase-$record_case-$record_width" + record_scenes+=("$record_scene") + record_endpoint="$demo_api_url/v1" + if [ "$record_case" = missing ]; then record_endpoint="$demo_api_url/missing/v1"; fi + mkdir -p "$demo_runtime/$record_scene-home" + demo_capture_scene "$record_scene" 0 "$demo_runtime/$record_phase" "$record_endpoint" \ + "$record_phase: $record_case ($record_width columns)" \ + "HOME=$demo_runtime/$record_scene-home" "DEMO_CASE=$record_case" \ + OPENAI_ADMIN_KEY=synthetic-demo-admin-key NO_COLOR=1 FORCE_COLOR=0 GOMAXPROCS=2 + done + done + demo_assemble_capture 200 "${record_scenes[@]}" + for record_suffix in cast gif txt; do + mv "$demo_output/comparison.$record_suffix" "$demo_output/comparison-$record_width.$record_suffix" + done + mv "$demo_output/comparison-scenes.txt" "$demo_output/comparison-scenes-$record_width.txt" + mv "$demo_output/media.json" "$demo_output/media-$record_width.json" +done +demo_stop_api +"$demo_python" "$record_recipe/validate.py" "$demo_output" > "$demo_output/validation.txt" +( + cd "$demo_output" + shasum -a 256 -- *.cast *.gif *.png *.txt *.json *.jsonl *.sh > SHA256SUMS +) +printf 'Recorded spend threshold terminal replays in %s\n' "$demo_output" diff --git a/scripts/demos/spend-thresholds/README.md b/scripts/demos/spend-thresholds/README.md new file mode 100644 index 00000000..d1310cb9 --- /dev/null +++ b/scripts/demos/spend-thresholds/README.md @@ -0,0 +1,62 @@ +# Spend threshold terminal replays + +This recipe compares real CLI binaries against fixed synthetic responses. +It uses the shared `capture_and_render.sh` lifecycle and a Python loopback fixture. +It requires no fixture build, installation, live API request, or billing change. + +The recordings are Bash PTY terminal replays, not native graphical terminal validation. +Each command uses fake admin credentials and an isolated configuration directory. +The fixture checks request authentication without logging credentials or request headers. + +## Record + +Reserve a PTY execution slot before running this recipe. +Prepare independently verified binaries for the baseline and candidate commits. +Provide their full 40-character commit IDs. +The recorder records binary hashes but cannot establish build provenance. + +Required tools are Python 3, Bash, asciinema, agg, ffmpeg, ffprobe, and shasum. +Use an empty output directory outside the repository. + +```sh +bash scripts/demos/record-spend-thresholds.sh --run-authorized-pty-slot \ + /path/to/before/openai /path/to/after/openai \ + "$BEFORE_SHA" "$AFTER_SHA" /path/to/evidence/spend-thresholds +``` + +Optionally set `DEMO_SOURCE_MANIFEST` to a verified source/build manifest. +The recorder copies and hashes that manifest alongside the evidence. +The authorization flag records the operator's decision; it does not reserve a slot. + +The initial comparison base is `e68939820415144d769ed02de6aa72d5b7d32948`. +The batch coordinator verified a shared baseline binary with this SHA-256: + +```text +23414722af83f1d4acfd1e80b1723448fc711083d7b76b9bcf36ee7813ff3df2 +``` + +Verify any reused binary against its own build record and digest. + +## Scenes and assertions + +Both 100-column and 40-column captures include these before/after scenes: + +- Organization limit: raw `10000` becomes `USD 100.00 per month`; reported enforcement stays visible. +- Project alert: raw `20000` becomes `USD 200.00 per month`; readable output identifies notifications without caps. +- Explicit JSON: original cents and response fields remain unchanged. +- Missing enforcement: readable output states `not reported in this response`. + +The fixture serves one alert with `has_more: false`. +This replay does not establish multi-page behavior; dedicated public tests cover pagination. +Commands use existing canonical routes and mutation-free GET requests. + +The recorder checks all command exit statuses and fixture shutdown. +The validator checks every transcript, PTY width, request path, and response hash. +Fixture request, response-write, flush, and close failures stop the recording. +SIGTERM closes the fixture cleanly with status zero. + +Each scene produces an asciicast, GIF, PNG, and plain transcript. +`comparison-100.gif` and `comparison-40.gif` contain the respective complete comparisons. +Metadata records commits, binary/tool hashes, environment, and command exit statuses. +`fixtures.json`, `requests.jsonl`, `validation.txt`, and `SHA256SUMS` retain reproducible evidence. +Review screenshots before publication. Keep generated media outside Git. diff --git a/scripts/demos/spend-thresholds/scene.sh b/scripts/demos/spend-thresholds/scene.sh new file mode 100755 index 00000000..c8b14772 --- /dev/null +++ b/scripts/demos/spend-thresholds/scene.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail +test -t 0 && test -t 1 && test -t 2 || exit 99 +printf '\033[2J\033[H' +printf '%s\n' "$DEMO_SCENE_LABEL" 'Terminal replay. Synthetic data.' +printf '\n' +case "$DEMO_CASE" in + limit|missing) + printf '%s\n' '$ openai admin organization spend-limit retrieve' + openai admin organization spend-limit retrieve + ;; + alerts) + printf '%s\n' '$ openai admin organization projects spend-alerts list --project-id proj_demo --max-items 2' + openai admin organization projects spend-alerts list --project-id proj_demo --max-items 2 + ;; + json) + printf '%s\n' '$ openai admin organization spend-limit retrieve --format json' + openai admin organization spend-limit retrieve --format json + ;; + *) echo 'Unknown spend demo scene.' >&2; exit 2;; +esac +printf '\n$ ' +sleep 2 diff --git a/scripts/demos/spend-thresholds/server.py b/scripts/demos/spend-thresholds/server.py new file mode 100755 index 00000000..6d1e18d8 --- /dev/null +++ b/scripts/demos/spend-thresholds/server.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Serve fixed spend responses on loopback and fail on fixture I/O errors.""" + +import hashlib +import http.server +import json +import os +from pathlib import Path +import signal +import sys +from urllib.parse import urlsplit + + +def write_text(path, text): + with Path(path).open("x", encoding="utf-8") as destination: + if destination.write(text) != len(text): + raise OSError("Incomplete fixture file write") + + +def main(): + if len(sys.argv) != 4: + raise SystemExit("usage: server.py ADDRESS_FILE REQUEST_LOG FIXTURE_METADATA") + limit = {"object": "organization.spend_limit", "threshold_amount": 10000, + "currency": "USD", "interval": "month", "enforcement": {"status": "enforcing"}} + missing = {key: value for key, value in limit.items() if key != "enforcement"} + alerts = [{"id": "alert_demo", "object": "project.spend_alert", "threshold_amount": 20000, + "currency": "USD", "interval": "month", + "notification_channel": {"type": "email", "recipients": ["finance@example.test"]}}] + fixtures = { + "/v1/organization/spend_limit": limit, + "/missing/v1/organization/spend_limit": missing, + "/v1/organization/projects/proj_demo/spend_alerts": { + "object": "list", "data": alerts, "first_id": "alert_demo", "last_id": "alert_demo", "has_more": False}, + } + bodies = {path: json.dumps(value, separators=(",", ":")).encode() for path, value in fixtures.items()} + write_text(sys.argv[3], json.dumps(fixtures, indent=2) + "\n") + running = True + + def stop(*_): + nonlocal running + running = False + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + with open(sys.argv[2], "x", encoding="utf-8") as request_log: + class Fixture(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + path = urlsplit(self.path).path + if self.headers.get("Authorization") != "Bearer synthetic-demo-admin-key": + raise ValueError("Unexpected synthetic fixture authentication") + body = bodies[path] + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.wfile.write(body) != len(body): + raise OSError("Incomplete fixture response write") + self.wfile.flush() + entry = json.dumps({"method": "GET", "path": path, "status": 200, + "response_sha256": hashlib.sha256(body).hexdigest()}) + "\n" + if request_log.write(entry) != len(entry): + raise OSError("Incomplete fixture request-log write") + request_log.flush() + + class Server(http.server.HTTPServer): + failed = False + + def get_request(self): + connection, address = super().get_request() + connection.settimeout(2) + return connection, address + + def handle_error(self, *_): + self.failed = True + print("Synthetic fixture request or write failed.", file=sys.stderr) + + with Server(("127.0.0.1", 0), Fixture) as server: + server.timeout = 0.05 + address_file = Path(sys.argv[1]) + pending_address = address_file.with_suffix(".tmp") + write_text(pending_address, f"http://127.0.0.1:{server.server_port}\n") + os.replace(pending_address, address_file) + while running and not server.failed: + server.handle_request() + if server.failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/spend-thresholds/validate.py b/scripts/demos/spend-thresholds/validate.py new file mode 100755 index 00000000..6e3f6fb8 --- /dev/null +++ b/scripts/demos/spend-thresholds/validate.py @@ -0,0 +1,64 @@ +#!/usr/bin/env python3 +"""Verify every replay, command status, request, and unchanged JSON response.""" + +import hashlib +import json +from pathlib import Path +import sys + + +def check(condition, message): + if not condition: + raise SystemExit(message) + + +def main(): + check(len(sys.argv) == 2, "usage: validate.py CAPTURE_DIRECTORY") + root = Path(sys.argv[1]) + fixtures = json.loads((root / "fixtures.json").read_text()) + metadata = (root / "metadata.txt").read_text() + requests = [] + for width in (100, 40): + for case in ("limit", "alerts", "json", "missing"): + for phase in ("before", "after"): + scene = f"{phase}-{case}-{width}" + text = (root / f"{scene}.txt").read_text() + compact = "".join(text.split()) + header = json.loads((root / f"{scene}.cast").read_text().splitlines()[0]) + check(header["width"] == width, f"{scene}: incorrect PTY width") + check("Terminalreplay.Syntheticdata." in compact, f"{scene}: missing evidence label") + check(f"{scene} exit status: 0 (expected 0)" in metadata, f"{scene}: missing successful exit") + path = "/v1/organization/spend_limit" + if case == "alerts": + path = "/v1/organization/projects/proj_demo/spend_alerts" + check("alert_demo" in text and "finance@example.test" in text, f"{scene}: lost alert fields") + elif case == "missing": + path = "/missing/v1/organization/spend_limit" + if case == "json": + value, _ = json.JSONDecoder().raw_decode(text[text.index("{"):]) + check(value == fixtures[path], f"{scene}: explicit JSON changed") + check("Spendthreshold:" not in compact, f"{scene}: projection leaked into JSON") + elif phase == "before": + amount = "20000" if case == "alerts" else "10000" + check(f"Thresholdamount:{amount}" in compact, f"{scene}: baseline cents are missing") + check("Spendthreshold:" not in compact, f"{scene}: baseline already contains the projection") + else: + amount = "200.00" if case == "alerts" else "100.00" + check(f"Spendthreshold:USD{amount}permonth" in compact, f"{scene}: missing units") + if case == "alerts": + check("Alertsnotify;theyarenotspendingcaps." in compact, f"{scene}: missing alert semantics") + elif case == "missing": + check("Enforcement:notreportedinthisresponse" in compact, f"{scene}: missing enforcement caveat") + else: + check("Status:enforcing" in compact, f"{scene}: reported enforcement changed") + body = json.dumps(fixtures[path], separators=(",", ":")).encode() + requests.append({"method": "GET", "path": path, "status": 200, + "response_sha256": hashlib.sha256(body).hexdigest()}) + actual = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] + check(actual == requests, "Unexpected fixture requests, order, or response hashes") + print("PASS: 16 successful scenes; 100/40-column PTYs; exact fixture requests and hashes.") + print("PASS: baseline cents; readable USD/month; preserved JSON; alert semantics; missing enforcement.") + + +if __name__ == "__main__": + main() From ae7fe7fa13ebdd4a7c767553884ecd136c35a817 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Fri, 9 Oct 2026 12:35:07 -0700 Subject: [PATCH 02/44] test: size spend replays to their output --- scripts/demos/record-spend-thresholds.sh | 6 +++--- scripts/demos/spend-thresholds/validate.py | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/scripts/demos/record-spend-thresholds.sh b/scripts/demos/record-spend-thresholds.sh index f014e90b..32460f02 100755 --- a/scripts/demos/record-spend-thresholds.sh +++ b/scripts/demos/record-spend-thresholds.sh @@ -20,7 +20,7 @@ cp "$record_recipe/scene.sh" "$demo_output/scene.sh" echo "before commit: $demo_before_sha" echo "after commit: $demo_after_sha" echo 'scope: Bash PTY terminal replay; synthetic loopback API; no native graphical terminal validation' - echo 'render: 100x42 and 40x56; Menlo 18px; asciinema theme; NO_COLOR=1' + echo 'render: 100x22 and 40x24; Menlo 18px; asciinema theme; NO_COLOR=1' echo 'data: synthetic USD amounts in integer cents; fake admin credentials; isolated HOME' demo_capture_metadata "$demo_python" --version @@ -35,8 +35,8 @@ fi demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ --theme asciinema --fps-cap 15 --last-frame-duration 2) for record_width in 100 40; do - demo_window_size="${record_width}x42" - if [ "$record_width" -eq 40 ]; then demo_window_size=40x56; fi + demo_window_size="${record_width}x22" + if [ "$record_width" -eq 40 ]; then demo_window_size=40x24; fi record_scenes=() for record_case in limit alerts json missing; do for record_phase in before after; do diff --git a/scripts/demos/spend-thresholds/validate.py b/scripts/demos/spend-thresholds/validate.py index 6e3f6fb8..ae5967cb 100755 --- a/scripts/demos/spend-thresholds/validate.py +++ b/scripts/demos/spend-thresholds/validate.py @@ -26,6 +26,7 @@ def main(): compact = "".join(text.split()) header = json.loads((root / f"{scene}.cast").read_text().splitlines()[0]) check(header["width"] == width, f"{scene}: incorrect PTY width") + check(header["height"] == (24 if width == 40 else 22), f"{scene}: incorrect PTY height") check("Terminalreplay.Syntheticdata." in compact, f"{scene}: missing evidence label") check(f"{scene} exit status: 0 (expected 0)" in metadata, f"{scene}: missing successful exit") path = "/v1/organization/spend_limit" From 56bc61b92941f8753a143eaf626dc63055615ebf Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Fri, 9 Oct 2026 12:46:45 -0700 Subject: [PATCH 03/44] fix: preserve ambiguous duplicate spend response fields --- cmd/openai/spend_thresholds_test.go | 87 +++++++++++++++++++ docs/spend-thresholds.md | 2 + pkg/transformers/project_spend_threshold.go | 19 ++++ .../project_spend_threshold_test.go | 37 +++++++- 4 files changed, 144 insertions(+), 1 deletion(-) diff --git a/cmd/openai/spend_thresholds_test.go b/cmd/openai/spend_thresholds_test.go index 36c03f78..176c158b 100644 --- a/cmd/openai/spend_thresholds_test.go +++ b/cmd/openai/spend_thresholds_test.go @@ -480,3 +480,90 @@ func TestMainSpendThresholdLargeUnknownField(t *testing.T) { t.Fatalf("large extracted field changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) } } + +func TestMainSpendThresholdDuplicateKeys(t *testing.T) { + for _, tc := range []struct { + name, resource string + project bool + fields, text string + }{ + { + name: "threshold amount", resource: "spend-limit", + fields: `{"threshold_amount":10000,"threshold_amount":90000,"currency":"USD","interval":"month"}`, + text: "Threshold amount: 10000\nThreshold amount: 90000\nCurrency: USD\nInterval: month\n", + }, + { + name: "currency", resource: "spend-limit", project: true, + fields: `{"threshold_amount":10000,"currency":"USD","currency":"EUR","interval":"month"}`, + text: "Threshold amount: 10000\nCurrency: USD\nCurrency: EUR\nInterval: month\n", + }, + { + name: "interval", resource: "spend-alerts", + fields: `{"threshold_amount":10000,"currency":"USD","interval":"month","interval":"year"}`, + text: "Threshold amount: 10000\nCurrency: USD\nInterval: month\nInterval: year\n", + }, + { + name: "enforcement", resource: "spend-limit", project: true, + fields: `{"threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"inactive"},"enforcement":{"status":"enforcing"}}`, + text: "Threshold amount: 10000\nCurrency: USD\nInterval: month\nEnforcement:\n Status: inactive\nEnforcement:\n Status: enforcing\n", + }, + { + name: "object", resource: "spend-alerts", project: true, + fields: `{"object":"future.spend_alert","threshold_amount":10000,"currency":"USD","interval":"month"}`, + text: "Object: future.spend_alert\nThreshold amount: 10000\nCurrency: USD\nInterval: month\n", + }, + { + name: "escaped threshold key", resource: "spend-alerts", project: true, + fields: `{"threshold_amount":10000,"threshold_\u0061mount":90000,"currency":"USD","interval":"month"}`, + text: "Threshold amount: 10000\nThreshold amount: 90000\nCurrency: USD\nInterval: month\n", + }, + { + name: "unknown field", resource: "spend-alerts", + fields: `{"threshold_amount":10000,"currency":"USD","interval":"month","future":"first","future":"second"}`, + text: "Threshold amount: 10000\nCurrency: USD\nInterval: month\nFuture: first\nFuture: second\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + body := spendThresholdBody(tc.project, tc.resource, tc.fields) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := spendThresholdPath(tc.project, tc.resource) + if tc.resource == "spend-alerts" { + path += "/alert_synthetic" + } + if r.Method != http.MethodGet || r.URL.Path != path { + t.Errorf("unexpected duplicate-key request: %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + scope := "organization" + if tc.project { + scope = "project" + } + object := scope + "." + strings.TrimSuffix(strings.ReplaceAll(tc.resource, "-", "_"), "s") + for _, format := range []string{"", "text", "raw"} { + t.Run("format="+format, func(t *testing.T) { + args := spendThresholdArgs(tc.project, tc.resource, "retrieve") + if format != "" { + args = append([]string{"--format", format}, args...) + } + got := runSpendThresholdCommand(t, server, args...) + // Compare text and bytes directly: decoding into maps discards duplicates. + want := "Object: " + object + "\n" + tc.text + if format == "raw" { + want = body + "\n" + } + if got.code != 0 || got.stderr != "" || got.stdout != want { + t.Fatalf("duplicate fields changed: got=%+v; want stdout=%q", got, want) + } + for _, synthesized := range []string{"Spend threshold:", "spend_threshold", "not reported in this response", "Alert behavior:", "alert_behavior"} { + if strings.Contains(got.stdout, synthesized) { + t.Errorf("ambiguous response synthesized %q: %q", synthesized, got.stdout) + } + } + }) + } + }) + } +} diff --git a/docs/spend-thresholds.md b/docs/spend-thresholds.md index 69696ad3..48b5313b 100644 --- a/docs/spend-thresholds.md +++ b/docs/spend-thresholds.md @@ -55,6 +55,8 @@ Unknown currencies retain the original amount in cents without assuming a curren Unknown intervals remain literal values. Missing currency or interval receives a `not reported` label. The CLI preserves exact large integers without floating-point conversion. Unexpected decimal or exponent values retain their original numeric spelling in cents. +Responses with duplicate top-level keys retain their original readable fields instead of a summary. +This avoids assigning ambiguous currency, interval, or enforcement information to a threshold. Existing update/create flags still accept cents. This change does not convert inputs or change validation. Limit inputs require at least one cent in the API schema; alert inputs allow zero. diff --git a/pkg/transformers/project_spend_threshold.go b/pkg/transformers/project_spend_threshold.go index e41f33d4..ce487a40 100644 --- a/pkg/transformers/project_spend_threshold.go +++ b/pkg/transformers/project_spend_threshold.go @@ -44,6 +44,25 @@ func projectSpendThreshold(ctx context.Context, value gjson.Result, object strin if !value.IsObject() || value.Get("object").String() != object { return value, nil } + // Get selects the first occurrence, while ForEach presents every occurrence. + // Preserve ambiguous responses instead of attaching one field's units/value + // to another field with the same name, including escaped key spellings. + seen := make(map[string]bool) + duplicate := false + value.ForEach(func(key, _ gjson.Result) bool { + if ctx.Err() != nil { + return false + } + duplicate = seen[key.Str] + seen[key.Str] = true + return !duplicate + }) + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + if duplicate { + return value, nil + } amount := value.Get("threshold_amount") summary := "" var summaryJSON []byte diff --git a/pkg/transformers/project_spend_threshold_test.go b/pkg/transformers/project_spend_threshold_test.go index 6aa1a4ee..1d0a0df7 100644 --- a/pkg/transformers/project_spend_threshold_test.go +++ b/pkg/transformers/project_spend_threshold_test.go @@ -145,7 +145,8 @@ func TestSpendThresholdCancellationAndLargeValues(t *testing.T) { func TestSpendThresholdCancelsDuringScan(t *testing.T) { ctx, cancel := context.WithCancel(t.Context()) defer cancel() - controlled := &cancelSummaryContext{Context: ctx, cancel: cancel, after: 4} + // Entry and duplicate-key checks consume six polls before amount scanning. + controlled := &cancelSummaryContext{Context: ctx, cancel: cancel, after: 9} raw := `{"object":"organization.spend_limit","threshold_amount":` + strings.Repeat("9", 200000) + `,"currency":"USD","interval":"month"}` transform := Select(Route{"(resource) admin.organization.spend_limit > (method) retrieve", OutputResponse}) got, err := transform(controlled, gjson.Parse(raw)) @@ -153,3 +154,37 @@ func TestSpendThresholdCancelsDuringScan(t *testing.T) { t.Fatalf("cancellation during scan returned partial output: bytes=%d error=%v", len(got.Raw), err) } } + +func TestSpendThresholdPreservesDuplicateKeys(t *testing.T) { + for _, fields := range []string{ + `"threshold_amount":10000,"threshold_amount":90000`, + `"threshold_amount":10000,"threshold\u005famount":90000`, + `"threshold_amount":10000,"currency":"USD","currency":"EUR"`, + `"threshold_amount":10000,"interval":"month","interval":"year"`, + `"threshold_amount":10000,"enforcement":{"status":"inactive"},"enforcement":{"status":"enforcing"}`, + `"threshold_amount":10000,"object":"future.spend_limit"`, + `"threshold_amount":10000,"future":9007199254740993,"future":0.1234567890123456789`, + `"threshold_amount":10000,"spend_threshold":null,"spend_threshold":"future"`, + `"threshold_amount":10000,"alert_behavior":null,"alert_behavior":"future"`, + } { + for _, object := range []string{"organization.spend_limit", "project.spend_alert"} { + raw := ` { "object":"` + object + `",` + fields + ` } ` + value := gjson.Parse(raw) + got, err := projectSpendThreshold(t.Context(), value, object, strings.HasSuffix(object, "spend_alert")) + if err != nil || got.Raw != value.Raw { + t.Fatalf("ambiguous object changed: got=%s want=%s error=%v", got.Raw, value.Raw, err) + } + } + } +} + +func TestSpendThresholdCancelsDuringDuplicateScan(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + controlled := &cancelSummaryContext{Context: ctx, cancel: cancel, after: 4} + value := gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month"}`) + got, err := projectSpendThreshold(controlled, value, "organization.spend_limit", false) + if !errors.Is(err, context.Canceled) || got.Raw != "" { + t.Fatalf("cancellation during duplicate scan returned partial output: %s error=%v", got.Raw, err) + } +} From b86245e1a66bd41128a94626763e51a41c7e1d8c Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 10:45:27 -0700 Subject: [PATCH 04/44] feat: add interactive limits workspace and recovery guidance --- cmd/openai/limits_workflow_test.go | 533 +++++++++++ cmd/openai/limits_workflow_unix_test.go | 56 ++ cmd/openai/spend_error_recovery_test.go | 384 ++++++++ docs/limits-workspace.md | 135 +++ .../transform/project_limits_overview.go | 166 ++++ .../transform/project_limits_overview_test.go | 127 +++ pkg/custom/command.go | 1 + pkg/custom/limits_commands.go | 119 +++ pkg/custom/limits_draft.go | 309 +++++++ pkg/custom/limits_service.go | 733 +++++++++++++++ pkg/custom/limits_service_test.go | 812 +++++++++++++++++ pkg/custom/limits_values.go | 443 +++++++++ pkg/custom/limits_values_test.go | 181 ++++ pkg/custom/limits_workspace.go | 851 ++++++++++++++++++ pkg/custom/limits_workspace_test.go | 727 +++++++++++++++ pkg/custom/limits_workspace_view.go | 135 +++ pkg/custom/local_errors.go | 24 +- pkg/custom/readable_errors.go | 25 +- pkg/custom/spend_error_guidance.go | 100 ++ pkg/custom/spend_error_guidance_test.go | 164 ++++ scripts/demos/limits-workspace/README.md | 78 ++ scripts/demos/limits-workspace/record.sh | 79 ++ scripts/demos/limits-workspace/scene.py | 269 ++++++ scripts/demos/limits-workspace/server.py | 199 ++++ scripts/demos/limits-workspace/validate.py | 70 ++ 25 files changed, 6712 insertions(+), 8 deletions(-) create mode 100644 cmd/openai/limits_workflow_test.go create mode 100644 cmd/openai/limits_workflow_unix_test.go create mode 100644 cmd/openai/spend_error_recovery_test.go create mode 100644 docs/limits-workspace.md create mode 100644 internal/output/transform/project_limits_overview.go create mode 100644 internal/output/transform/project_limits_overview_test.go create mode 100644 pkg/custom/limits_commands.go create mode 100644 pkg/custom/limits_draft.go create mode 100644 pkg/custom/limits_service.go create mode 100644 pkg/custom/limits_service_test.go create mode 100644 pkg/custom/limits_values.go create mode 100644 pkg/custom/limits_values_test.go create mode 100644 pkg/custom/limits_workspace.go create mode 100644 pkg/custom/limits_workspace_test.go create mode 100644 pkg/custom/limits_workspace_view.go create mode 100644 pkg/custom/spend_error_guidance.go create mode 100644 pkg/custom/spend_error_guidance_test.go create mode 100644 scripts/demos/limits-workspace/README.md create mode 100755 scripts/demos/limits-workspace/record.sh create mode 100755 scripts/demos/limits-workspace/scene.py create mode 100755 scripts/demos/limits-workspace/server.py create mode 100755 scripts/demos/limits-workspace/validate.py diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go new file mode 100644 index 00000000..6baee588 --- /dev/null +++ b/cmd/openai/limits_workflow_test.go @@ -0,0 +1,533 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "testing" + + "github.com/tidwall/gjson" +) + +const limitsWorkflowProject = `{"id":"proj_synthetic","object":"organization.project","name":"Synthetic project","status":"active","future":0.100000000000000001}` +const limitsWorkflowRate = `{"id":"rl_synthetic","object":"project.rate_limit","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993,"max_audio_megabytes_per_1_minute":1.25,"future":{"exact":123456789012345678901234567890}}` + +func limitsWorkflowArgs(project bool, show bool) []string { + args := []string{"admin", "limits"} + if project { + args = []string{"projects", "limits", "--project-id", "proj_synthetic"} + } + if show { + args = append(args, "show") + } + return args +} + +func limitsWorkflowRecords(project bool) (limit, alert string) { + limit = spendThresholdBody(project, "spend-limit", `{"threshold_amount":9007199254740993,"currency":"USD","interval":"month","enforcement":{"status":"enforcing"},"future":{"decimal":0.100000000000000001}}`) + alert = spendThresholdBody(project, "spend-alerts", `{"id":"alert_synthetic","threshold_amount":10000,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["synthetic@example.invalid"],"subject_prefix":null},"future":{"exact":123456789012345678901234567890}}`) + return limit, alert +} + +func limitsWorkflowPage(items string, more bool, last string) string { + return fmt.Sprintf(`{"object":"list","data":[%s],"has_more":%t,"last_id":%q}`, items, more, last) +} + +func limitsWorkflowEnvelope(project bool, limit, alerts string, configured string, alertsComplete bool, rates, errors string) string { + fields := `"scope":"organization",` + if project { + fields = `"scope":"project","project_id":"proj_synthetic","project":` + limitsWorkflowProject + `,` + } + fields += `"spend_limit":` + limit + `,"spend_limit_configured":` + configured + `,"spend_alerts":[` + alerts + `],"spend_alerts_complete":` + fmt.Sprint(alertsComplete) + if project { + fields += `,"project_rate_limits":[` + rates + `],"project_rate_limits_complete":true` + } + return `{` + fields + `,"source_errors":[` + errors + `]}` +} + +type limitsWorkflowRequests struct { + mu sync.Mutex + paths []string +} + +func (r *limitsWorkflowRequests) count(path string) int { + r.mu.Lock() + defer r.mu.Unlock() + count := 0 + for _, request := range r.paths { + if strings.SplitN(request, "?", 2)[0] == path { + count++ + } + } + return count +} + +func newLimitsWorkflowServer(t *testing.T, respond func(*http.Request) (int, string)) (*httptest.Server, *limitsWorkflowRequests) { + t.Helper() + requests := &limitsWorkflowRequests{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.mu.Lock() + requests.paths = append(requests.paths, r.URL.RequestURI()) + requests.mu.Unlock() + if r.Method != http.MethodGet { + t.Errorf("read-only overview sent %s", r.Method) + } + if r.Header.Get("Authorization") != "Bearer sk-fake-spend-admin" { + t.Error("overview did not select the synthetic admin credential") + } + body, err := io.ReadAll(r.Body) + if err != nil || len(body) != 0 { + t.Errorf("read-only request has a body or read error: bytes=%d error=%v", len(body), err) + } + status, response := respond(r) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("x-should-retry", "false") + w.WriteHeader(status) + io.WriteString(w, response) + })) + t.Cleanup(server.Close) + return server, requests +} + +func limitsWorkflowSuccess(t *testing.T, project bool, r *http.Request) (int, string) { + t.Helper() + limit, alert := limitsWorkflowRecords(project) + switch r.URL.Path { + case "/organization/costs": + return http.StatusOK, `{"object":"page","data":[],"has_more":false,"next_page":null}` + case "/organization/projects/proj_synthetic": + return http.StatusOK, limitsWorkflowProject + case spendThresholdPath(project, "spend-limit"): + return http.StatusOK, limit + case spendThresholdPath(project, "spend-alerts"): + return http.StatusOK, limitsWorkflowPage(alert, false, "alert_synthetic") + case spendThresholdPath(true, "rate-limits"): + return http.StatusOK, limitsWorkflowPage(limitsWorkflowRate, false, "rl_synthetic") + default: + t.Errorf("unexpected overview route: %s", r.URL.Path) + return http.StatusNotFound, `{"error":{"message":"unexpected synthetic route"}}` + } +} + +func TestMainLimitsWorkflowReadableAndFormats(t *testing.T) { + for _, project := range []bool{false, true} { + for _, mode := range []string{"bare", "show", "text", "JSON", "jsonl", "raw", "yaml", "pretty", "extraction", "raw-output"} { + t.Run(fmt.Sprintf("project=%t/%s", project, mode), func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, project, r) + }) + args := limitsWorkflowArgs(project, mode != "bare") + switch mode { + case "bare", "show": + case "extraction": + args = append([]string{"--transform", "spend_limit.threshold_amount"}, args...) + case "raw-output": + args = append([]string{"--raw-output"}, args...) + default: + args = append([]string{"--format", mode}, args...) + } + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("overview failed: %+v", got) + } + for _, resource := range []string{"spend-limit", "spend-alerts"} { + if n := requests.count(spendThresholdPath(project, resource)); n != 1 { + t.Errorf("%s requests=%d, want one", resource, n) + } + } + if project && (requests.count("/organization/projects/proj_synthetic") != 1 || requests.count(spendThresholdPath(true, "rate-limits")) != 1) { + t.Error("project overview must validate the project and fetch its rates") + } + limit, alert := limitsWorkflowRecords(project) + want := limitsWorkflowEnvelope(project, limit, alert, "true", true, limitsWorkflowRate, "") + switch mode { + case "bare", "show", "text": + for _, text := range []string{"USD 90071992547409.93 per month", "USD 100.00 per month", "Alerts notify; they are not spending caps.", "0.100000000000000001", "123456789012345678901234567890"} { + if !strings.Contains(got.stdout, text) { + t.Errorf("readable overview lost %q: %q", text, got.stdout) + } + } + if project { + for _, text := range []string{"Max requests per minute: 0", "Max tokens per minute: 9007199254740993", "Max audio megabytes per minute: 1.25"} { + if !strings.Contains(got.stdout, text) { + t.Errorf("rate units lost %q: %q", text, got.stdout) + } + } + } + case "yaml", "pretty": + for _, text := range []string{"threshold_amount: 9007199254740993", "decimal: 0.100000000000000001", "exact: 123456789012345678901234567890"} { + if !strings.Contains(got.stdout, text) { + t.Errorf("%s lost %q: %q", mode, text, got.stdout) + } + } + case "extraction": + if got.stdout != "9007199254740993\n" { + t.Errorf("overview extraction changed: %q", got.stdout) + } + default: + costs := gjson.Get(got.stdout, "costs") + if costs.Get("total").Str != "0.00" || costs.Get("currency").Str != "USD" || !costs.Get("complete").Bool() { + t.Fatalf("Costs summary missing: %s", costs.Raw) + } + want = strings.TrimSuffix(want, "}") + `,"costs":` + costs.Raw + "}" + assertSpendThresholdJSON(t, got.stdout, want) + } + if strings.Contains(got.stdout, "\x1b") { + t.Error("nonterminal overview emitted terminal control sequences") + } + }) + } + } +} + +func TestMainLimitsWorkflowMissingLimitAndPartialFailures(t *testing.T) { + for _, tc := range []struct { + name, failedResource string + project bool + status int + }{ + {"organization no limit", "spend-limit", false, http.StatusNotFound}, + {"validated project no limit", "spend-limit", true, http.StatusNotFound}, + {"limit forbidden", "spend-limit", true, http.StatusForbidden}, + {"missing project", "project", true, http.StatusNotFound}, + {"alerts forbidden", "spend-alerts", false, http.StatusForbidden}, + {"rates unavailable", "rate-limits", true, http.StatusBadGateway}, + } { + t.Run(tc.name, func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + failedPath := spendThresholdPath(tc.project, tc.failedResource) + if tc.failedResource == "project" { + failedPath = "/organization/projects/proj_synthetic" + } + if r.URL.Path == failedPath { + return tc.status, `{"error":{"type":"synthetic_error","message":"synthetic private limits response"}}` + } + return limitsWorkflowSuccess(t, tc.project, r) + }) + args := append([]string{"--format", "json"}, limitsWorkflowArgs(tc.project, true)...) + got := runSpendThresholdCommand(t, server, args...) + if !json.Valid([]byte(got.stdout)) { + t.Fatalf("partial overview is not complete JSON: %+v", got) + } + value := gjson.Parse(got.stdout) + { + if tc.failedResource == "spend-limit" && value.Get("spend_limit_configured").Type != gjson.Null { + t.Fatalf("failed lookup incorrectly established configuration: %s", got.stdout) + } + if got.code != 1 || strings.TrimSpace(got.stderr) == "" || value.Get("source_errors.#").Int() == 0 { + t.Fatalf("partial failure disappeared: %+v", got) + } + if !json.Valid([]byte(got.stderr)) { + t.Errorf("structured error stream is invalid: %q", got.stderr) + } + resource := strings.ReplaceAll(tc.failedResource, "-", "_") + if resource == "rate_limits" { + resource = "project_rate_limits" + } + matched := false + for _, failure := range value.Get("source_errors").Array() { + if failure.Get("resource").Str == resource { + matched = failure.Get("status").Str == "api_error" && failure.Get("status_code").Int() == int64(tc.status) + } + failure.ForEach(func(key, _ gjson.Result) bool { + if key.Str != "resource" && key.Str != "status" && key.Str != "status_code" { + t.Errorf("source failure includes undocumented field %q", key.Str) + } + return true + }) + } + if !matched { + t.Errorf("missing safe source/status record: %s", got.stdout) + } + if tc.failedResource == "project" { + if value.Get("spend_limit_configured").Raw != "null" || requests.count(spendThresholdPath(true, "spend-limit")) != 0 { + t.Errorf("unvalidated project inferred an absent limit: %s", got.stdout) + } + } else if tc.failedResource == "spend-limit" { + if value.Get("spend_limit_configured").Raw != "null" || value.Get("spend_limit").Raw != "null" || value.Get("spend_alerts.#").Int() != 1 { + t.Errorf("failed limit became unconfigured or lost independent alerts: %s", got.stdout) + } + } else if value.Get("spend_limit.threshold_amount").Raw != "9007199254740993" { + t.Errorf("partial failure lost successful spend data: %s", got.stdout) + } + } + for _, secret := range []string{"synthetic private limits response", "sk-fake-spend-admin", "sk-fake-spend-test"} { + if strings.Contains(got.stdout+got.stderr, secret) { + t.Errorf("overview diagnostics included private fixture data %q", secret) + } + } + }) + } +} + +func TestMainLimitsWorkflowAmbiguousAndOptionalRecords(t *testing.T) { + const limit = `{"object":"project.spend_limit","threshold_amount":10000,"threshold_amount":90000,"currency":"USD","interval":"month"}` + const alert = `{"object":"project.spend_alert","id":"alert_synthetic","threshold_amount":10000,"currency":"USD","currency":"EUR","interval":"month"}` + const rate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":7,"future":"first","future":"second"}` + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + switch r.URL.Path { + case spendThresholdPath(true, "spend-limit"): + return http.StatusOK, limit + case spendThresholdPath(true, "spend-alerts"): + return http.StatusOK, limitsWorkflowPage(alert, false, "alert_synthetic") + case spendThresholdPath(true, "rate-limits"): + return http.StatusOK, limitsWorkflowPage(rate, false, "rl_synthetic") + default: + return limitsWorkflowSuccess(t, true, r) + } + }) + args := limitsWorkflowArgs(true, true) + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("ambiguous records lost read-only access: %+v", got) + } + for _, text := range []string{"Threshold amount: 10000", "Threshold amount: 90000", "Currency: USD", "Currency: EUR", "Max requests per 1 minute: 0", "Future: first", "Future: second"} { + if !strings.Contains(got.stdout, text) { + t.Errorf("readable overview lost %q: %q", text, got.stdout) + } + } + for _, absent := range []string{"Spend threshold:", "Alerts notify;", "not reported in this response", "Max images", "Max audio", "Max requests per day", "Max batch"} { + if strings.Contains(got.stdout, absent) { + t.Errorf("ambiguous overview invented %q: %q", absent, got.stdout) + } + } + got = runSpendThresholdCommand(t, server, append([]string{"--format", "raw"}, args...)...) + if got.code != 0 || got.stderr != "" || !json.Valid([]byte(got.stdout)) { + t.Fatalf("raw ambiguous overview failed: %+v", got) + } + for _, record := range []string{limit, alert, rate} { + if !strings.Contains(got.stdout, record) { + t.Errorf("raw overview discarded duplicate fields: %q", got.stdout) + } + } +} + +func TestMainLimitsWorkflowNoninteractiveGuards(t *testing.T) { + for _, tc := range []struct { + name string + flags, env []string + }{ + {"quiet", []string{"--quiet"}, nil}, + {"CI", nil, []string{"CI=1"}}, + {"dumb", nil, []string{"TERM=dumb"}}, + {"explicit auto", []string{"--format", "auto"}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, false, r) + }) + env := append([]string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_API_KEY=sk-fake-spend-test", "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "FORCE_COLOR=0"}, tc.env...) + args := append([]string{"openai"}, tc.flags...) + args = append(args, limitsWorkflowArgs(false, false)...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "USD 90071992547409.93 per month") || strings.Contains(got.stdout, "\x1b") { + t.Fatalf("noninteractive invocation entered a workspace or lost data: %+v", got) + } + }) + } +} + +func TestMainLimitsWorkflowEnvironmentHeadersAndDebug(t *testing.T) { + for _, debug := range []bool{false, true} { + t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) { + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.Header.Get("X-Limits-Env") != "synthetic-private-header" { + t.Error("overview lost configured environment headers") + } + return limitsWorkflowSuccess(t, false, r) + }) + env := []string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_API_KEY=sk-fake-spend-test", "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "OPENAI_CUSTOM_HEADERS=X-Limits-Env: synthetic-private-header", "FORCE_COLOR=0"} + args := []string{"openai", "--format", "json"} + if debug { + args = append(args, "--debug") + } + args = append(args, limitsWorkflowArgs(false, true)...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if got.code != 0 || !json.Valid([]byte(got.stdout)) || (got.stderr != "") != debug { + t.Fatalf("debug or environment-header handling changed: %+v", got) + } + for _, secret := range []string{"synthetic-private-header", "sk-fake-spend-admin", "sk-fake-spend-test"} { + if strings.Contains(got.stdout+got.stderr, secret) { + t.Errorf("debug output exposed fixture credential/header %q", secret) + } + } + }) + } +} + +func TestMainLimitsWorkflowPaginationAndBounds(t *testing.T) { + for _, resource := range []string{"spend-alerts", "rate-limits"} { + for _, tc := range []struct { + name, max string + count, calls int + complete, cycle bool + }{ + {name: "unlimited", max: "-1", count: 3, calls: 2, complete: true}, + {name: "zero", max: "0", calls: 1}, + {name: "one", max: "1", count: 1, calls: 1}, + {name: "page boundary", max: "2", count: 2, calls: 1}, + {name: "cursor cycle", max: "-1", count: -1, calls: 2, cycle: true}, + } { + t.Run(resource+"/"+tc.name, func(t *testing.T) { + project := resource == "rate-limits" + path := spendThresholdPath(project, resource) + _, alert := limitsWorkflowRecords(project) + item := alert + field := "spend_alerts" + if project { + item, field = limitsWorkflowRate, "project_rate_limits" + } + originalID := gjson.Get(item, "id").Str + first := strings.Replace(item, originalID, "item_a", 1) + second := strings.Replace(item, originalID, "item_b", 1) + third := strings.Replace(item, originalID, "item_c", 1) + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.URL.Path != path { + return limitsWorkflowSuccess(t, project, r) + } + if tc.max == "0" && r.URL.Query().Get("limit") != "1" { + t.Errorf("zero bound requested limit=%q, want 1", r.URL.Query().Get("limit")) + } + switch r.URL.Query().Get("after") { + case "": + return http.StatusOK, limitsWorkflowPage(first+","+second, true, "item_b") + case "item_b": + if tc.cycle { + return http.StatusOK, limitsWorkflowPage(first+","+second, true, "item_b") + } + return http.StatusOK, limitsWorkflowPage(third, false, "item_c") + default: + t.Errorf("unexpected pagination cursor: %s", r.URL.Query().Get("after")) + return http.StatusBadRequest, `{"error":{"message":"unexpected cursor"}}` + } + }) + args := append([]string{"--format", "json"}, limitsWorkflowArgs(project, false)...) + args = append(args, "--max-items", tc.max, "show") + got := runSpendThresholdCommand(t, server, args...) + if !json.Valid([]byte(got.stdout)) || requests.count(path) != tc.calls { + t.Fatalf("pagination failed: %+v; calls=%d", got, requests.count(path)) + } + value := gjson.Parse(got.stdout) + if tc.cycle { + if got.code != 1 || value.Get("source_errors.#").Int() == 0 || value.Get(field+".#").Int() == 0 { + t.Errorf("cursor cycle lost failure or partial data: %+v", got) + } + } else if got.code != 0 || got.stderr != "" || value.Get(field+".#").Int() != int64(tc.count) { + t.Errorf("bounded list changed: %+v", got) + } + if value.Get(field+"_complete").Bool() != tc.complete { + t.Errorf("list completeness changed: %s", got.stdout) + } + }) + } + } +} + +func TestMainLimitsWorkflowRootFlagsAndHelp(t *testing.T) { + for _, trailing := range []bool{false, true} { + t.Run(fmt.Sprintf("trailing=%t", trailing), func(t *testing.T) { + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.Header.Get("OpenAI-Organization") != "org_header" || r.Header.Get("OpenAI-Project") != "proj_header" || r.Header.Get("X-Limits-Test") != "synthetic" { + t.Error("root context or custom headers changed") + } + return limitsWorkflowSuccess(t, true, r) + }) + flags := []string{"--format", "json", "--organization", "org_header", "--project", "proj_header", "-H", "X-Limits-Test: synthetic"} + args := limitsWorkflowArgs(true, true) + if trailing { + args = append(args, flags...) + } else { + args = append(flags, args...) + } + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" || gjson.Get(got.stdout, "project_id").Str != "proj_synthetic" { + t.Fatalf("root flags changed project route or output: %+v", got) + } + }) + } + for _, args := range [][]string{ + {"admin", "limits", "--help"}, + {"admin", "limits", "show", "--help"}, + {"projects", "limits", "--help"}, + {"projects", "limits", "show", "--help"}, + } { + t.Run(strings.Join(args, "/"), func(t *testing.T) { + got := runMainDispatch(t, "bash", append([]string{"openai"}, args...)...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "limits") || !strings.Contains(got.stdout, "--max-items") { + t.Fatalf("limits help unavailable: %+v", got) + } + }) + } +} + +func TestMainLimitsWorkflowRequiresAdminCredentials(t *testing.T) { + for _, project := range []bool{false, true} { + t.Run(fmt.Sprintf("project=%t", project), func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(server.Close) + args := append([]string{"openai", "--base-url", "https://api.openai.com/v1"}, limitsWorkflowArgs(project, true)...) + got := runMainDispatchWithEnv(t, "bash", adminCredentialsProxyEnv(server.URL, []string{"OPENAI_API_KEY=synthetic-project-key"}), args...) + if got.code != 1 || got.stdout != "" || strings.TrimSpace(got.stderr) != missingAdminCredentialsMessage || requests.Load() != 0 { + t.Fatalf("workspace bypassed admin credential validation: result=%+v requests=%d", got, requests.Load()) + } + }) + } +} + +func TestMainLimitsWorkflowCostsPrecisionAndPartialFailure(t *testing.T) { + for _, failNext := range []bool{false, true} { + t.Run(fmt.Sprintf("partial=%t", failNext), func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.URL.Path != "/organization/costs" { + return limitsWorkflowSuccess(t, true, r) + } + query := r.URL.Query() + if query.Get("project_ids[]") != "proj_synthetic" { + t.Errorf("Costs lost the selected project: %s", r.URL.RawQuery) + } + if query.Get("page") == "next_costs" { + if failNext { + return http.StatusServiceUnavailable, `{"error":{"message":"private synthetic costs failure"}}` + } + return http.StatusOK, `{"object":"page","data":[],"has_more":false,"next_page":null}` + } + start, err := strconv.ParseInt(query.Get("start_time"), 10, 64) + end, endErr := strconv.ParseInt(query.Get("end_time"), 10, 64) + if err != nil || endErr != nil || start%86400 != 0 || end%86400 != 0 || end <= start { + t.Fatalf("Costs period is not explicit UTC days: %s", r.URL.RawQuery) + } + return http.StatusOK, fmt.Sprintf(`{"object":"page","data":[{"object":"bucket","start_time":%d,"end_time":%d,"results":[{"object":"organization.costs.result","amount":{"value":0.100000000000000001,"currency":"usd"},"project_id":"proj_synthetic","future":9007199254740993},{"object":"organization.costs.result","amount":{"value":0.2,"currency":"usd"},"project_id":"proj_synthetic"}]}],"has_more":true,"next_page":"next_costs"}`, start, start+86400) + }) + got := runSpendThresholdCommand(t, server, "projects", "limits", "show", "--project-id", "proj_synthetic", "--format", "json") + value := gjson.Parse(got.stdout) + if !json.Valid([]byte(got.stdout)) || requests.count("/organization/costs") != 2 { + t.Fatalf("Costs pagination failed: %+v", got) + } + if value.Get("costs.buckets.0.results.0.amount.value").Raw != "0.100000000000000001" || value.Get("costs.buckets.0.results.0.future").Raw != "9007199254740993" { + t.Fatalf("Costs raw data lost precision: %s", got.stdout) + } + if failNext { + if got.code != 1 || value.Get("costs.complete").Bool() || value.Get("costs.total").Str != "0.300000000000000001" || value.Get("source_errors.#").Int() != 1 { + t.Fatalf("Partial Costs became a complete total: %+v", got) + } + if strings.Contains(got.stderr, "private synthetic") || !json.Valid([]byte(got.stderr)) { + t.Fatalf("Costs recovery exposed source data or broke JSON: %s", got.stderr) + } + } else if got.code != 0 || got.stderr != "" || !value.Get("costs.complete").Bool() || value.Get("costs.total").Str != "0.300000000000000001" { + t.Fatalf("Costs total is not exact: %+v", got) + } + if !value.Get("spend_limit").IsObject() || value.Get("spend_alerts.#").Int() != 1 { + t.Fatalf("Costs handling lost independent sources: %s", got.stdout) + } + }) + } +} diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go new file mode 100644 index 00000000..bef45044 --- /dev/null +++ b/cmd/openai/limits_workflow_unix_test.go @@ -0,0 +1,56 @@ +//go:build !windows + +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "os/exec" + "strings" + "testing" + "time" +) + +func TestMainLimitsWorkflowPartialFailureSurvivesClosedPipe(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.URL.Path == "/organization/spend_alerts" { + return http.StatusForbidden, `{"error":{"message":"private synthetic source failure"}}` + } + status, body := limitsWorkflowSuccess(t, false, r) + if r.URL.Path == "/organization/spend_limit" { + body = strings.TrimSuffix(body, "}") + `,"future":"` + strings.Repeat("synthetic", 2<<20) + `"}` + } + return status, body + }) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + child := batchesWorkflowProcess(t, ctx, server, "admin", "limits", "show", "--format", "json") + child.Env = append(child.Env, "OPENAI_ADMIN_KEY=sk-fake-spend-admin") + stdout, err := child.StdoutPipe() + if err != nil { + t.Fatal(err) + } + defer stdout.Close() + var stderr bytes.Buffer + child.Stderr = &stderr + if err := child.Start(); err != nil { + t.Fatal(err) + } + if _, err := io.ReadFull(stdout, make([]byte, 1)); err != nil { + t.Fatal(err) + } + if err := stdout.Close(); err != nil { + t.Fatal(err) + } + err = child.Wait() + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 || ctx.Err() != nil || requests.count("/organization/spend_alerts") != 1 { + t.Fatalf("source failure disappeared after EPIPE: error=%v context=%v stderr=%q", err, ctx.Err(), stderr.String()) + } + if !json.Valid(stderr.Bytes()) || !strings.Contains(stderr.String(), "overview is incomplete") || strings.Contains(stderr.String(), "private synthetic") { + t.Fatalf("safe structured recovery missing: %q", stderr.String()) + } +} diff --git a/cmd/openai/spend_error_recovery_test.go b/cmd/openai/spend_error_recovery_test.go new file mode 100644 index 00000000..069135a7 --- /dev/null +++ b/cmd/openai/spend_error_recovery_test.go @@ -0,0 +1,384 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" +) + +func spendRecoveryEnvironment(t *testing.T, endpoint string) []string { + t.Helper() + home := t.TempDir() + return []string{ + "HOME=" + home, "USERPROFILE=" + home, "XDG_CONFIG_HOME=" + home, + "OPENAI_BASE_URL=" + endpoint, "OPENAI_API_KEY=sk-fake-spend-project", + "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "OPENAI_ORG_ID=org_synthetic_recovery", + "OPENAI_PROJECT_ID=proj_synthetic_context", "OPENAI_CUSTOM_HEADERS=X-Spend-Recovery: retained", + "NO_COLOR=1", "FORCE_COLOR=0", "CI=1", + } +} + +func spendRecoverySetEnvironment(env []string, key, value string) []string { + result := slices.Clone(env) + for i, entry := range result { + if strings.HasPrefix(entry, key+"=") { + result[i] = key + "=" + value + return result + } + } + return append(result, key+"="+value) +} + +func checkSpendRecoveryContext(t *testing.T, r *http.Request, adminKey string) { + t.Helper() + if r.Header.Get("Authorization") != "Bearer "+adminKey || + r.Header.Get("OpenAI-Organization") != "org_synthetic_recovery" || + r.Header.Get("OpenAI-Project") != "proj_synthetic_context" || + r.Header.Get("X-Spend-Recovery") != "retained" { + t.Error("recovery changed the original synthetic request context") + } +} + +func writeSpendRecoveryFailure(w http.ResponseWriter, status int, code string) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Should-Retry", "false") + w.Header().Set("X-Request-Id", "req_synthetic_recovery") + w.WriteHeader(status) + fmt.Fprintf(w, `{"error":%s}`, spendRecoveryDetails(code)) +} + +func spendRecoveryDetails(code string) string { + return `{"message":"synthetic-private-response\u001b]52;c;synthetic-private\u0007","type":"invalid_request_error","code":"` + code + `","param":null,"future":{"sequence":9007199254740993}}` +} + +func checkSpendRecoveryFailure(t *testing.T, got mainDispatchResult, wants ...string) { + t.Helper() + if got.code != 1 || got.stdout != "" { + t.Fatalf("failure changed exit or output routing: %+v", got) + } + for _, want := range wants { + if !strings.Contains(got.stderr, want) { + t.Errorf("missing recovery instruction %q: %q", want, got.stderr) + } + } + for _, private := range []string{"synthetic-private", "sk-fake-", "\x1b", "\a"} { + if strings.Contains(got.stderr, private) { + t.Errorf("readable recovery exposed synthetic private data: %q", got.stderr) + } + } +} + +// These fixtures invoke the public entrypoint with the literal openai identity. +// Follow its displayed help, rather than maintaining a second recovery command. +func followSpendRecoveryHelp(t *testing.T, env []string, diagnostic, label string) mainDispatchResult { + t.Helper() + for _, line := range strings.Split(diagnostic, "\n") { + command, ok := strings.CutPrefix(line, label+": ") + if !ok { + continue + } + if !strings.HasPrefix(command, "openai help ") { + t.Fatalf("unexpected recovery command: %q", command) + } + got := runMainDispatchWithEnv(t, "bash", env, strings.Fields(command)...) + if got.code != 0 || got.stderr != "" || got.stdout == "" { + t.Fatalf("displayed recovery help failed: %+v", got) + } + return got + } + t.Fatalf("missing %s recovery line: %q", label, diagnostic) + return mainDispatchResult{} +} + +func TestMainSpendErrorRecoveryAdminSetup(t *testing.T) { + for _, args := range [][]string{ + {"admin", "organization", "spend-limit", "retrieve"}, + {"admin", "spend-limit", "retrieve"}, + {"admin:organization:spend-limit", "retrieve"}, + {"projects", "spend-limit", "retrieve", "--project-id", "proj_synthetic"}, + } { + t.Run(strings.Join(args, "/"), func(t *testing.T) { + var requests atomic.Int32 + project := args[0] == "projects" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempt := requests.Add(1) + if r.Method != http.MethodGet || r.URL.Path != spendThresholdPath(project, "spend-limit") { + t.Error("unexpected authentication recovery request") + } + if attempt == 1 { + checkSpendRecoveryContext(t, r, "sk-fake-rejected-admin") + writeSpendRecoveryFailure(w, http.StatusUnauthorized, "invalid_api_key") + return + } + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(project, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", "sk-fake-rejected-admin") + argv := append([]string{"openai"}, args...) + got := runMainDispatchWithEnv(t, "bash", env, argv...) + checkSpendRecoveryFailure(t, got, "HTTP 401", "OPENAI_ADMIN_KEY", "help setup admin") + help := followSpendRecoveryHelp(t, env, got.stderr, "Key setup") + if !strings.Contains(help.stdout, "organization/admin-keys") || !strings.Contains(help.stdout, "OPENAI_ADMIN_KEY") || requests.Load() != 1 { + t.Fatal("admin setup recovery omitted admin instructions or performed a request") + } + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", "sk-fake-spend-admin") + recovered := runMainDispatchWithEnv(t, "bash", env, argv...) + if recovered.code != 0 || recovered.stderr != "" || requests.Load() != 2 { + t.Fatalf("following admin setup did not recover the fixture: %+v", recovered) + } + }) + } +} + +func TestMainSpendErrorRecoveryNotFound(t *testing.T) { + for _, test := range []struct { + name, resource string + project, missingProject bool + }{ + {"organization limit", "spend-limit", false, false}, + {"project missing", "spend-limit", true, true}, + {"project limit missing", "spend-limit", true, false}, + {"organization alert", "spend-alerts", false, false}, + {"project alert", "spend-alerts", true, false}, + } { + t.Run(test.name, func(t *testing.T) { + var requests, writes atomic.Int32 + var configured atomic.Bool + path := spendThresholdPath(test.project, test.resource) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPost && r.URL.Path == path && test.name == "organization limit": + writes.Add(1) + configured.Store(true) + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + case r.Method == http.MethodGet && r.URL.Path == path && test.resource == "spend-alerts": + alert := spendThresholdBody(test.project, "spend-alerts", `{"id":"alert_recovered","threshold_amount":10000,"currency":"USD","interval":"month"}`) + fmt.Fprintf(w, `{"object":"list","data":[%s],"has_more":false}`, alert) + case r.Method == http.MethodGet && r.URL.Path == "/organization/projects/proj_synthetic": + if test.missingProject { + writeSpendRecoveryFailure(w, 404, "not_found") + } else { + io.WriteString(w, `{"id":"proj_synthetic","object":"organization.project","name":"Synthetic project","status":"active"}`) + } + case r.Method == http.MethodGet && r.URL.Path == path && configured.Load(): + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + default: + writeSpendRecoveryFailure(w, 404, "not_found") + } + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + args := spendThresholdArgs(test.project, test.resource, "retrieve") + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai"}, args...)...) + checkSpendRecoveryFailure(t, got, "HTTP 404", "Keep the original API address") + label := "Limit options" + if test.project && test.resource == "spend-limit" { + label = "Project lookup" + if !strings.Contains(got.stderr, "If the project exists") { + t.Error("project404 lost missing-project versus missing-limit uncertainty") + } + } else if test.resource == "spend-alerts" { + label = "List options" + } else if !strings.Contains(got.stderr, "may not be configured") { + t.Error("organization404 falsely classified an unknown server state") + } + followSpendRecoveryHelp(t, env, got.stderr, label) + if requests.Load() != 1 || writes.Load() != 0 { + t.Fatal("recovery help performed a request") + } + var read []string + if test.resource == "spend-alerts" { + read = spendThresholdArgs(test.project, test.resource, "list") + } else if test.project { + read = []string{"admin", "organization", "projects", "retrieve", "--project-id", "proj_synthetic"} + } else { + update := []string{"openai", "admin", "organization", "spend-limit", "update", "--currency", "USD", "--interval", "month", "--threshold-amount", "10000"} + if saved := runMainDispatchWithEnv(t, "bash", env, update...); saved.code != 0 { + t.Fatalf("documented limit configuration failed: %+v", saved) + } + read = args + } + recovered := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai"}, read...)...) + wantExit := 0 + if test.missingProject { + wantExit = 1 + } + if recovered.code != wantExit { + t.Fatalf("documented recovery did not identify the fixture state: %+v", recovered) + } + if test.resource == "spend-alerts" && !strings.Contains(recovered.stdout, "alert_recovered") { + t.Error("alert recovery did not expose the available alert ID") + } + }) + } +} + +func TestMainSpendErrorRecoveryOrganizationCap(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodGet { + t.Error("cap recovery attempted a mutation") + } + if r.URL.Path == "/models/synthetic-model" { + checkSpendRecoveryContext(t, r, "sk-fake-spend-project") + writeSpendRecoveryFailure(w, 429, "organization_spend_limit_exceeded") + return + } + if r.URL.Path != "/organization/spend_limit" { + t.Error("cap recovery selected the wrong scope") + } + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + })) + defer server.Close() + env := spendRecoverySetEnvironment(spendRecoveryEnvironment(t, server.URL), "OPENAI_ADMIN_KEY", "") + got := runMainDispatchWithEnv(t, "bash", env, "openai", "models", "retrieve", "--model", "synthetic-model") + checkSpendRecoveryFailure(t, got, "HTTP 429", "organization spend limit", "admin API key", "Keep the original API address") + if strings.Contains(got.stderr, "project's billing") || strings.Contains(got.stderr, "Wait before trying") { + t.Error("organization cap used project-billing or rate-throttling guidance") + } + followSpendRecoveryHelp(t, env, got.stderr, "Limit lookup") + followSpendRecoveryHelp(t, env, got.stderr, "Admin key setup") + if requests.Load() != 1 { + t.Fatal("cap recovery help performed a request") + } + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", "sk-fake-spend-admin") + read := runMainDispatchWithEnv(t, "bash", env, "openai", "admin", "organization", "spend-limit", "retrieve") + if read.code != 0 || read.stderr != "" || requests.Load() != 2 { + t.Fatalf("cap recovery failed with retained context and an admin key: %+v", read) + } +} + +func TestMainSpendErrorRecoveryMachineContracts(t *testing.T) { + for _, test := range []struct { + status int + code string + }{ + {401, "invalid_api_key"}, {404, "not_found"}, {429, "organization_spend_limit_exceeded"}, + } { + t.Run(fmt.Sprint(test.status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeSpendRecoveryFailure(w, test.status, test.code) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + args := []string{"admin", "organization", "spend-limit", "retrieve"} + for _, flag := range []string{"--format", "--format-error"} { + for _, format := range []string{"json", "jsonl", "raw", "yaml"} { + argv := append([]string{"openai", flag, format}, args...) + got := runMainDispatchWithEnv(t, "bash", env, argv...) + if got.code != 1 || got.stdout != "" { + t.Fatalf("machine error changed exit or output routing: %+v", got) + } + if format == "yaml" { + want := "message: \"synthetic-private-response\\x1B]52;c;synthetic-private\\x07\"\n" + + "type: invalid_request_error\ncode: " + test.code + "\nparam: null\nfuture:\n sequence: 9007199254740993\n" + if got.stderr != want { + t.Errorf("YAML error bytes changed: %q", got.stderr) + } + } else { + assertSpendThresholdJSON(t, got.stderr, spendRecoveryDetails(test.code)) + if (format == "raw" || format == "jsonl") && got.stderr != spendRecoveryDetails(test.code)+"\n" { + t.Errorf("%s error bytes changed", format) + } + } + } + } + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format-error", "text", "--transform-error", "future.sequence"}, args...)...) + if got.code != 1 || got.stdout != "" || got.stderr != "9007199254740993\n" { + t.Fatalf("error extraction changed: %+v", got) + } + }) + } +} + +func TestMainSpendErrorRecoveryLostAndTruncatedResponse(t *testing.T) { + for _, truncated := range []bool{false, true} { + t.Run(fmt.Sprintf("truncated=%t", truncated), func(t *testing.T) { + var writes, reads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + if r.URL.Path != "/organization/spend_limit" { + t.Error("unexpected mutation recovery route") + } + if r.Method == http.MethodGet { + reads.Add(1) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + return + } + if r.Method != http.MethodPost { + t.Error("unexpected mutation method") + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["threshold_amount"] != float64(10000) { + t.Error("mutation body changed") + } + writes.Add(1) + connection, buffer, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer connection.Close() + if truncated { + io.WriteString(buffer, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000\r\nConnection: close\r\n\r\n{\"object\":\"organization.spend_limit\"") + buffer.Flush() + } + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + mutation := []string{"admin", "organization", "spend-limit", "update", "--currency", "USD", "--interval", "month", "--threshold-amount", "10000"} + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai"}, mutation...)...) + checkSpendRecoveryFailure(t, got, "The API may have received the change", "before repeating the command") + wantAttempts := int32(3) + if truncated { + wantAttempts = 1 + } + if writes.Load() != wantAttempts { + t.Errorf("request retry behavior changed: got %d attempts, want %d", writes.Load(), wantAttempts) + } + followSpendRecoveryHelp(t, env, got.stderr, "Read options") + if writes.Load() != wantAttempts || reads.Load() != 0 { + t.Fatal("response-loss recovery help performed a request") + } + read := runMainDispatchWithEnv(t, "bash", env, "openai", "admin", "organization", "spend-limit", "retrieve") + if read.code != 0 || read.stderr != "" || writes.Load() != wantAttempts || reads.Load() != 1 { + t.Fatalf("recovery did not inspect state without another mutation: %+v", read) + } + baseline := "Could not connect to the API. Check your connection, proxy, and --base-url setting." + if truncated { + baseline = "The command could not be completed. Check your arguments with --help." + } + for _, flags := range [][]string{{"--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { + argv := append(append([]string{"openai"}, flags...), mutation...) + machine := runMainDispatchWithEnv(t, "bash", env, argv...) + if machine.code != 1 || machine.stdout != "" { + t.Fatalf("machine transport error changed exit or output routing: %+v", machine) + } + if flags[1] == "json" { + var value map[string]string + if err := json.Unmarshal([]byte(machine.stderr), &value); err != nil || len(value) != 1 || value["message"] != baseline { + t.Errorf("machine transport fallback changed: %q", machine.stderr) + } + } else if machine.stderr != baseline+"\n" { + t.Errorf("transport extraction changed: %q", machine.stderr) + } + } + }) + } +} diff --git a/docs/limits-workspace.md b/docs/limits-workspace.md new file mode 100644 index 00000000..51d3e35d --- /dev/null +++ b/docs/limits-workspace.md @@ -0,0 +1,135 @@ +# Limits in the terminal + +The limits workspace manages the controls exposed by the public administration API. +It requires an admin API key through the existing CLI authentication settings. + +```sh +openai admin limits +openai projects limits --project-id proj_example +``` + +An interactive terminal opens the workspace. +Pipes, CI, quiet mode, explicit output formats, and extraction print a read-only overview. +The `show` command always prints the overview. + +```sh +openai admin limits show +openai projects limits show --project-id proj_example --format json +openai admin limits show --max-items -1 +openai admin limits show --format json --transform spend_alerts +``` + +`--max-items` defaults to 100 records per list. +Use `-1` for all records or `0` to validate the list request without retaining records. +The overview reports whether each list is complete. +Failed reads preserve available data and return a nonzero exit status. +Cancellation can stop output before the CLI prints those records. +The `source_errors` field identifies failed sources without exposing response error bodies. +Machine formats and extraction retain original resource fields and numeric values. + +## Current spend + +The overview includes reported current-month Costs with UTC daily boundaries. +Today’s costs remain partial and can arrive later. +Costs amounts use currency units; spend-limit amounts use integer cents. +The CLI preserves exact decimal totals and original buckets. +Incomplete Costs totals cover only returned buckets. +Check `costs.complete` before treating a total as the complete reported period. +Ambiguous Costs data produces no total. +Costs data cannot establish authoritative enforcement state. +The derived summary supports up to 10,000 decimal digits. +Values beyond that precision remain available in `costs.buckets`; the summary reports its limitation. + +## Workspace controls + +| Control | Action | +| --- | --- | +| Left / Right | Select Spend, Alerts, or project Rates | +| Up / Down | Select a record or scroll a review | +| Enter | Open the selected action | +| Tab / Shift+Tab | Move through form fields or confirmation actions | +| r | Refresh the overview or a rejected draft | +| i | Inspect the selected original API record | +| ? | Show workspace help and API boundaries | +| d | Delete the selected alert | +| Escape | Return without saving | +| q | Close the overview | +| Ctrl+C | Interrupt from any state | + +The workspace needs at least 40 columns and 12 rows. +Smaller terminals retain cancellation and block submission. +The workspace restores the terminal screen when it exits. + +Every change opens a before-and-after review. +Scroll to the end before selecting Apply. +Back is the initial selection, including after resizing. +The review shows a shell command when the CLI can reproduce the request context safely. +It omits command previews that require private options. +Existing administration commands remain available for scripts. + +```sh +# Existing scriptable commands use integer cents. +openai admin spend-limit update --currency USD --interval month --threshold-amount 12550 +openai admin spend-alerts list --max-items -1 +openai projects rate-limits list-rate-limits --project-id proj_example +openai admin usage costs --help +``` + +## Supported changes + +Spend supports setting, replacing, and removing hard monthly USD limits. +Enter decimal dollars, such as `125.50`. +The CLI converts dollars to integer cents without floating-point arithmetic. +A hard limit must be at least USD 0.01. +Enforcement can be delayed after the threshold is reached. +Removing a hard limit removes enforcement. +Use the Platform website to inspect or edit any retained soft budget. + +Alerts support creation, editing, and deletion. +Alerts notify recipients; they do not enforce a cap. +Enter recipients as comma-separated email addresses. +Subject prefixes can preserve the original value, replace it, or clear it to null. +An empty replacement remains an empty string. +The workspace does not require a hard limit before creating an alert. +The API still validates account-specific constraints. + +Project Rates supports all six public rate fields. +The form sends only changed fields. +Blank fields remain unchanged. +Zero has no assumed unlimited meaning. +The API decides which integer values the project permits. +Rate updates require a complete preflight list; use `--max-items -1` when necessary. + +## Concurrent changes and recovery + +The CLI checks the selected record again before writing. +This check is not atomic and cannot prevent every concurrent change. +A changed baseline stops the write. +A 404 response leaves the current limit unconfirmed. +Setting a limit requires reviewing that uncertainty and explicitly approving creation or replacement. +Refresh the retained draft before reviewing it again. +Ambiguous or unfamiliar replacement configuration remains available for inspection. +The form refuses edits that could discard that configuration. + +The workspace never automatically retries a mutation. +Scriptable API commands retain their existing retry behavior. +A command preview does not reproduce workspace preflight checks or retry suppression. +An uncertain save locks further writes in that workspace session. +Inspect the resource before reopening the workspace or repeating the action. +A refresh alone cannot prove that an uncertain alert creation failed. +Cancellation does not prove that the server canceled a dispatched write. + +## Platform controls without matching contracts + +The [Platform limits page](https://platform.openai.com/settings/organization/limits) includes controls beyond these public operations. +The inspected public contract does not expose the website's soft-budget and enforcement-toggle combination. +It also lacks the organization tier catalog, account upgrade operations, and approved usage-limit controls. +The workspace does not emulate these controls through undocumented requests. + +Project rate records do not describe every organization-wide model limit or shared model-family limit. +Historical costs are not a live allowance or enforcement counter. +The documented fine-tuning model-limits endpoint is absent from the pinned generated schema and SDK. +That endpoint needs authoritative schema and generator support before CLI integration. + +See the [spend-limits guide](https://developers.openai.com/api/docs/guides/spend-limits) +and [rate-limits guide](https://developers.openai.com/api/docs/guides/rate-limits) for API behavior. diff --git a/internal/output/transform/project_limits_overview.go b/internal/output/transform/project_limits_overview.go new file mode 100644 index 00000000..1d0ee0be --- /dev/null +++ b/internal/output/transform/project_limits_overview.go @@ -0,0 +1,166 @@ +package transform + +import ( + "context" + "strings" + + "github.com/tidwall/gjson" +) + +// ProjectLimitsOverview adds units to recognized records in a limits snapshot. +// The caller selects readable output; machine formats keep the original snapshot. +func ProjectLimitsOverview(ctx context.Context, value gjson.Result) (gjson.Result, error) { + unique, err := limitsOverviewUniqueObject(ctx, value) + if err != nil || !unique { + return value, err + } + scope := value.Get("scope") + if scope.Type != gjson.String || scope.Str != "organization" && scope.Str != "project" { + return value, nil + } + var out strings.Builder + out.WriteByte('{') + first := true + value.ForEach(func(key, field gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + projected := field + switch key.Str { + case "spend_limit": + projected, err = projectSpendThreshold(ctx, field, scope.Str+".spend_limit", false) + case "spend_alerts": + projected, err = projectLimitsOverviewArray(ctx, field, func(item gjson.Result) (gjson.Result, error) { + return projectSpendThreshold(ctx, item, scope.Str+".spend_alert", true) + }) + case "project_rate_limits": + if scope.Str == "project" { + projected, err = projectLimitsOverviewArray(ctx, field, func(item gjson.Result) (gjson.Result, error) { + return projectLimitsOverviewRate(ctx, item) + }) + } + } + if err != nil { + return false + } + if !first { + out.WriteByte(',') + } + first = false + out.WriteString(key.Raw) + out.WriteByte(':') + out.WriteString(projected.Raw) + return true + }) + if err != nil { + return gjson.Result{}, err + } + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + out.WriteByte('}') + return gjson.Parse(out.String()), nil +} + +func projectLimitsOverviewArray(ctx context.Context, value gjson.Result, project func(gjson.Result) (gjson.Result, error)) (gjson.Result, error) { + if !value.IsArray() { + return value, ctx.Err() + } + var out strings.Builder + out.WriteByte('[') + first := true + var err error + value.ForEach(func(_, item gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + var projected gjson.Result + projected, err = project(item) + if err != nil { + return false + } + if !first { + out.WriteByte(',') + } + first = false + out.WriteString(projected.Raw) + return true + }) + if err != nil { + return gjson.Result{}, err + } + out.WriteByte(']') + return gjson.Parse(out.String()), ctx.Err() +} + +var limitsOverviewRateLabels = map[string]string{ + "max_requests_per_1_minute": "max_requests_per_minute", + "max_tokens_per_1_minute": "max_tokens_per_minute", + "max_images_per_1_minute": "max_images_per_minute", + "max_audio_megabytes_per_1_minute": "max_audio_megabytes_per_minute", + "max_requests_per_1_day": "max_requests_per_day", + "batch_1_day_max_input_tokens": "max_batch_input_tokens_per_day", +} + +func projectLimitsOverviewRate(ctx context.Context, value gjson.Result) (gjson.Result, error) { + unique, err := limitsOverviewUniqueObject(ctx, value) + if err != nil || !unique || value.Get("object").Str != "project.rate_limit" { + return value, err + } + // A future response field must never collide with a derived readable label. + for source, label := range limitsOverviewRateLabels { + if value.Get(source).Type == gjson.Number && value.Get(label).Exists() { + return value, nil + } + } + var out strings.Builder + out.WriteByte('{') + first := true + value.ForEach(func(key, field gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + if !first { + out.WriteByte(',') + } + first = false + if label, ok := limitsOverviewRateLabels[key.Str]; ok && field.Type == gjson.Number { + out.WriteByte('"') + out.WriteString(label) + out.WriteByte('"') + } else { + out.WriteString(key.Raw) + } + out.WriteByte(':') + out.WriteString(field.Raw) + return true + }) + if err != nil { + return gjson.Result{}, err + } + out.WriteByte('}') + return gjson.Parse(out.String()), ctx.Err() +} + +func limitsOverviewUniqueObject(ctx context.Context, value gjson.Result) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + if !value.IsObject() { + return false, nil + } + seen := make(map[string]bool) + unique := true + value.ForEach(func(key, _ gjson.Result) bool { + if ctx.Err() != nil { + return false + } + if seen[key.Str] { + unique = false + return false + } + seen[key.Str] = true + return true + }) + return unique, ctx.Err() +} diff --git a/internal/output/transform/project_limits_overview_test.go b/internal/output/transform/project_limits_overview_test.go new file mode 100644 index 00000000..cadc2d95 --- /dev/null +++ b/internal/output/transform/project_limits_overview_test.go @@ -0,0 +1,127 @@ +package transform + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/tidwall/gjson" +) + +func TestProjectLimitsOverviewSpendScopes(t *testing.T) { + for _, scope := range []string{"organization", "project"} { + t.Run(scope, func(t *testing.T) { + raw := `{"scope":"` + scope + `","spend_limit":{"object":"` + scope + `.spend_limit","threshold_amount":9007199254740993,"currency":"USD","interval":"month","enforcement":{"status":"future_status"},"unknown":0.100000000000000001},"spend_limit_configured":true,"spend_alerts":[{"object":"` + scope + `.spend_alert","id":"alert_synthetic","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"recipients":["synthetic@example.invalid"],"subject_prefix":null}}],"spend_alerts_complete":false,"source_errors":[],"future":{"value":9007199254740993123456789}}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || !gjson.Valid(value.Raw) { + t.Fatalf("invalid overview: %s; %v", value.Raw, err) + } + if value.Get("spend_limit.spend_threshold").Str != "USD 90071992547409.93 per month" || value.Get("spend_alerts.0.spend_threshold").Str != "USD 0.00 per month" { + t.Fatalf("incorrect spend units: %s", value.Raw) + } + if value.Get("spend_alerts.0.alert_behavior").Str != "Alerts notify; they are not spending caps." { + t.Fatalf("missing alert behavior: %s", value.Raw) + } + for _, field := range []string{"scope", "spend_limit_configured", "spend_alerts_complete", "source_errors", "future", "spend_limit.enforcement", "spend_limit.unknown", "spend_alerts.0.notification_channel"} { + if got, want := value.Get(field).Raw, gjson.Get(raw, field).Raw; got != want { + t.Errorf("%s changed: %q; want %q", field, got, want) + } + } + }) + } +} + +func TestProjectLimitsOverviewExactRateUnits(t *testing.T) { + const rate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993123456789,"max_images_per_1_minute":-1,"max_audio_megabytes_per_1_minute":0.1234567890123456789,"max_requests_per_1_day":1e9,"batch_1_day_max_input_tokens":9223372036854775807,"future":{"count":9007199254740993}}` + raw := `{"scope":"project","project_id":"proj_synthetic","project":{"id":"proj_synthetic","future":1e100},"spend_limit":null,"spend_limit_configured":false,"spend_alerts":[],"spend_alerts_complete":true,"project_rate_limits":[` + rate + `],"project_rate_limits_complete":true,"source_errors":[]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil { + t.Fatal(err) + } + projected := value.Get("project_rate_limits.0") + for label, want := range map[string]string{ + "max_requests_per_minute": "0", "max_tokens_per_minute": "9007199254740993123456789", + "max_images_per_minute": "-1", "max_audio_megabytes_per_minute": "0.1234567890123456789", + "max_requests_per_day": "1e9", "max_batch_input_tokens_per_day": "9223372036854775807", + } { + if projected.Get(label).Raw != want { + t.Errorf("rate %s changed units or precision: %s", label, projected.Raw) + } + } + for _, field := range []string{"object", "id", "model", "future"} { + if projected.Get(field).Raw != gjson.Get(rate, field).Raw { + t.Errorf("rate field %s changed: %s", field, projected.Raw) + } + } + for _, field := range []string{"project_id", "project", "spend_limit", "spend_limit_configured", "spend_alerts", "spend_alerts_complete", "project_rate_limits_complete", "source_errors"} { + if value.Get(field).Raw != gjson.Get(raw, field).Raw { + t.Errorf("overview field %s changed: %s", field, value.Raw) + } + } +} + +func TestProjectLimitsOverviewRatePresenceAndAmbiguity(t *testing.T) { + for _, raw := range []string{ + `{"object":"project.rate_limit","id":"rl_synthetic"}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":null,"max_tokens_per_1_minute":"100"}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_minute":null}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_1_minute":900}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_1_\u006dinute":900}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"future":1,"future":2}`, + `{"object":"future.rate_limit","max_requests_per_1_minute":100}`, + `{"max_requests_per_1_minute":100}`, + `null`, `[]`, `"future"`, + } { + t.Run(raw, func(t *testing.T) { + value, err := projectLimitsOverviewRate(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("ambiguous or unknown rate changed: %s; %v", value.Raw, err) + } + }) + } +} + +func TestProjectLimitsOverviewUnrecognizedEnvelope(t *testing.T) { + for _, raw := range []string{ + `null`, `[]`, `{"scope":"future","spend_limit":{}}`, + `{"scope":"project","scope":"organization","spend_limit":{}}`, + `{"scope":"project","future":1,"future":2}`, + `{"scope":"project","spend_alerts":[],"spend_\u0061lerts":null}`, + `{"scope":null}`, `{"scope":{}}`, + } { + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("unrecognized envelope changed: %s; %v", value.Raw, err) + } + } + const raw = `{"scope":"organization","spend_limit":null,"spend_limit_configured":null,"spend_alerts":null,"project_rate_limits":[{"object":"project.rate_limit","max_requests_per_1_minute":7}],"source_errors":[{"resource":"spend_limit","status":403}]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("absent data or project-only rate changed: %s; %v", value.Raw, err) + } +} + +func TestProjectLimitsOverviewKeepsAmbiguousSpendRecords(t *testing.T) { + const limit = `{"object":"project.spend_limit","threshold_amount":10000,"threshold_amount":90000,"currency":"USD","interval":"month"}` + const alert = `{"object":"project.spend_alert","threshold_amount":10000,"currency":"USD","currency":"EUR","interval":"month"}` + raw := `{"scope":"project","spend_limit":` + limit + `,"spend_alerts":[` + alert + `,null,{"object":"future.spend_alert","threshold_amount":10000}]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("ambiguous spend or unknown alert changed: %s; %v", value.Raw, err) + } +} + +func TestProjectLimitsOverviewCancellationAndLargeUnknownField(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := ProjectLimitsOverview(ctx, gjson.Parse(`{"scope":"project"}`)); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + unknown := strings.Repeat("x", 1<<20) + raw := `{"scope":"project","project_rate_limits":[{"object":"project.rate_limit","max_requests_per_1_minute":1,"future":"` + unknown + `"}]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Get("project_rate_limits.0.future").Str != unknown || value.Get("project_rate_limits.0.max_requests_per_minute").Raw != "1" { + t.Fatalf("large unknown field lost: length=%d, error=%v", len(value.Raw), err) + } +} diff --git a/pkg/custom/command.go b/pkg/custom/command.go index 346be81e..8e55ab19 100644 --- a/pkg/custom/command.go +++ b/pkg/custom/command.go @@ -43,6 +43,7 @@ func ConfigureCommand(root *cli.Command) { registerAdminSetup(root) configureAdminCredentials(root) localcommands.ConfigureSubgroups(root) + registerLimitsCommands(root) localcommands.ConfigureTasks(root) registerAudioCommands(root) configureFileCommands(root) diff --git a/pkg/custom/limits_commands.go b/pkg/custom/limits_commands.go new file mode 100644 index 00000000..5fca45b6 --- /dev/null +++ b/pkg/custom/limits_commands.go @@ -0,0 +1,119 @@ +package custom + +import ( + "context" + "errors" + "os" + "strings" + + "github.com/charmbracelet/x/term" + "github.com/openai/openai-cli/internal/clihelp" + localcommands "github.com/openai/openai-cli/internal/commands" + "github.com/openai/openai-cli/internal/output/transform" + "github.com/urfave/cli/v3" +) + +// Aggregate workflows retain their causes without presenting private source errors. +type limitsWorkflowFailure struct { + message string + cause error +} + +func (failure *limitsWorkflowFailure) Error() string { return failure.message } +func (failure *limitsWorkflowFailure) Unwrap() error { return failure.cause } + +// Register before task promotion so the canonical and task routes share behavior. +func registerLimitsCommands(root *cli.Command) { + admin := root.Command("admin") + if admin == nil { + return + } + organization := admin.Command("organization") + if organization == nil { + return + } + organization.Commands = append(organization.Commands, newLimitsCommand(false)) + if projects := organization.Command("projects"); projects != nil { + projects.Commands = append(projects.Commands, newLimitsCommand(true)) + } +} + +func newLimitsCommand(project bool) *cli.Command { + example := "admin limits" + if project { + example = "projects limits --project-id proj_example" + } + flags := func() []cli.Flag { + result := []cli.Flag{&cli.Int64Flag{Name: "max-items", Value: 100, Usage: "Maximum records per list; -1 fetches all records."}} + if project { + result = append(result, &cli.StringFlag{Name: "project-id", Required: true, Usage: "Project to inspect or manage."}) + } + return result + } + return &cli.Command{ + Name: "limits", Usage: "Inspect and manage hard spend limits and spend alerts.", + Metadata: map[string]any{adminCredentialsRequiredMetadata: true, "help-content": clihelp.Content{Examples: []clihelp.Example{{Description: "Open the limits workspace:", Command: example}}}}, + Description: "Open the limits workspace in an interactive terminal. Every change requires review.\n" + + "Pipes, CI, explicit output formats, and show print a read-only overview.\n" + + "Project workspaces also manage project rate limits. Requires an admin key.\n" + + "Soft budgets, usage tiers, and organization rate catalogs require the Platform website.", + Flags: flags(), Action: limitsCommandAction(project, false), + Commands: []*cli.Command{{Name: "show", Usage: "Print a read-only limits overview.", Metadata: map[string]any{adminCredentialsRequiredMetadata: true}, Action: limitsCommandAction(project, true)}}, + } +} + +func limitsCommandAction(project, show bool) cli.ActionFunc { + return func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Present() { + return &localcommands.LocalError{Message: "Limits does not accept positional arguments. Use --help to inspect its options."} + } + backend, err := newLimitsBackend(cmd, project) + if err != nil { + return err + } + root := cmd.Root() + opts := ShowJSONOpts{Context: ctx, Format: root.String("format"), ExplicitFormat: root.IsSet("format"), + RawOutput: root.Bool("raw-output"), Transform: root.String("transform"), Stdout: root.Writer, Stderr: root.ErrWriter} + opts.setDefaults() + if !show && limitsInteractive(cmd, opts) { + return runLimitsWorkspace(ctx, cmd, backend, opts) + } + snapshot, loadErr := backend.Load(ctx) + var sourceErr error + if loadErr != nil || len(snapshot.Errors) != 0 { + causes := []error{loadErr} + for _, failure := range snapshot.Errors { + causes = append(causes, failure.Err) + } + route := "admin limits show" + if project { + route = "projects limits show" + } + message := "The limits overview is incomplete.\nThe source_errors field identifies failed sections when output is available.\n" + + "Overview options: " + clihelp.DisplayInvocation(root) + " help " + route + "\n" + + "Keep the original connection, organization, and project settings when refreshing." + causes = append(causes, &localcommands.LocalError{Message: message}) + sourceErr = &limitsWorkflowFailure{message: message, cause: errors.Join(causes...)} + } + value, err := snapshot.JSON() + if err != nil { + return errors.Join(sourceErr, ctx.Err(), err) + } + if resolvedOutputFormat(opts) == "text" && opts.Transform == "" && !opts.RawOutput { + value, err = transform.ProjectLimitsOverview(ctx, value) + if err != nil { + return errors.Join(sourceErr, ctx.Err(), err) + } + } + outputErr := ShowJSON(value, opts) + return errors.Join(sourceErr, ctx.Err(), outputErr) + } +} + +func limitsInteractive(cmd *cli.Command, opts ShowJSONOpts) bool { + format := strings.ToLower(opts.Format) + input, ok := cmd.Root().Reader.(*os.File) + return ok && term.IsTerminal(input.Fd()) && isTerminal(opts.Stdout) && + (format == "" || format == "auto") && !opts.ExplicitFormat && !opts.RawOutput && opts.Transform == "" && + !cmd.Root().Bool("quiet") && os.Getenv("CI") == "" && os.Getenv("TERM") != "dumb" +} diff --git a/pkg/custom/limits_draft.go b/pkg/custom/limits_draft.go new file mode 100644 index 00000000..b96f7e7b --- /dev/null +++ b/pkg/custom/limits_draft.go @@ -0,0 +1,309 @@ +package custom + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "charm.land/bubbles/v2/textinput" + "github.com/openai/openai-cli/internal/readable" + "github.com/tidwall/gjson" +) + +var limitsRateInputs = []struct{ key, label string }{ + {"max_requests_per_1_minute", "Requests / minute"}, + {"max_tokens_per_1_minute", "Tokens / minute"}, + {"max_requests_per_1_day", "Requests / day"}, + {"max_images_per_1_minute", "Images / minute"}, + {"max_audio_megabytes_per_1_minute", "Audio megabytes / minute"}, + {"batch_1_day_max_input_tokens", "Batch input tokens / day"}, +} + +type limitsDraft struct { + change limitsChange + labels []string + inputs []textinput.Model + initial []string + prefixMode int // Preserve, replace, or null. Empty replacement stays an empty string. +} + +func newLimitsDraft(change limitsChange) (limitsDraft, error) { + d := limitsDraft{change: change} + if change.Before.Exists() { + if err := limitsEditableBaseline(change.Before, change.Kind); err != nil { + return d, err + } + } + add := func(label, value string) { + input := textinput.New() + input.Prompt = "" + input.CharLimit = 0 + input.SetValue(value) + d.labels = append(d.labels, label) + d.inputs = append(d.inputs, input) + d.initial = append(d.initial, value) + } + switch change.Kind { + case "limit_set", "alert_create", "alert_update": + amount := "" + if change.Before.Get("threshold_amount").Exists() { + amount = formatLimitsDollars(change.Before.Get("threshold_amount").Int()) + } + add("Monthly USD", amount) + if change.Kind != "limit_set" { + var recipients []string + for _, value := range change.Before.Get("notification_channel.recipients").Array() { + recipients = append(recipients, value.String()) + } + add("Email recipients (comma separated)", strings.Join(recipients, ", ")) + add("Subject prefix", change.Before.Get("notification_channel.subject_prefix").String()) + } + case "rate_update": + for _, field := range limitsRateInputs { + value := change.Before.Get(field.key) + text := "" + if value.Type == gjson.Number { + text = value.Raw + } + add(field.label+" (blank: unchanged)", text) + } + } + for i, input := range d.inputs { + if input.Value() != d.initial[i] { + return d, errors.New("This record contains text that the form cannot preserve. Inspect the original API record.") + } + } + return d, nil +} + +func (d limitsDraft) validate() (limitsChange, error) { + c := d.change + switch c.Kind { + case "limit_set", "alert_create", "alert_update": + minimum := int64(0) + if c.Kind == "limit_set" { + minimum = 1 + } + amount, err := parseLimitsDollars(d.inputs[0].Value(), minimum) + if err != nil { + return c, err + } + c.Amount = amount + if c.Kind != "limit_set" { + c.Recipients, err = parseLimitsRecipients(d.inputs[1].Value()) + if err != nil { + return c, err + } + value := d.inputs[2].Value() + switch { + case d.prefixMode == 2: + c.SubjectPrefixSet, c.SubjectPrefix = true, nil + case d.prefixMode == 1 || value != d.initial[2]: + c.SubjectPrefixSet, c.SubjectPrefix = true, &value + default: + prefix := c.Before.Get("notification_channel.subject_prefix") + c.SubjectPrefixSet = prefix.Exists() + if prefix.Type == gjson.String { + s := prefix.String() + c.SubjectPrefix = &s + } + } + } + case "rate_update": + c.RateValues = map[string]int64{} + for i, field := range limitsRateInputs { + value := d.inputs[i].Value() + if value == "" || value == d.initial[i] { + continue + } + digits := strings.TrimPrefix(value, "-") + if digits == "" { + return c, fmt.Errorf("%s needs an integer.", field.label) + } + for _, ch := range digits { + if ch < '0' || ch > '9' { + return c, fmt.Errorf("%s needs an integer.", field.label) + } + } + number, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return c, fmt.Errorf("%s exceeds the supported integer range.", field.label) + } + c.RateValues[field.key] = number + } + if len(c.RateValues) == 0 { + return c, errors.New("Change at least one rate value.") + } + } + return c, nil +} + +func limitsChangeTitle(kind string) string { + switch kind { + case "limit_set": + return "Set hard spend limit" + case "limit_delete": + return "Remove hard spend limit" + case "alert_create": + return "Create spend alert" + case "alert_update": + return "Edit spend alert" + case "alert_delete": + return "Delete spend alert" + case "rate_update": + return "Update project rate limits" + } + return "Review change" +} + +func limitsChangeBody(c limitsChange) map[string]any { + body := map[string]any{} + if c.Kind == "limit_set" || c.Kind == "alert_create" || c.Kind == "alert_update" { + body["currency"], body["interval"], body["threshold_amount"] = "USD", "month", c.Amount + } + if c.Kind == "alert_create" || c.Kind == "alert_update" { + channel := map[string]any{"type": "email", "recipients": c.Recipients} + if c.SubjectPrefixSet { + channel["subject_prefix"] = c.SubjectPrefix + } + body["notification_channel"] = channel + } + for name, value := range c.RateValues { + body[name] = value + } + return body +} + +func limitsReview(c limitsChange, projectID string, requestArgs []string, omit bool, shell string, executable ...string) string { + scope := "Organization" + if projectID != "" { + scope = "Project " + projectID + } + before := "Current limit not confirmed" + if c.Before.Exists() { + before = c.Before.Raw + } + if c.Kind == "alert_create" { + before = "New alert" + } + after, _ := json.MarshalIndent(limitsChangeBody(c), "", " ") + result := scope + "\n" + limitsChangeTitle(c.Kind) + "\n" + if c.ID != "" { + result += "ID: " + c.ID + "\n" + } + result += "\nBefore\n" + before + "\n\nAfter\n" + if strings.HasSuffix(c.Kind, "_delete") { + result += "Remove this resource.\n" + } else { + result += string(after) + "\n" + } + switch c.Kind { + case "limit_set": + result += "\nHard limit: USD " + formatLimitsDollars(c.Amount) + " / month. Enforcement can be delayed.\n" + if c.BeforeMissing { + result += "The current limit could not be confirmed. This action creates or replaces the hard limit.\n" + } + case "limit_delete": + result += "\nThis removes the hard limit. The public API does not guarantee soft-budget behavior.\nInspect soft budgets: https://platform.openai.com/settings/organization/limits\n" + case "alert_create", "alert_update": + result += "\nAlerts notify recipients. Alerts do not enforce a spending cap.\n" + case "rate_update": + result += "\nOnly changed rate fields are sent. Zero has no assumed unlimited meaning.\n" + } + result += "\nThe CLI checks the current record before saving. This check is not atomic.\n" + result += "The workspace sends this change once, without automatic retries.\n" + result = readable.Text(result) + if omit { + return result + "\nCommand preview omitted: request context cannot be reproduced safely.\n" + } + args := append([]string{}, requestArgs...) + args = append(args, "admin", "organization") + if projectID != "" { + args = append(args, "projects") + } + switch c.Kind { + case "limit_set": + args = append(args, "spend-limit", "update") + case "limit_delete": + args = append(args, "spend-limit", "delete") + case "alert_create": + args = append(args, "spend-alerts", "create") + case "alert_update": + args = append(args, "spend-alerts", "update") + case "alert_delete": + args = append(args, "spend-alerts", "delete") + case "rate_update": + args = append(args, "rate-limits", "update-rate-limit") + } + if projectID != "" { + args = append(args, "--project-id", projectID) + } + if c.ID != "" { + flag := "--alert-id" + if c.Kind == "rate_update" { + flag = "--rate-limit-id" + } + args = append(args, flag, c.ID) + } + if c.Kind == "limit_set" || c.Kind == "alert_create" || c.Kind == "alert_update" { + args = append(args, "--currency", "USD", "--interval", "month", "--threshold-amount", strconv.FormatInt(c.Amount, 10)) + } + if c.Kind == "alert_create" || c.Kind == "alert_update" { + channelValues := limitsChangeBody(c)["notification_channel"].(map[string]any) + if c.SubjectPrefixSet && c.SubjectPrefix != nil { + prefix := *c.SubjectPrefix + if strings.HasPrefix(prefix, "\\@") { + return result + "\nCommand preview omitted: the request parser cannot preserve this literal prefix.\n" + } + if strings.HasPrefix(prefix, "@") { + channelValues["subject_prefix"] = "\\" + prefix + } + } + channel, _ := json.Marshal(channelValues) + args = append(args, "--notification-channel", string(channel)) + } + for _, field := range limitsRateInputs { + if value, ok := c.RateValues[field.key]; ok { + args = append(args, "--"+strings.ReplaceAll(field.key, "_", "-"), strconv.FormatInt(value, 10)) + } + } + command := formatImagePickerCommand(args, shell) + if command == "" { + return result + "\n" + imagePickerUnsupportedShell + "\n" + } + if len(executable) > 0 && executable[0] != "" { + command = imagePickerShellQuoter(shell)(executable[0]) + strings.TrimPrefix(command, "openai") + if shell == "pwsh" { + command = "& " + command + } + } + return result + "\nDirect API command\n" + command + "\n\nThis command skips the workspace preflight and can retry automatically.\nRecheck current settings before running it.\n" +} + +// Use the generated operation identity so successful receipts retain projections. +func limitsChangeOperation(kind string, project bool) string { + resource := "admin.organization." + if project { + resource += "projects." + } + method := "" + switch kind { + case "limit_set": + resource, method = resource+"spend_limit", "update" + case "limit_delete": + resource, method = resource+"spend_limit", "delete" + case "alert_create": + resource, method = resource+"spend_alerts", "create" + case "alert_update": + resource, method = resource+"spend_alerts", "update" + case "alert_delete": + resource, method = resource+"spend_alerts", "delete" + case "rate_update": + resource, method = resource+"rate_limits", "update_rate_limit" + default: + return "" + } + return "(resource) " + resource + " > (method) " + method +} diff --git a/pkg/custom/limits_service.go b/pkg/custom/limits_service.go new file mode 100644 index 00000000..472f73a7 --- /dev/null +++ b/pkg/custom/limits_service.go @@ -0,0 +1,733 @@ +package custom + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/openai/openai-cli/internal/apiquery" + localcommands "github.com/openai/openai-cli/internal/commands" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +type limitsBackend interface { + Load(context.Context) (limitsSnapshot, error) + Mutate(context.Context, limitsChange) (gjson.Result, error) +} + +type limitsSourceFailure struct { + Resource string + Err error +} + +type limitsCostSummaryError struct{ Reason string } + +func (*limitsCostSummaryError) Error() string { + return "Costs summary unavailable. Inspect costs.buckets or use the usage costs command." +} + +type limitsMutationFailure struct { + Err error + Attempted bool +} + +func (failure *limitsMutationFailure) Error() string { return failure.Err.Error() } +func (failure *limitsMutationFailure) Unwrap() error { return failure.Err } + +type limitsSnapshot struct { + ProjectID string + Project gjson.Result + Limit gjson.Result + LimitAbsent bool + LimitUnconfirmed bool + Alerts []gjson.Result + Rates []gjson.Result + AlertsComplete bool + RatesComplete bool + Costs []gjson.Result + CostsComplete bool + CostsTotal string + CostsStart int64 + CostsEnd int64 + Errors []limitsSourceFailure +} + +type limitsChange struct { + Kind string + ID string + Before gjson.Result + BeforeMissing bool + Amount int64 + Recipients []string + SubjectPrefix *string + SubjectPrefixSet bool + RateValues map[string]int64 +} + +type limitsAPIBackend struct { + client openai.Client + projectID string + maxItems int64 + mutationMu sync.Mutex + now func() time.Time +} + +func newLimitsBackend(cmd *cli.Command, project bool) (limitsBackend, error) { + maximum := cmd.Int64("max-items") + if maximum < -1 { + return nil, &localcommands.LocalError{Message: "--max-items must be -1, zero, or a positive integer."} + } + projectID := "" + if project { + projectID = cmd.String("project-id") + if projectID == "" { + return nil, &localcommands.LocalError{Message: "Set --project-id to inspect project limits."} + } + } + options, err := FlagOptions(cmd, apiquery.NestedQueryFormatBrackets, apiquery.ArrayQueryFormatBrackets, EmptyBody, true) + if err != nil { + return nil, err + } + options = append(GetDefaultRequestOptions(cmd), options...) + return &limitsAPIBackend{client: openai.NewClient(options...), projectID: projectID, maxItems: maximum}, nil +} + +func (snapshot limitsSnapshot) JSON() (gjson.Result, error) { + raw := func(value gjson.Result) json.RawMessage { + if !value.Exists() { + return json.RawMessage("null") + } + return json.RawMessage(value.Raw) + } + array := func(items []gjson.Result) []json.RawMessage { + result := make([]json.RawMessage, 0, len(items)) + for _, item := range items { + result = append(result, raw(item)) + } + return result + } + sourceErrors := make([]map[string]any, 0, len(snapshot.Errors)) + for _, failure := range snapshot.Errors { + source := map[string]any{"resource": failure.Resource, "status": "request_failed"} + var apiErr *openai.Error + var summaryErr *limitsCostSummaryError + if errors.Is(failure.Err, context.Canceled) || errors.Is(failure.Err, context.DeadlineExceeded) { + source["status"] = "canceled" + } else if errors.As(failure.Err, &apiErr) { + source["status"] = "api_error" + if apiErr.StatusCode > 0 { + source["status_code"] = apiErr.StatusCode + } + } else if errors.As(failure.Err, &summaryErr) { + source["status"] = "summary_unavailable" + } + if errors.As(failure.Err, &summaryErr) { + source["summary_reason"] = summaryErr.Reason + } + sourceErrors = append(sourceErrors, source) + } + var configured any + limitFailed := false + for _, failure := range snapshot.Errors { + limitFailed = limitFailed || failure.Resource == "spend_limit" || failure.Resource == "project" + } + if !limitFailed && snapshot.LimitAbsent { + configured = false + } else if !limitFailed && snapshot.Limit.Exists() { + configured = true + } + view := map[string]any{ + "scope": "organization", "spend_limit": raw(snapshot.Limit), "spend_limit_configured": configured, + "spend_alerts": array(snapshot.Alerts), "spend_alerts_complete": snapshot.AlertsComplete, + "source_errors": sourceErrors, + } + var costTotal any + if snapshot.CostsTotal != "" { + costTotal = snapshot.CostsTotal + } + view["costs"] = map[string]any{ + "start_time": snapshot.CostsStart, "end_time": snapshot.CostsEnd, "timezone": "UTC", + "currency": "USD", "total": costTotal, "complete": snapshot.CostsComplete, "buckets": array(snapshot.Costs), + } + if snapshot.ProjectID != "" { + view["scope"], view["project_id"], view["project"] = "project", snapshot.ProjectID, raw(snapshot.Project) + view["project_rate_limits"], view["project_rate_limits_complete"] = array(snapshot.Rates), snapshot.RatesComplete + } + encoded, err := json.Marshal(view) + if err != nil { + return gjson.Result{}, err + } + return gjson.ParseBytes(encoded), nil +} + +func (backend *limitsAPIBackend) Load(ctx context.Context) (limitsSnapshot, error) { + snapshot := limitsSnapshot{ProjectID: backend.projectID} + var failures []error + record := func(resource string, err error) { + if err != nil { + snapshot.Errors = append(snapshot.Errors, limitsSourceFailure{Resource: resource, Err: err}) + failures = append(failures, fmt.Errorf("%s: %w", resource, err)) + } + } + projectVerified := backend.projectID == "" + var err error + if backend.projectID != "" { + snapshot.Project, err = backend.get(ctx, "project", "") + record("project", err) + projectVerified = err == nil + } + if projectVerified { + snapshot.Limit, err = backend.get(ctx, "spend_limit", "") + if limitsHTTPStatus(err, http.StatusNotFound) { + // HTTP 404 does not distinguish absent configuration from endpoint or access errors. + snapshot.LimitUnconfirmed = true + } + record("spend_limit", err) + } + snapshot.Alerts, snapshot.AlertsComplete, err = backend.list(ctx, false) + record("spend_alerts", err) + if backend.projectID != "" { + snapshot.Rates, snapshot.RatesComplete, err = backend.list(ctx, true) + record("project_rate_limits", err) + } + now := time.Now().UTC() + if backend.now != nil { + now = backend.now().UTC() + } + snapshot.CostsStart = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC).Unix() + snapshot.CostsEnd = time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.UTC).Unix() + snapshot.Costs, snapshot.CostsComplete, err = backend.costs(ctx, snapshot.CostsStart, snapshot.CostsEnd) + if len(snapshot.Costs) != 0 || snapshot.CostsComplete { + var totalErr error + snapshot.CostsTotal, totalErr = limitsCostTotal(ctx, snapshot.Costs, snapshot.CostsStart, snapshot.CostsEnd, backend.projectID) + err = errors.Join(err, totalErr) + if totalErr != nil { + snapshot.CostsComplete = false + } + } + record("costs", err) + return snapshot, errors.Join(failures...) +} + +// Costs returns raw daily buckets. The independent summary never changes those records. +func (backend *limitsAPIBackend) costs(ctx context.Context, start, end int64) ([]gjson.Result, bool, error) { + items := []gjson.Result{} + cursor := "" + seen := map[string]bool{} + for { + if err := ctx.Err(); err != nil { + return items, false, err + } + pageSize := int64(100) + if backend.maxItems >= 0 { + pageSize = max(1, min(pageSize, backend.maxItems-int64(len(items)))) + } + params := openai.AdminOrganizationUsageCostsParams{ + StartTime: start, EndTime: openai.Int(end), Limit: openai.Int(pageSize), BucketWidth: "1d", + } + if backend.projectID != "" { + params.ProjectIDs = []string{backend.projectID} + } + if cursor != "" { + params.Page = openai.String(cursor) + } + var body []byte + _, err := backend.client.Admin.Organization.Usage.Costs(ctx, params, option.WithResponseBodyInto(&body)) + page, err := limitsResponse(body, err) + if err != nil { + return items, false, err + } + if err := limitsUniqueObjectContext(ctx, page); err != nil { + return items, false, err + } + data, more := page.Get("data"), page.Get("has_more") + if page.Get("object").Str != "page" || !data.IsArray() || more.Type != gjson.True && more.Type != gjson.False { + return items, false, errors.New("The Costs response is missing its buckets or pagination state.") + } + truncated := false + var readErr error + count := 0 + data.ForEach(func(_, item gjson.Result) bool { + count++ + if readErr = ctx.Err(); readErr != nil { + return false + } + if backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems { + truncated = true + return false + } + items = append(items, item) + return true + }) + if readErr != nil { + return items, false, readErr + } + if truncated || backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems && more.Bool() { + return items, false, nil + } + if !more.Bool() { + return items, true, nil + } + next := page.Get("next_page") + if count == 0 || next.Type != gjson.String || next.Str == "" || seen[next.Str] || next.Str == cursor { + return items, false, errors.New("The Costs pagination cursor is missing, repeated, or cannot advance.") + } + cursor, seen[next.Str] = next.Str, true + } +} + +func limitsHTTPStatus(err error, status int) bool { + var apiErr *openai.Error + return errors.As(err, &apiErr) && apiErr.StatusCode == status +} + +func limitsResponse(raw []byte, err error) (gjson.Result, error) { + if err != nil { + return gjson.Result{}, err + } + if !gjson.ValidBytes(raw) { + return gjson.Result{}, errors.New("The limits response contains invalid JSON.") + } + value := gjson.ParseBytes(raw) + if !value.IsObject() { + return gjson.Result{}, errors.New("The limits response must contain an object.") + } + return value, nil +} + +func (backend *limitsAPIBackend) get(ctx context.Context, resource, id string) (gjson.Result, error) { + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + var body []byte + options := []option.RequestOption{option.WithResponseBodyInto(&body)} + var err error + switch resource { + case "project": + _, err = backend.client.Admin.Organization.Projects.Get(ctx, backend.projectID, options...) + case "spend_limit": + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendLimit.Get(ctx, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendLimit.Get(ctx, backend.projectID, options...) + } + case "spend_alert": + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendAlerts.Get(ctx, id, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.Get(ctx, backend.projectID, id, options...) + } + default: + return gjson.Result{}, errors.New("Unknown limits resource.") + } + value, err := limitsResponse(body, err) + if err == nil { + expectedObject := "organization." + resource + if backend.projectID != "" && resource != "project" { + expectedObject = "project." + resource + } + if value.Get("object").Str != expectedObject { + err = errors.New("The limits response has an unexpected object type.") + } else if resource == "project" { + err = limitsUniqueObjectContext(ctx, value) + if err == nil && value.Get("id").Str != backend.projectID { + err = errors.New("The project response does not identify the requested project.") + } + } else if resource == "spend_alert" && value.Get("id").Str != id { + err = errors.New("The alert response does not identify the requested alert.") + } + } + return value, err +} + +func (backend *limitsAPIBackend) list(ctx context.Context, rates bool) ([]gjson.Result, bool, error) { + items := []gjson.Result{} + after := "" + seen := map[string]bool{} + for { + if err := ctx.Err(); err != nil { + return items, false, err + } + pageSize := int64(100) + if backend.maxItems >= 0 { + pageSize = max(1, min(pageSize, backend.maxItems-int64(len(items)))) + } + var body []byte + options := []option.RequestOption{option.WithResponseBodyInto(&body)} + var err error + if rates { + params := openai.AdminOrganizationProjectRateLimitListRateLimitsParams{Limit: openai.Int(pageSize)} + if after != "" { + params.After = openai.String(after) + } + _, err = backend.client.Admin.Organization.Projects.RateLimits.ListRateLimits(ctx, backend.projectID, params, options...) + } else if backend.projectID == "" { + params := openai.AdminOrganizationSpendAlertListParams{Limit: openai.Int(pageSize)} + if after != "" { + params.After = openai.String(after) + } + _, err = backend.client.Admin.Organization.SpendAlerts.List(ctx, params, options...) + } else { + params := openai.AdminOrganizationProjectSpendAlertListParams{Limit: openai.Int(pageSize)} + if after != "" { + params.After = openai.String(after) + } + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.List(ctx, backend.projectID, params, options...) + } + page, err := limitsResponse(body, err) + if err != nil { + return items, false, err + } + if err := limitsUniqueObjectContext(ctx, page); err != nil { + return items, false, err + } + data, more := page.Get("data"), page.Get("has_more") + if !data.IsArray() || more.Type != gjson.True && more.Type != gjson.False { + return items, false, errors.New("The limits page is missing its records or pagination state.") + } + truncated := false + var recordErr error + data.ForEach(func(_, item gjson.Result) bool { + if err := ctx.Err(); err != nil { + recordErr = err + return false + } + if backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems { + truncated = true + return false + } + if !item.IsObject() { + recordErr = errors.New("A limits page contains a non-object record.") + return false + } + items = append(items, item) + return true + }) + if recordErr != nil { + return items, false, recordErr + } + if truncated || backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems && more.Bool() { + return items, false, nil + } + if !more.Bool() { + return items, true, nil + } + cursor := page.Get("last_id") + if cursor.Type != gjson.String || cursor.Str == "" || cursor.Str == after || seen[cursor.Str] { + return items, false, errors.New("The limits pagination cursor is missing or repeated.") + } + if len(data.Array()) == 0 { + return items, false, errors.New("The limits page cannot advance without records.") + } + after, seen[cursor.Str] = cursor.Str, true + } +} + +func (backend *limitsAPIBackend) Mutate(ctx context.Context, change limitsChange) (result gjson.Result, resultErr error) { + attempted := false + defer func() { + if resultErr != nil { + resultErr = &limitsMutationFailure{Err: resultErr, Attempted: attempted} + } + }() + if !backend.mutationMu.TryLock() { + return gjson.Result{}, errors.New("A limits change is already in progress.") + } + defer backend.mutationMu.Unlock() + if err := backend.preflight(ctx, change); err != nil { + return gjson.Result{}, err + } + var body []byte + options := []option.RequestOption{option.WithMaxRetries(0), option.WithResponseBodyInto(&body)} + var err error + switch change.Kind { + case "limit_set": + attempted = true + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendLimit.Update(ctx, openai.AdminOrganizationSpendLimitUpdateParams{ + Currency: "USD", Interval: "month", ThresholdAmount: change.Amount, + }, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendLimit.Update(ctx, backend.projectID, openai.AdminOrganizationProjectSpendLimitUpdateParams{ + Currency: "USD", Interval: "month", ThresholdAmount: change.Amount, + }, options...) + } + case "limit_delete": + attempted = true + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendLimit.Delete(ctx, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendLimit.Delete(ctx, backend.projectID, options...) + } + case "alert_create", "alert_update": + payload, payloadErr := limitsAlertPayload(change) + if payloadErr != nil { + return gjson.Result{}, payloadErr + } + if backend.projectID == "" && change.Kind == "alert_create" { + var params openai.AdminOrganizationSpendAlertNewParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.SpendAlerts.New(ctx, params, options...) + } + } else if backend.projectID == "" { + var params openai.AdminOrganizationSpendAlertUpdateParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.SpendAlerts.Update(ctx, change.ID, params, options...) + } + } else if change.Kind == "alert_create" { + var params openai.AdminOrganizationProjectSpendAlertNewParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.New(ctx, backend.projectID, params, options...) + } + } else { + var params openai.AdminOrganizationProjectSpendAlertUpdateParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.Update(ctx, backend.projectID, change.ID, params, options...) + } + } + case "alert_delete": + attempted = true + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendAlerts.Delete(ctx, change.ID, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.Delete(ctx, backend.projectID, change.ID, options...) + } + case "rate_update": + params := openai.AdminOrganizationProjectRateLimitUpdateRateLimitParams{} + for field, value := range change.RateValues { + switch field { + case "max_requests_per_1_minute": + params.MaxRequestsPer1Minute = openai.Int(value) + case "max_tokens_per_1_minute": + params.MaxTokensPer1Minute = openai.Int(value) + case "max_requests_per_1_day": + params.MaxRequestsPer1Day = openai.Int(value) + case "max_images_per_1_minute": + params.MaxImagesPer1Minute = openai.Int(value) + case "max_audio_megabytes_per_1_minute": + params.MaxAudioMegabytesPer1Minute = openai.Int(value) + case "batch_1_day_max_input_tokens": + params.Batch1DayMaxInputTokens = openai.Int(value) + } + } + attempted = true + _, err = backend.client.Admin.Organization.Projects.RateLimits.UpdateRateLimit(ctx, backend.projectID, change.ID, params, options...) + } + value, responseErr := limitsResponse(body, err) + if responseErr == nil { + responseErr = limitsMutationConfirmedContext(ctx, value, change, backend.projectID != "") + } + return value, responseErr +} + +func limitsMutationConfirmed(value gjson.Result, change limitsChange, project bool) error { + return limitsMutationConfirmedContext(context.Background(), value, change, project) +} + +func limitsMutationConfirmedContext(ctx context.Context, value gjson.Result, change limitsChange, project bool) error { + scope := "organization" + if project { + scope = "project" + } + resource := "spend_limit" + if change.Kind == "alert_create" || change.Kind == "alert_update" || change.Kind == "alert_delete" { + resource = "spend_alert" + } + if change.Kind == "rate_update" { + resource = "rate_limit" + } + deleting := change.Kind == "limit_delete" || change.Kind == "alert_delete" + object := scope + "." + resource + if deleting { + object += ".deleted" + } + invalid := errors.New("The API returned an incomplete confirmation. Refresh and inspect the resource before another change.") + if value.Get("object").Type != gjson.String || value.Get("object").Str != object { + return invalid + } + if err := limitsUniqueJSONContext(ctx, value); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return invalid + } + if resource != "spend_limit" { + id := value.Get("id") + if id.Type != gjson.String || id.Str == "" || change.Kind != "alert_create" && id.Str != change.ID { + return invalid + } + } + if deleting { + if value.Get("deleted").Type != gjson.True { + return invalid + } + return ctx.Err() + } + if resource == "rate_limit" { + if value.Get("model").Type != gjson.String || value.Get("model").Str == "" || + value.Get("max_requests_per_1_minute").Type != gjson.Number || value.Get("max_tokens_per_1_minute").Type != gjson.Number { + return invalid + } + } else { + if value.Get("threshold_amount").Type != gjson.Number || value.Get("currency").Type != gjson.String || + value.Get("currency").Str == "" || value.Get("interval").Type != gjson.String || value.Get("interval").Str == "" { + return invalid + } + if resource == "spend_alert" { + channel := value.Get("notification_channel") + if !channel.IsObject() || channel.Get("type").Type != gjson.String || !channel.Get("recipients").IsArray() { + return invalid + } + } + } + mismatch := errors.New("The API returned different values from the reviewed change. Inspect the returned record before another change.") + if resource == "rate_limit" { + for field, requested := range change.RateValues { + if err := ctx.Err(); err != nil { + return err + } + returned, err := limitsInteger(value.Get(field)) + if err != nil || returned != requested { + return mismatch + } + } + } else { + returned, err := limitsInteger(value.Get("threshold_amount")) + if err != nil || returned != change.Amount || value.Get("currency").Str != "USD" || value.Get("interval").Str != "month" { + return mismatch + } + if resource == "spend_alert" { + channel := value.Get("notification_channel") + if channel.Get("type").Str != "email" { + return mismatch + } + recipients := map[string]int{} + for _, address := range change.Recipients { + recipients[address]++ + } + var recipientErr error + channel.Get("recipients").ForEach(func(_, address gjson.Result) bool { + if recipientErr = ctx.Err(); recipientErr != nil { + return false + } + if address.Type != gjson.String || recipients[address.Str] == 0 { + recipientErr = mismatch + return false + } + recipients[address.Str]-- + return true + }) + if recipientErr != nil { + return recipientErr + } + for _, count := range recipients { + if count != 0 { + return mismatch + } + } + if change.SubjectPrefixSet { + prefix := channel.Get("subject_prefix") + if !prefix.Exists() || change.SubjectPrefix == nil && prefix.Type != gjson.Null || + change.SubjectPrefix != nil && (prefix.Type != gjson.String || prefix.Str != *change.SubjectPrefix) { + return mismatch + } + } + } + } + return ctx.Err() +} + +func (backend *limitsAPIBackend) preflight(ctx context.Context, change limitsChange) error { + if err := ctx.Err(); err != nil { + return err + } + if err := limitsValidateChangeContext(ctx, change); err != nil { + return err + } + if backend.projectID != "" { + if _, err := backend.get(ctx, "project", ""); err != nil { + return err + } + } + if change.Kind == "alert_create" { + return ctx.Err() + } + var current gjson.Result + var err error + switch change.Kind { + case "limit_set", "limit_delete": + current, err = backend.get(ctx, "spend_limit", "") + if change.Kind == "limit_set" && change.BeforeMissing && limitsHTTPStatus(err, http.StatusNotFound) { + // BeforeMissing records explicit review of an unconfirmed baseline, not proven absence. + return ctx.Err() + } + case "alert_update", "alert_delete": + current, err = backend.get(ctx, "spend_alert", change.ID) + case "rate_update": + if backend.projectID == "" { + return errors.New("Rate-limit changes require a project scope.") + } + var items []gjson.Result + var complete bool + items, complete, err = backend.list(ctx, true) + if err == nil && !complete { + return errors.New("Rate-limit editing requires a complete list. Reopen with a larger --max-items value or -1.") + } + for _, item := range items { + if item.Get("id").Str == change.ID { + if current.Exists() { + return errors.New("The rate-limit ID appears more than once. Refresh before editing.") + } + current = item + } + } + } + if err != nil { + return err + } + if change.BeforeMissing || !current.Exists() { + return errors.New("The configuration changed. Refresh before applying this change.") + } + if err := limitsEditableBaselineContext(ctx, current, change.Kind); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + before, err := limitsCanonicalJSONContext(ctx, change.Before) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + now, err := limitsCanonicalJSONContext(ctx, current) + if err != nil { + return err + } + if before != now { + return errors.New("The configuration changed. Refresh before applying this change.") + } + return ctx.Err() +} + +func limitsAlertPayload(change limitsChange) ([]byte, error) { + channel := map[string]any{"type": "email", "recipients": change.Recipients} + if change.SubjectPrefixSet { + channel["subject_prefix"] = change.SubjectPrefix + } + return json.Marshal(map[string]any{ + "currency": "USD", "interval": "month", "threshold_amount": change.Amount, "notification_channel": channel, + }) +} diff --git a/pkg/custom/limits_service_test.go b/pkg/custom/limits_service_test.go new file mode 100644 index 00000000..ff52eb8c --- /dev/null +++ b/pkg/custom/limits_service_test.go @@ -0,0 +1,812 @@ +package custom + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + localcommands "github.com/openai/openai-cli/internal/commands" + requestconfig "github.com/openai/openai-cli/internal/request" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +const limitsTestLimit = `{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"enforcing"}}` +const limitsTestAlert = `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":100,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}` +const limitsTestRate = `{"id":"rate_demo","object":"project.rate_limit","model":"synthetic-model","max_requests_per_1_minute":100,"max_tokens_per_1_minute":200,"future":{"precision":0.1234567890123456789}}` + +func limitsTestBackend(t *testing.T, project string, maximum int64, handler http.HandlerFunc) *limitsAPIBackend { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer synthetic-limits-admin" { + t.Error("limits service did not select the admin credential") + } + w.Header().Set("Content-Type", "application/json") + handler(w, r) + })) + t.Cleanup(server.Close) + return &limitsAPIBackend{client: openai.NewClient(option.WithBaseURL(server.URL), + option.WithAPIKey("synthetic-limits-project"), option.WithAdminAPIKey("synthetic-limits-admin"), option.WithMaxRetries(0)), + projectID: project, maxItems: maximum} +} + +func limitsTestWrite(t *testing.T, w http.ResponseWriter, body string) { + t.Helper() + if _, err := io.WriteString(w, body); err != nil { + t.Errorf("write synthetic response: %v", err) + } +} + +func TestLimitsServicePartialLoadPreservesRawValues(t *testing.T) { + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitsTestWrite(t, w, `{"object":"organization.spend_limit","threshold_amount":9007199254740993,"future":{"decimal":0.1234567890123456789}}`) + return + } + w.WriteHeader(http.StatusForbidden) + limitsTestWrite(t, w, `{"error":{"message":"synthetic-private-field","type":"permission_error"}}`) + }) + snapshot, err := backend.Load(t.Context()) + require.Error(t, err) + require.False(t, snapshot.AlertsComplete) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "9007199254740993", value.Get("spend_limit.threshold_amount").Raw) + require.Equal(t, "0.1234567890123456789", value.Get("spend_limit.future.decimal").Raw) + require.Equal(t, "api_error", value.Get("source_errors.0.status").Str) + require.Equal(t, int64(403), value.Get("source_errors.0.status_code").Int()) + require.NotContains(t, value.Raw, "synthetic-private-field") + require.Equal(t, "[]", value.Get("spend_alerts").Raw) +} + +func TestLimitsServiceDoesNotInferLimitAbsenceAfterProjectFailure(t *testing.T) { + var limitRequests atomic.Int32 + backend := limitsTestBackend(t, "proj_demo", 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + w.WriteHeader(http.StatusNotFound) + limitsTestWrite(t, w, `{"error":{"message":"synthetic missing project"}}`) + } else if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitRequests.Add(1) + w.WriteHeader(http.StatusNotFound) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + snapshot, err := backend.Load(t.Context()) + require.Error(t, err) + require.Zero(t, limitRequests.Load()) + require.False(t, snapshot.LimitAbsent) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) +} + +func TestLimitsServicePaginationCapsAndCycle(t *testing.T) { + for _, test := range []struct { + maximum int64 + count int + complete, failure bool + requests int32 + }{ + {0, 0, false, false, 1}, {1, 1, false, false, 1}, {-1, 2, false, true, 2}, + } { + t.Run(fmt.Sprint(test.maximum), func(t *testing.T) { + var requests atomic.Int32 + backend := limitsTestBackend(t, "", test.maximum, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if test.maximum == 0 && r.URL.Query().Get("limit") != "1" { + t.Error("zero cap must inspect one-record page") + } + limitsTestWrite(t, w, `{"object":"list","data":[`+limitsTestAlert+`],"has_more":true,"last_id":"alert_demo"}`) + }) + items, complete, err := backend.list(t.Context(), false) + require.Len(t, items, test.count) + require.Equal(t, test.complete, complete) + require.Equal(t, test.failure, err != nil) + require.Equal(t, test.requests, requests.Load()) + }) + } +} + +func TestLimitsServiceRejectsStaleLimitAndDuplicateBaselineWithoutWriting(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + } + limitsTestWrite(t, w, strings.Replace(limitsTestLimit, "10000", "20000", 1)) + }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "limit_set", Amount: 30000, Before: gjson.Parse(limitsTestLimit)}) + require.ErrorContains(t, err, "configuration changed") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.False(t, failure.Attempted) + duplicate := strings.Replace(limitsTestAlert, `"type":"email"`, `"type":"email","type":"email"`, 1) + _, err = backend.Mutate(t.Context(), limitsChange{Kind: "alert_update", ID: "alert_demo", Amount: 300, + Recipients: []string{"finance@example.test"}, Before: gjson.Parse(duplicate)}) + require.ErrorContains(t, err, "duplicate") + require.Zero(t, writes.Load()) +} + +func TestLimitsServiceAlertPrefixPresenceAndNoMutationRetries(t *testing.T) { + empty, subject := "", "synthetic subject" + for _, test := range []struct { + name string + set bool + value *string + want string + }{ + {"absent", false, nil, ""}, {"null", true, nil, "null"}, {"empty", true, &empty, `""`}, {"text", true, &subject, `"synthetic subject"`}, + } { + t.Run(test.name, func(t *testing.T) { + var writes atomic.Int32 + var sent []byte + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + limitsTestWrite(t, w, limitsTestAlert) + return + } + writes.Add(1) + var err error + sent, err = io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + w.WriteHeader(http.StatusInternalServerError) + limitsTestWrite(t, w, `{"error":{"message":"synthetic failed write"}}`) + }) + backend.client = openai.NewClient(append(backend.client.Options, option.WithMaxRetries(3))...) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "alert_update", ID: "alert_demo", Before: gjson.Parse(limitsTestAlert), + Amount: 0, Recipients: []string{"ops@example.test"}, SubjectPrefixSet: test.set, SubjectPrefix: test.value}) + require.Error(t, err) + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.True(t, failure.Attempted) + require.Equal(t, int32(1), writes.Load()) + value := gjson.ParseBytes(sent) + require.Equal(t, "0", value.Get("threshold_amount").Raw) + require.Equal(t, test.want, value.Get("notification_channel.subject_prefix").Raw) + }) + } +} + +func TestLimitsServiceRateUpdatePreservesZeroNegativeAndUnknownResponseFields(t *testing.T) { + var body []byte + backend := limitsTestBackend(t, "proj_demo", 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + } else if r.Method == http.MethodGet { + limitsTestWrite(t, w, `{"object":"list","data":[`+limitsTestRate+`],"has_more":false}`) + } else { + var err error + body, err = io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + var updated map[string]json.RawMessage + var changed map[string]json.RawMessage + if err := json.Unmarshal([]byte(limitsTestRate), &updated); err != nil { + t.Error(err) + } + if err := json.Unmarshal(body, &changed); err != nil { + t.Error(err) + } + for key, value := range changed { + updated[key] = value + } + encoded, err := json.Marshal(updated) + if err != nil { + t.Error(err) + } + limitsTestWrite(t, w, string(encoded)) + } + }) + value, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(limitsTestRate), + RateValues: map[string]int64{"max_requests_per_1_minute": 0, "max_tokens_per_1_minute": -1, + "max_requests_per_1_day": 123, "max_images_per_1_minute": 4, + "max_audio_megabytes_per_1_minute": 5, "batch_1_day_max_input_tokens": 9007199254740993}}) + require.NoError(t, err) + require.Equal(t, "9007199254740993", gjson.GetBytes(body, "batch_1_day_max_input_tokens").Raw) + require.JSONEq(t, `{"max_requests_per_1_minute":0,"max_tokens_per_1_minute":-1,"max_requests_per_1_day":123,"max_images_per_1_minute":4,"max_audio_megabytes_per_1_minute":5,"batch_1_day_max_input_tokens":9007199254740993}`, string(body)) + require.Equal(t, "0.1234567890123456789", value.Get("future.precision").Raw) +} + +func TestLimitsServiceCancellationAndRepeatedSubmission(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + limitsTestWrite(t, w, limitsTestLimit) + return + } + close(started) + select { + case <-r.Context().Done(): + case <-release: + } + }) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + change := limitsChange{Kind: "limit_set", Amount: 20000, Before: gjson.Parse(limitsTestLimit)} + finished := make(chan error, 1) + go func() { _, err := backend.Mutate(ctx, change); finished <- err }() + <-started + _, err := backend.Mutate(t.Context(), change) + require.ErrorContains(t, err, "already in progress") + cancel() + close(release) + require.ErrorIs(t, <-finished, context.Canceled) +} + +func TestLimitsSnapshotErrorEnvelopeNeverIncludesProviderText(t *testing.T) { + snapshot := limitsSnapshot{Errors: []limitsSourceFailure{ + {Resource: "spend_limit", Err: context.Canceled}, + {Resource: "spend_alerts", Err: errors.New("synthetic-private-url")}, + }} + value, err := snapshot.JSON() + require.NoError(t, err) + require.True(t, json.Valid([]byte(value.Raw))) + require.Equal(t, "canceled", value.Get("source_errors.0.status").Str) + require.Equal(t, "request_failed", value.Get("source_errors.1.status").Str) + require.NotContains(t, value.Raw, "synthetic-private-url") +} + +func TestLimitsBackendConstructorPreservesRequestContextAndTLSClient(t *testing.T) { + requests := make(chan http.Header, 3) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- r.Header.Clone() + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitsTestWrite(t, w, limitsTestLimit) + } else if strings.HasSuffix(r.URL.Path, "/costs") { + limitsTestWrite(t, w, `{"object":"page","data":[],"has_more":false,"next_page":null}`) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + })) + t.Cleanup(server.Close) + root := &cli.Command{Name: "openai", Metadata: map[string]any{requestconfig.MTLSHTTPClientMetadata: server.Client()}, + Flags: []cli.Flag{&cli.StringFlag{Name: "base-url"}, &cli.StringFlag{Name: "api-key"}, + &cli.StringFlag{Name: "admin-api-key"}, &cli.StringFlag{Name: "organization"}, + &cli.StringFlag{Name: "project"}, NewRequestHeaderFlag(), &cli.Int64Flag{Name: "max-items", Value: 100}}, + Action: func(ctx context.Context, cmd *cli.Command) error { + backend, err := newLimitsBackend(cmd, false) + if err != nil { + return err + } + _, err = backend.Load(ctx) + return err + }} + require.NoError(t, root.Run(t.Context(), []string{"openai", "--base-url", server.URL, + "--api-key", "synthetic-project-key", "--admin-api-key", "synthetic-admin-key", + "--organization", "org_demo", "--project", "proj_header", "--header", "X-Limits-Demo: preserved"})) + for range 3 { + header := <-requests + require.Equal(t, "Bearer synthetic-admin-key", header.Get("Authorization")) + require.Equal(t, "org_demo", header.Get("OpenAI-Organization")) + require.Equal(t, "proj_header", header.Get("OpenAI-Project")) + require.Equal(t, "preserved", header.Get("X-Limits-Demo")) + } +} + +func TestLimitsServiceRefusesIncompleteRatePreflight(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "proj_demo", 1, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + } + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[`+limitsTestRate+`],"has_more":true,"last_id":"rate_demo"}`) + } + }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(limitsTestRate), + RateValues: map[string]int64{"max_requests_per_1_minute": 1}}) + require.ErrorContains(t, err, "complete list") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.False(t, failure.Attempted) + require.Zero(t, writes.Load()) +} + +func TestLimitsServiceMutationOperationsUseTypedRoutes(t *testing.T) { + for _, project := range []bool{false, true} { + for _, kind := range []string{"limit_set", "limit_delete", "alert_create", "alert_update", "alert_delete"} { + t.Run(fmt.Sprintf("%t/%s", project, kind), func(t *testing.T) { + projectID, scope, prefix := "", "organization", "/organization" + if project { + projectID, scope, prefix = "proj_demo", "project", "/organization/projects/proj_demo" + } + limit := strings.ReplaceAll(limitsTestLimit, "organization.spend_limit", scope+".spend_limit") + alert := strings.ReplaceAll(limitsTestAlert, "organization.spend_alert", scope+".spend_alert") + type request struct{ method, path, body string } + sent := make(chan request, 1) + backend := limitsTestBackend(t, projectID, 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + switch r.URL.Path { + case "/organization/projects/proj_demo": + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + case prefix + "/spend_limit": + limitsTestWrite(t, w, limit) + case prefix + "/spend_alerts/alert_demo": + limitsTestWrite(t, w, alert) + default: + t.Errorf("unexpected read: %s", r.URL.Path) + w.WriteHeader(400) + } + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + sent <- request{r.Method, r.URL.Path, string(body)} + switch kind { + case "limit_set": + limitsTestWrite(t, w, strings.Replace(limit, `"threshold_amount":10000`, `"threshold_amount":12345`, 1)) + case "alert_create", "alert_update": + limitsTestWrite(t, w, strings.Replace(alert, `"threshold_amount":100`, `"threshold_amount":12345`, 1)) + case "limit_delete": + limitsTestWrite(t, w, `{"object":"`+scope+`.spend_limit.deleted","deleted":true}`) + case "alert_delete": + limitsTestWrite(t, w, `{"id":"alert_demo","object":"`+scope+`.spend_alert.deleted","deleted":true}`) + } + }) + change := limitsChange{Kind: kind, Amount: 12345, Before: gjson.Parse(limit)} + path, method := prefix+"/spend_limit", http.MethodPost + if strings.HasPrefix(kind, "alert_") { + change.Before, change.ID = gjson.Parse(alert), "alert_demo" + change.Recipients = []string{"finance@example.test"} + path = prefix + "/spend_alerts/alert_demo" + if kind == "alert_create" { + path = prefix + "/spend_alerts" + change.Before, change.ID = gjson.Result{}, "" + } + } + if strings.HasSuffix(kind, "_delete") { + method = http.MethodDelete + } + _, err := backend.Mutate(t.Context(), change) + require.NoError(t, err) + got := <-sent + require.Equal(t, method, got.method) + require.Equal(t, path, got.path) + if method == http.MethodPost { + require.Equal(t, "12345", gjson.Get(got.body, "threshold_amount").Raw) + require.Equal(t, "USD", gjson.Get(got.body, "currency").Str) + require.Equal(t, "month", gjson.Get(got.body, "interval").Str) + } else { + require.Empty(t, got.body) + } + }) + } + } +} + +func TestLimitsServiceIncompleteMutationConfirmationIsUncertain(t *testing.T) { + for _, test := range []struct{ kind, response string }{ + {"limit_set", `{}`}, + {"limit_set", `{"object":"project.spend_limit","threshold_amount":100,"currency":"USD","interval":"month"}`}, + {"alert_create", `{"object":"organization.spend_alert","threshold_amount":100,"currency":"USD","interval":"month"}`}, + {"alert_update", strings.ReplaceAll(limitsTestAlert, "alert_demo", "alert_other")}, + {"limit_delete", `{"object":"organization.spend_limit.deleted","deleted":false}`}, + {"alert_delete", `{"id":"alert_other","object":"organization.spend_alert.deleted","deleted":true}`}, + } { + t.Run(test.kind+test.response, func(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + limitsTestWrite(t, w, test.response) + return + } + if strings.Contains(r.URL.Path, "spend_alerts") { + limitsTestWrite(t, w, limitsTestAlert) + } else { + limitsTestWrite(t, w, limitsTestLimit) + } + }) + change := limitsChange{Kind: test.kind, Amount: 100, Before: gjson.Parse(limitsTestLimit)} + if strings.HasPrefix(test.kind, "alert_") { + change.Before, change.ID, change.Recipients = gjson.Parse(limitsTestAlert), "alert_demo", []string{"finance@example.test"} + } + _, err := backend.Mutate(t.Context(), change) + require.ErrorContains(t, err, "incomplete confirmation") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.True(t, failure.Attempted) + require.Equal(t, int32(1), writes.Load()) + }) + } +} + +func TestLimitsServiceWrongDiscriminatorRemainsUnavailable(t *testing.T) { + for _, project := range []bool{false, true} { + t.Run(fmt.Sprint(project), func(t *testing.T) { + projectID := "" + if project { + projectID = "proj_demo" + } + var limitRequests atomic.Int32 + backend := limitsTestBackend(t, projectID, 100, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/proj_demo"): + limitsTestWrite(t, w, `{"id":"proj_demo","object":"project.spend_limit","future":123}`) + case strings.HasSuffix(r.URL.Path, "/spend_limit"): + limitRequests.Add(1) + limitsTestWrite(t, w, `{"object":"project.spend_limit","threshold_amount":10000,"future":123}`) + default: + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + snapshot, err := backend.Load(t.Context()) + require.Error(t, err) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) + require.Equal(t, "request_failed", value.Get("source_errors.0.status").Str) + if project { + require.Zero(t, limitRequests.Load()) + require.Equal(t, "project", value.Get("source_errors.0.resource").Str) + require.Equal(t, "123", value.Get("project.future").Raw) + } else { + require.Equal(t, int32(1), limitRequests.Load()) + require.Equal(t, "spend_limit", value.Get("source_errors.0.resource").Str) + require.Equal(t, "123", value.Get("spend_limit.future").Raw) + } + }) + } +} + +func TestLimitsSnapshotProjectFailureOverridesPriorLimitPresence(t *testing.T) { + for _, absent := range []bool{false, true} { + snapshot := limitsSnapshot{ + ProjectID: "proj_demo", Limit: gjson.Parse(limitsTestLimit), LimitAbsent: absent, + Errors: []limitsSourceFailure{{Resource: "project", Err: errors.New("synthetic project failure")}}, + } + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) + require.Equal(t, "10000", value.Get("spend_limit.threshold_amount").Raw) + } +} + +func TestLimitsBackendValidationUsesSafeLocalErrors(t *testing.T) { + for _, test := range []struct { + project bool + args []string + message string + }{ + {false, []string{"openai", "--max-items=-2"}, "--max-items must be -1, zero, or a positive integer."}, + {true, []string{"openai"}, "Set --project-id to inspect project limits."}, + } { + var validationErr error + root := &cli.Command{ + Name: "openai", Flags: []cli.Flag{&cli.Int64Flag{Name: "max-items", Value: 100}, &cli.StringFlag{Name: "project-id"}}, + Action: func(_ context.Context, cmd *cli.Command) error { + _, validationErr = newLimitsBackend(cmd, test.project) + return nil + }, + } + require.NoError(t, root.Run(t.Context(), test.args)) + var localErr *localcommands.LocalError + require.ErrorAs(t, validationErr, &localErr) + require.EqualError(t, validationErr, test.message) + } +} + +func limitsTestCostsBackend(t *testing.T, project string, maximum int64, costs http.HandlerFunc) *limitsAPIBackend { + t.Helper() + backend := limitsTestBackend(t, project, maximum, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/costs"): + costs(w, r) + case strings.HasSuffix(r.URL.Path, "/proj_demo"): + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + case strings.HasSuffix(r.URL.Path, "/spend_limit"): + limit := limitsTestLimit + if project != "" { + limit = strings.ReplaceAll(limit, "organization.spend_limit", "project.spend_limit") + } + limitsTestWrite(t, w, limit) + default: + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + backend.now = func() time.Time { return time.Date(2026, 10, 19, 12, 30, 0, 0, time.UTC) } + return backend +} + +func limitsTestCostBucket(start int64, results string) string { + return fmt.Sprintf(`{"object":"bucket","start_time":%d,"end_time":%d,"results":[%s],"future":{"exact":0.1234567890123456789}}`, start, start+86400, results) +} + +func limitsTestCostResult(amount, currency, project string) string { + return fmt.Sprintf(`{"object":"organization.costs.result","amount":{"value":%s,"currency":%q},"project_id":%q}`, amount, currency, project) +} + +func TestLimitsCostsExactPaginationAndUTCProjectScope(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + end := time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC).Unix() + for _, project := range []string{"", "proj_demo"} { + t.Run("project="+project, func(t *testing.T) { + var calls atomic.Int32 + backend := limitsTestCostsBackend(t, project, 100, func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.Method != http.MethodGet || r.URL.Query().Get("start_time") != fmt.Sprint(start) || + r.URL.Query().Get("end_time") != fmt.Sprint(end) || r.URL.Query().Get("bucket_width") != "1d" { + t.Error("Costs request changed its method or UTC period") + } + if got := r.URL.Query().Get("project_ids[]"); got != project { + t.Errorf("project filter = %q, want %q", got, project) + } + if r.URL.Query().Get("group_by[]") != "" { + t.Error("Costs summary unexpectedly grouped results") + } + if r.URL.Query().Get("page") == "" { + bucket := limitsTestCostBucket(start, limitsTestCostResult("9007199254740993.000000000000000001", "usd", project)) + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":true,"next_page":"next_demo"}`) + } else { + if r.URL.Query().Get("page") != "next_demo" { + t.Error("Costs did not retain its cursor") + } + bucket := limitsTestCostBucket(start+86400, limitsTestCostResult("2e-1", "USD", project)+","+limitsTestCostResult("-0.1", "usd", project)) + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":false,"next_page":null}`) + } + }) + snapshot, err := backend.Load(t.Context()) + require.NoError(t, err) + require.True(t, snapshot.CostsComplete) + require.Equal(t, "9007199254740993.100000000000000001", snapshot.CostsTotal) + require.Equal(t, int32(2), calls.Load()) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, start, value.Get("costs.start_time").Int()) + require.Equal(t, end, value.Get("costs.end_time").Int()) + require.Equal(t, "UTC", value.Get("costs.timezone").Str) + require.Equal(t, snapshot.CostsTotal, value.Get("costs.total").Str) + require.Equal(t, "0.1234567890123456789", value.Get("costs.buckets.0.future.exact").Raw) + }) + } +} + +func TestLimitsCostsBoundsAndLaterFailuresPreservePartialData(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + for _, test := range []struct { + name string + maximum int64 + mode string + calls int32 + wantTotal string + wantError bool + }{ + {"zero", 0, "", 1, "", false}, + {"bound", 1, "", 1, "0.10", false}, + {"later HTTP error", -1, "failure", 2, "0.10", true}, + {"cursor cycle", -1, "cycle", 2, "0.30", true}, + {"missing cursor", -1, "missing", 1, "0.10", true}, + {"empty continuation", -1, "empty", 2, "0.10", true}, + } { + t.Run(test.name, func(t *testing.T) { + var calls atomic.Int32 + backend := limitsTestCostsBackend(t, "", test.maximum, func(w http.ResponseWriter, r *http.Request) { + call := calls.Add(1) + if test.maximum == 0 && r.URL.Query().Get("limit") != "1" { + t.Error("zero maximum must make a one-bucket request") + } + if call == 2 && test.mode == "failure" { + w.WriteHeader(http.StatusForbidden) + limitsTestWrite(t, w, `{"error":{"message":"synthetic-private-cost-detail"}}`) + return + } + cursor := `"next_demo"` + if test.mode == "missing" { + cursor = "null" + } + data := limitsTestCostBucket(start+int64(call-1)*86400, limitsTestCostResult(fmt.Sprintf("0.%d", call), "usd", "")) + if call == 2 && test.mode == "empty" { + data = "" + } + limitsTestWrite(t, w, `{"object":"page","data":[`+data+`],"has_more":true,"next_page":`+cursor+`}`) + }) + snapshot, err := backend.Load(t.Context()) + require.Equal(t, test.wantError, err != nil) + require.Equal(t, test.calls, calls.Load()) + require.False(t, snapshot.CostsComplete) + require.Equal(t, test.wantTotal, snapshot.CostsTotal) + require.True(t, snapshot.Limit.Exists()) + value, jsonErr := snapshot.JSON() + require.NoError(t, jsonErr) + require.NotContains(t, value.Raw, "synthetic-private-cost-detail") + if test.mode == "failure" { + require.Equal(t, "api_error", value.Get("source_errors.0.status").Str) + require.Equal(t, int64(403), value.Get("source_errors.0.status_code").Int()) + } + }) + } +} + +func TestLimitsCostsSummaryFailuresRetainRawBuckets(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + for _, test := range []struct{ name, result, reason string }{ + {"huge exponent", limitsTestCostResult("1e9223372036854775807", "usd", ""), "unsupported_precision"}, + {"overflow exponent", limitsTestCostResult("1e999999999999999999999999999999", "usd", ""), "unsupported_precision"}, + {"currency", limitsTestCostResult("0.1", "EUR", ""), "unsupported_currency"}, + {"missing amount", `{"object":"organization.costs.result","amount":{"currency":"usd"}}`, "invalid_amount"}, + {"duplicate amount", `{"object":"organization.costs.result","amount":{"currency":"usd","value":0.1,"value":0.2}}`, "ambiguous_fields"}, + {"wrong result", `{"object":"organization.usage.completions.result","amount":{"currency":"usd","value":0.1}}`, "unexpected_result"}, + } { + t.Run(test.name, func(t *testing.T) { + bucket := limitsTestCostBucket(start, test.result) + backend := limitsTestCostsBackend(t, "", 100, func(w http.ResponseWriter, _ *http.Request) { + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":false,"next_page":null}`) + }) + snapshot, err := backend.Load(t.Context()) + var summary *limitsCostSummaryError + require.ErrorAs(t, err, &summary) + require.Equal(t, test.reason, summary.Reason) + require.ErrorContains(t, err, "costs.buckets") + require.False(t, snapshot.CostsComplete) + require.Empty(t, snapshot.CostsTotal) + require.Equal(t, bucket, snapshot.Costs[0].Raw) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("costs.total").Raw) + require.Equal(t, "summary_unavailable", value.Get("source_errors.0.status").Str) + require.Equal(t, test.reason, value.Get("source_errors.0.summary_reason").Str) + require.True(t, value.Get("spend_limit_configured").Bool()) + }) + } +} + +func TestLimitsUnknown404RemainsUnconfirmed(t *testing.T) { + for _, project := range []string{"", "proj_demo"} { + t.Run("project="+project, func(t *testing.T) { + backend := limitsTestBackend(t, project, 100, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/proj_demo"): + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + case strings.HasSuffix(r.URL.Path, "/spend_limit"): + w.WriteHeader(http.StatusNotFound) + limitsTestWrite(t, w, `{"error":{"code":"not_found","message":"synthetic private unavailable state"}}`) + case strings.HasSuffix(r.URL.Path, "/costs"): + limitsTestWrite(t, w, `{"object":"page","data":[],"has_more":false,"next_page":null}`) + default: + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + snapshot, err := backend.Load(t.Context()) + var apiErr *openai.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusNotFound, apiErr.StatusCode) + require.False(t, snapshot.LimitAbsent) + require.True(t, snapshot.LimitUnconfirmed) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) + require.Equal(t, int64(404), value.Get("source_errors.0.status_code").Int()) + }) + } +} + +func TestLimitsReviewedUnconfirmedSetRechecksBeforeWriting(t *testing.T) { + for _, test := range []struct { + name string + status int + project bool + projectMissing bool + write bool + }{ + {"organization still unconfirmed", 404, false, false, true}, + {"project still unconfirmed", 404, true, false, true}, + {"new configuration", 200, false, false, false}, + {"access changed", 403, false, false, false}, + {"project disappeared", 404, true, true, false}, + } { + t.Run(test.name, func(t *testing.T) { + var writes atomic.Int32 + project, scope := "", "organization" + if test.project { + project, scope = "proj_demo", "project" + } + backend := limitsTestBackend(t, project, 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + if test.projectMissing { + w.WriteHeader(http.StatusNotFound) + limitsTestWrite(t, w, `{"error":{"message":"missing"}}`) + return + } + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + return + } + if r.Method == http.MethodGet { + w.WriteHeader(test.status) + if test.status == 200 { + limitsTestWrite(t, w, limitsTestLimit) + } else { + limitsTestWrite(t, w, `{"error":{"code":"not_found","message":"unconfirmed"}}`) + } + return + } + writes.Add(1) + limitsTestWrite(t, w, `{"object":"`+scope+`.spend_limit","threshold_amount":12345,"currency":"USD","interval":"month"}`) + }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "limit_set", Amount: 12345, BeforeMissing: true}) + if test.write { + require.NoError(t, err) + require.Equal(t, int32(1), writes.Load()) + } else { + require.Error(t, err) + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.False(t, failure.Attempted) + require.Zero(t, writes.Load()) + } + }) + } +} + +func TestLimitsMismatchedMutationKeepsReturnedRecordAndUncertainty(t *testing.T) { + var writes atomic.Int32 + response := strings.TrimSuffix(limitsTestLimit, "}") + `,"future":{"exact":123456789012345678901234567890}}` + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + limitsTestWrite(t, w, limitsTestLimit) + return + } + writes.Add(1) + limitsTestWrite(t, w, response) + }) + value, err := backend.Mutate(t.Context(), limitsChange{Kind: "limit_set", Amount: 12345, Before: gjson.Parse(limitsTestLimit)}) + require.ErrorContains(t, err, "different values") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.True(t, failure.Attempted) + require.Equal(t, int32(1), writes.Load()) + require.Equal(t, response, value.Raw) +} + +func TestLimitsCostsCancellationRetainsEarlierBuckets(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + started := make(chan struct{}) + backend := limitsTestCostsBackend(t, "", -1, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + bucket := limitsTestCostBucket(start, limitsTestCostResult("0.1", "usd", "")) + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":true,"next_page":"next_demo"}`) + return + } + close(started) + <-r.Context().Done() + }) + type outcome struct { + snapshot limitsSnapshot + err error + } + finished := make(chan outcome, 1) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + go func() { snapshot, err := backend.Load(ctx); finished <- outcome{snapshot, err} }() + <-started + cancel() + result := <-finished + require.ErrorIs(t, result.err, context.Canceled) + require.Len(t, result.snapshot.Costs, 1) + require.False(t, result.snapshot.CostsComplete) + require.Empty(t, result.snapshot.CostsTotal) + value, err := result.snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "canceled", value.Get("source_errors.0.status").Str) + require.Equal(t, "0.1", value.Get("costs.buckets.0.results.0.amount.value").Raw) +} diff --git a/pkg/custom/limits_values.go b/pkg/custom/limits_values.go new file mode 100644 index 00000000..6ee70bb4 --- /dev/null +++ b/pkg/custom/limits_values.go @@ -0,0 +1,443 @@ +package custom + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + "net/mail" + "strconv" + "strings" + + "github.com/tidwall/gjson" +) + +// parseLimitsDollars converts explicit decimal USD input to integer cents. +func parseLimitsDollars(value string, minimum int64) (int64, error) { + invalid := fmt.Errorf("Enter a USD amount using digits and at most two decimal places.") + if value == "" { + return 0, invalid + } + dollars, cents, decimal := strings.Cut(value, ".") + if dollars == "" || decimal && (cents == "" || len(cents) > 2) { + return 0, invalid + } + for _, part := range []string{dollars, cents} { + for i := range len(part) { + if part[i] < '0' || part[i] > '9' { + return 0, invalid + } + } + } + whole, err := strconv.ParseUint(dollars, 10, 64) + if err != nil || whole > uint64(math.MaxInt64)/100 { + return 0, fmt.Errorf("The USD amount is too large.") + } + var fraction uint64 + if cents != "" { + fraction = uint64(cents[0]-'0') * 10 + if len(cents) == 2 { + fraction += uint64(cents[1] - '0') + } + } + if whole*100 > uint64(math.MaxInt64)-fraction { + return 0, fmt.Errorf("The USD amount is too large.") + } + result := int64(whole*100 + fraction) + if result < minimum { + return 0, fmt.Errorf("The USD amount must be at least %s.", formatLimitsDollars(minimum)) + } + return result, nil +} + +func formatLimitsDollars(cents int64) string { + digits := strconv.FormatInt(cents, 10) + sign := "" + if strings.HasPrefix(digits, "-") { + sign, digits = "-", digits[1:] + } + for len(digits) < 3 { + digits = "0" + digits + } + return sign + digits[:len(digits)-2] + "." + digits[len(digits)-2:] +} + +func parseLimitsRecipients(value string) ([]string, error) { + parts := strings.Split(value, ",") + recipients := make([]string, 0, len(parts)) + for _, part := range parts { + address := strings.TrimSpace(part) + parsed, err := mail.ParseAddress(address) + if err != nil || parsed.Name != "" || parsed.Address != address || strings.ContainsAny(address, "\r\n") { + return nil, fmt.Errorf("Enter email addresses separated by commas, without display names.") + } + recipients = append(recipients, address) + } + return recipients, nil +} + +func limitsUniqueObject(value gjson.Result) error { + return limitsUniqueObjectContext(context.Background(), value) +} + +func limitsUniqueObjectContext(ctx context.Context, value gjson.Result) error { + seen := map[string]bool{} + var err error + value.ForEach(func(key, _ gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + if seen[key.Str] { + err = errors.New("The response has duplicate fields. Use the original API command to inspect it.") + return false + } + seen[key.Str] = true + return true + }) + if ctx.Err() != nil { + return ctx.Err() + } + return err +} + +func limitsUniqueJSON(value gjson.Result) error { + return limitsUniqueJSONContext(context.Background(), value) +} + +func limitsUniqueJSONContext(ctx context.Context, value gjson.Result) error { + pending := []gjson.Result{value} + for len(pending) > 0 { + if err := ctx.Err(); err != nil { + return err + } + item := pending[len(pending)-1] + pending = pending[:len(pending)-1] + if item.IsObject() { + if err := limitsUniqueObjectContext(ctx, item); err != nil { + return err + } + } + if item.IsObject() || item.IsArray() { + item.ForEach(func(_, field gjson.Result) bool { + if ctx.Err() != nil { + return false + } + if field.IsObject() || field.IsArray() { + pending = append(pending, field) + } + return true + }) + } + } + return ctx.Err() +} + +func limitsKnownFields(value gjson.Result, fields ...string) bool { + known := make(map[string]bool, len(fields)) + for _, field := range fields { + known[field] = true + } + valid := true + value.ForEach(func(key, _ gjson.Result) bool { + valid = known[key.Str] + return valid + }) + return valid +} + +func limitsInteger(value gjson.Result) (int64, error) { + if value.Type != gjson.Number { + return 0, errors.New("The returned amount or rate is not an integer.") + } + integer, err := strconv.ParseInt(value.Raw, 10, 64) + if err != nil { + return 0, errors.New("The returned amount or rate cannot be represented by this form.") + } + return integer, nil +} + +func limitsEditableBaseline(value gjson.Result, kind string) error { + return limitsEditableBaselineContext(context.Background(), value, kind) +} + +func limitsEditableBaselineContext(ctx context.Context, value gjson.Result, kind string) error { + if !value.IsObject() || !gjson.Valid(value.Raw) { + return errors.New("The returned configuration cannot be edited by this form.") + } + if err := limitsUniqueJSONContext(ctx, value); err != nil { + return err + } + object := value.Get("object") + if object.Type != gjson.String { + return errors.New("The returned configuration has no recognized object type.") + } + switch kind { + case "limit_set", "limit_delete": + if object.Str != "organization.spend_limit" && object.Str != "project.spend_limit" { + return errors.New("The returned object is not a spend limit.") + } + if kind == "limit_delete" { + return nil + } + if !limitsKnownFields(value, "object", "threshold_amount", "currency", "interval", "enforcement") { + return errors.New("This spend limit contains unfamiliar configuration. Use the original API command to edit it.") + } + enforcement := value.Get("enforcement") + if enforcement.Exists() && enforcement.Type != gjson.Null { + status := enforcement.Get("status") + if !enforcement.IsObject() || !limitsKnownFields(enforcement, "status") || status.Type != gjson.String || + status.Str != "inactive" && status.Str != "enforcing" { + return errors.New("This spend limit has unfamiliar enforcement configuration. Use the original API command to edit it.") + } + } + case "alert_update", "alert_delete": + if object.Str != "organization.spend_alert" && object.Str != "project.spend_alert" { + return errors.New("The returned object is not a spend alert.") + } + if value.Get("id").Type != gjson.String || value.Get("id").Str == "" { + return errors.New("The returned spend alert has no identifier.") + } + if kind == "alert_delete" { + return nil + } + if !limitsKnownFields(value, "id", "object", "threshold_amount", "currency", "interval", "notification_channel") { + return errors.New("This alert contains unfamiliar configuration. Use the original API command to edit it.") + } + channel := value.Get("notification_channel") + if !channel.IsObject() || !limitsKnownFields(channel, "type", "recipients", "subject_prefix") || channel.Get("type").Str != "email" { + return errors.New("This alert has unfamiliar notification settings. Use the original API command to edit it.") + } + recipients := channel.Get("recipients") + if !recipients.IsArray() || len(recipients.Array()) == 0 { + return errors.New("This alert has no editable email recipients.") + } + for _, address := range recipients.Array() { + if err := ctx.Err(); err != nil { + return err + } + if address.Type != gjson.String { + return errors.New("This alert has an unrecognized email recipient.") + } + parsed, err := parseLimitsRecipients(address.Str) + if err != nil || len(parsed) != 1 || parsed[0] != address.Str { + return errors.New("This alert has an unrecognized email recipient.") + } + } + prefix := channel.Get("subject_prefix") + if prefix.Exists() && prefix.Type != gjson.Null && prefix.Type != gjson.String { + return errors.New("This alert has an unrecognized email subject prefix.") + } + case "rate_update": + if object.Str != "project.rate_limit" || value.Get("id").Type != gjson.String || value.Get("id").Str == "" || + value.Get("model").Type != gjson.String || value.Get("model").Str == "" { + return errors.New("The returned rate limit has no recognized identity.") + } + for _, field := range limitsRateFields { + item := value.Get(field) + if item.Exists() || field == "max_requests_per_1_minute" || field == "max_tokens_per_1_minute" { + if _, err := limitsInteger(item); err != nil { + return err + } + } + } + return nil + default: + return errors.New("Unknown limits edit operation.") + } + if value.Get("currency").Str != "USD" || value.Get("interval").Str != "month" { + return errors.New("This form can edit only returned monthly USD configurations.") + } + _, err := limitsInteger(value.Get("threshold_amount")) + return err +} + +var limitsRateFields = []string{ + "max_requests_per_1_minute", "max_tokens_per_1_minute", "max_requests_per_1_day", + "max_images_per_1_minute", "max_audio_megabytes_per_1_minute", "batch_1_day_max_input_tokens", +} + +func limitsValidateChange(change limitsChange) error { + return limitsValidateChangeContext(context.Background(), change) +} + +func limitsValidateChangeContext(ctx context.Context, change limitsChange) error { + if err := ctx.Err(); err != nil { + return err + } + if change.BeforeMissing && change.Before.Exists() { + return errors.New("The displayed configuration has an inconsistent presence state. Refresh before editing.") + } + switch change.Kind { + case "limit_set": + if change.Amount < 1 { + return errors.New("A hard spend limit must be at least USD 0.01.") + } + case "alert_create", "alert_update": + if change.Amount < 0 { + return errors.New("An alert threshold cannot be negative.") + } + if len(change.Recipients) == 0 { + return errors.New("An alert requires at least one email recipient.") + } + for _, address := range change.Recipients { + if err := ctx.Err(); err != nil { + return err + } + parsed, err := parseLimitsRecipients(address) + if err != nil || len(parsed) != 1 || parsed[0] != address { + return errors.New("Enter valid email recipients without display names.") + } + } + case "rate_update": + if len(change.RateValues) == 0 { + return errors.New("Set at least one rate-limit value.") + } + for key := range change.RateValues { + known := false + for _, field := range limitsRateFields { + known = known || key == field + } + if !known { + return errors.New("This form does not support the requested rate-limit field.") + } + } + case "limit_delete", "alert_delete": + default: + return errors.New("Unknown limits operation.") + } + if change.Kind == "alert_create" || change.Kind == "limit_set" && change.BeforeMissing { + return nil + } + if err := limitsEditableBaselineContext(ctx, change.Before, change.Kind); err != nil { + return err + } + if change.Kind == "alert_update" || change.Kind == "alert_delete" || change.Kind == "rate_update" { + if change.ID == "" || change.Before.Get("id").Str != change.ID { + return errors.New("The selected identifier does not match the displayed configuration.") + } + } + return nil +} + +func limitsCanonicalJSON(value gjson.Result) (string, error) { + return limitsCanonicalJSONContext(context.Background(), value) +} + +func limitsCanonicalJSONContext(ctx context.Context, value gjson.Result) (string, error) { + if err := limitsUniqueJSONContext(ctx, value); err != nil { + return "", err + } + decoder := json.NewDecoder(strings.NewReader(value.Raw)) + decoder.UseNumber() + var decoded any + if err := decoder.Decode(&decoded); err != nil { + return "", errors.New("The configuration cannot be compared safely.") + } + encoded, err := json.Marshal(decoded) + if ctx.Err() != nil { + return "", ctx.Err() + } + return string(encoded), err +} + +// limitsCostTotal summarizes raw USD amounts, never integer threshold cents. +// Unsupported records remain in the snapshot even when a total cannot be calculated. +func limitsCostTotal(ctx context.Context, buckets []gjson.Result, start, end int64, projectID string) (string, error) { + total := new(big.Rat) + precision := 2 + var ranges [][2]int64 + invalid := &limitsCostSummaryError{Reason: "invalid_bucket"} + for _, bucket := range buckets { + if err := limitsUniqueJSONContext(ctx, bucket); err != nil { + if ctx.Err() != nil { + return "", ctx.Err() + } + return "", &limitsCostSummaryError{Reason: "ambiguous_fields"} + } + if !bucket.IsObject() || bucket.Get("object").Str != "bucket" || !bucket.Get("results").IsArray() { + return "", invalid + } + from, fromErr := limitsInteger(bucket.Get("start_time")) + to, toErr := limitsInteger(bucket.Get("end_time")) + if fromErr != nil || toErr != nil || from < start || to > end || from >= to || to-from > 86400 { + return "", invalid + } + for _, previous := range ranges { + if from < previous[1] && previous[0] < to { + return "", &limitsCostSummaryError{Reason: "overlapping_buckets"} + } + } + ranges = append(ranges, [2]int64{from, to}) + var resultErr error + bucket.Get("results").ForEach(func(_, result gjson.Result) bool { + if resultErr = ctx.Err(); resultErr != nil { + return false + } + if !result.IsObject() || result.Get("object").Str != "organization.costs.result" { + resultErr = &limitsCostSummaryError{Reason: "unexpected_result"} + return false + } + if result.Get("amount.currency").Type != gjson.String || !strings.EqualFold(result.Get("amount.currency").Str, "USD") { + resultErr = &limitsCostSummaryError{Reason: "unsupported_currency"} + return false + } + project := result.Get("project_id") + if projectID != "" && project.Exists() && project.Type != gjson.Null && (project.Type != gjson.String || project.Str != projectID) { + resultErr = &limitsCostSummaryError{Reason: "unexpected_project"} + return false + } + amount, scale, err := limitsCostDecimal(result.Get("amount.value")) + if err != nil { + resultErr = err + return false + } + total.Add(total, amount) + precision = max(precision, scale) + return true + }) + if resultErr != nil { + return "", resultErr + } + } + if err := ctx.Err(); err != nil { + return "", err + } + return total.FloatString(precision), nil +} + +func limitsCostDecimal(value gjson.Result) (*big.Rat, int, error) { + invalid := &limitsCostSummaryError{Reason: "invalid_amount"} + if value.Type != gjson.Number || !gjson.Valid(value.Raw) { + return nil, 0, invalid + } + coefficient := value.Raw + exponent := int64(0) + if at := strings.IndexAny(coefficient, "eE"); at >= 0 { + var err error + exponent, err = strconv.ParseInt(coefficient[at+1:], 10, 64) + if err != nil { + return nil, 0, &limitsCostSummaryError{Reason: "unsupported_precision"} + } + coefficient = coefficient[:at] + } + // Bound only derived display arithmetic. Raw API records are never truncated or rejected. + const summaryDigits = 10000 + if len(coefficient) > summaryDigits || exponent > summaryDigits || exponent < -summaryDigits { + return nil, 0, &limitsCostSummaryError{Reason: "unsupported_precision"} + } + scale := int64(0) + if at := strings.IndexByte(coefficient, '.'); at >= 0 { + scale = int64(len(coefficient) - at - 1) + } + scale = max(0, scale-exponent) + if scale > summaryDigits || int64(len(coefficient))+max(0, exponent) > summaryDigits { + return nil, 0, &limitsCostSummaryError{Reason: "unsupported_precision"} + } + amount, ok := new(big.Rat).SetString(value.Raw) + if !ok { + return nil, 0, invalid + } + return amount, int(scale), nil +} diff --git a/pkg/custom/limits_values_test.go b/pkg/custom/limits_values_test.go new file mode 100644 index 00000000..0b66ea14 --- /dev/null +++ b/pkg/custom/limits_values_test.go @@ -0,0 +1,181 @@ +package custom + +import ( + "context" + "math" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestLimitsDollarsExactBoundaries(t *testing.T) { + for _, test := range []struct { + input string + want int64 + }{ + {"0", 0}, {"0.00", 0}, {"0.01", 1}, {"1.2", 120}, {"00100.01", 10001}, + {"90071992547409.93", 9007199254740993}, {"92233720368547758.07", math.MaxInt64}, + } { + t.Run(test.input, func(t *testing.T) { + got, err := parseLimitsDollars(test.input, 0) + require.NoError(t, err) + require.Equal(t, test.want, got) + roundTrip, err := parseLimitsDollars(formatLimitsDollars(got), 0) + require.NoError(t, err) + require.Equal(t, got, roundTrip) + }) + } + for _, input := range []string{"", ".01", "1.", "1.001", "1.2.3", "+1", "-0", "-1", "$1", "1,000", "1e2", "NaN", " 1", "1\n", "12", "92233720368547758.08", "92233720368547759", "18446744073709551616"} { + t.Run("reject "+input, func(t *testing.T) { _, err := parseLimitsDollars(input, 0); require.Error(t, err) }) + } + _, err := parseLimitsDollars("0", 1) + require.Error(t, err) + require.Equal(t, "-92233720368547758.08", formatLimitsDollars(math.MinInt64)) + require.Equal(t, "-0.01", formatLimitsDollars(-1)) +} + +func TestLimitsRecipientsDoNotInterpretDisplayNamesOrControls(t *testing.T) { + got, err := parseLimitsRecipients("finance@example.test, ops+alerts@example.test") + require.NoError(t, err) + require.Equal(t, []string{"finance@example.test", "ops+alerts@example.test"}, got) + for _, input := range []string{"", "finance", "Finance ", "finance@example.test,", "finance@example.test\r\nBcc: other@example.test", "a@example.test\x1b"} { + _, err := parseLimitsRecipients(input) + require.Error(t, err, input) + } +} + +func TestLimitsEditBaselineRejectsAmbiguousOrUnrepresentableConfiguration(t *testing.T) { + valid := `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}` + require.NoError(t, limitsEditableBaseline(gjson.Parse(valid), "alert_update")) + for _, raw := range []string{ + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"],"recipients":["other@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"],"delivery_policy":{"mode":"unknown"}}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":9223372036854775808,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":1.0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":1,"currency":"EUR","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":1,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"],"subject_prefix":false}}`, + } { + require.Error(t, limitsEditableBaseline(gjson.Parse(raw), "alert_update"), raw) + } + require.Error(t, limitsUniqueJSON(gjson.Parse(`{"records":[{"id":"a","\u0069d":"b"}]}`))) +} + +func TestLimitsBaselineComparisonRetainsPrecisionAndIgnoresObjectOrder(t *testing.T) { + a, err := limitsCanonicalJSON(gjson.Parse(`{"a":9007199254740993,"b":{"x":0.1234567890123456789}}`)) + require.NoError(t, err) + b, err := limitsCanonicalJSON(gjson.Parse(`{ "b": {"x": 0.1234567890123456789}, "a": 9007199254740993 }`)) + require.NoError(t, err) + require.Equal(t, a, b) + c, err := limitsCanonicalJSON(gjson.Parse(`{"a":9007199254740992,"b":{"x":0.1234567890123456789}}`)) + require.NoError(t, err) + require.NotEqual(t, a, c) +} + +func TestLimitsCostsExactDecimalValues(t *testing.T) { + for _, test := range []struct { + input, want string + scale int + }{ + {"0.1", "1/10", 1}, {"0.20", "1/5", 2}, {"-0.015", "-3/200", 3}, + {"9007199254740993.000000000000000001", "9007199254740993000000000000000001/1000000000000000000", 18}, + {"1.23e2", "123", 0}, {"1.23e-2", "123/10000", 4}, {"1E+2", "100", 0}, {"-0", "0", 0}, + } { + t.Run(test.input, func(t *testing.T) { + value, scale, err := limitsCostDecimal(gjson.Parse(test.input)) + require.NoError(t, err) + require.Equal(t, test.want, value.RatString()) + require.Equal(t, test.scale, scale) + }) + } + for _, input := range []string{"null", `"0.1"`, "true", `{}`, `[]`} { + _, _, err := limitsCostDecimal(gjson.Parse(input)) + require.Error(t, err) + } + for _, input := range []string{ + "1e9223372036854775807", "1e-9223372036854775808", "1e999999999999999999999999999999999", + "1e10001", "1e-10001", "1." + strings.Repeat("1", 10001), strings.Repeat("1", 10001), + } { + _, _, err := limitsCostDecimal(gjson.Parse(input)) + var summary *limitsCostSummaryError + require.ErrorAs(t, err, &summary) + require.Equal(t, "unsupported_precision", summary.Reason) + } +} + +func TestLimitsCostsTotalRejectsAmbiguousScopeAndBuckets(t *testing.T) { + const start, end int64 = 86400, 4 * 86400 + valid := gjson.Parse(limitsTestCostBucket(start, limitsTestCostResult("0.1", "usd", "proj_demo"))) + for _, test := range []struct { + name, project string + buckets []gjson.Result + reason string + }{ + {"overlap", "", []gjson.Result{valid, valid}, "overlapping_buckets"}, + {"outside period", "", []gjson.Result{gjson.Parse(limitsTestCostBucket(0, ""))}, "invalid_bucket"}, + {"non-object", "", []gjson.Result{gjson.Parse(`42`)}, "invalid_bucket"}, + {"wrong project", "proj_other", []gjson.Result{valid}, "unexpected_project"}, + {"malformed results", "", []gjson.Result{gjson.Parse(`{"object":"bucket","start_time":86400,"end_time":172800,"results":null}`)}, "invalid_bucket"}, + } { + t.Run(test.name, func(t *testing.T) { + value, err := limitsCostTotal(t.Context(), test.buckets, start, end, test.project) + var summary *limitsCostSummaryError + require.ErrorAs(t, err, &summary) + require.Empty(t, value) + require.Equal(t, test.reason, summary.Reason) + }) + } + value, err := limitsCostTotal(t.Context(), []gjson.Result{valid, + gjson.Parse(limitsTestCostBucket(start+86400, limitsTestCostResult("-0.3", "USD", "proj_demo")+","+limitsTestCostResult("2e-1", "usd", "proj_demo"))), + }, start, end, "proj_demo") + require.NoError(t, err) + require.Equal(t, "0.00", value) + value, err = limitsCostTotal(t.Context(), nil, start, end, "") + require.NoError(t, err) + require.Equal(t, "0.00", value) +} + +func TestLimitsMutationConfirmationMatchesReviewedFields(t *testing.T) { + change := limitsChange{Kind: "alert_update", ID: "alert_demo", Amount: 12345, + Recipients: []string{"finance@example.test", "ops@example.test"}, SubjectPrefixSet: true} + matching := `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":12345,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["ops@example.test","finance@example.test"],"subject_prefix":null},"future":{"exact":0.1234567890123456789}}` + require.NoError(t, limitsMutationConfirmed(gjson.Parse(matching), change, false)) + for _, response := range []string{ + strings.Replace(matching, `"threshold_amount":12345`, `"threshold_amount":12344`, 1), + strings.Replace(matching, `"threshold_amount":12345`, `"threshold_amount":12345.0`, 1), + strings.Replace(matching, `"currency":"USD"`, `"currency":"EUR"`, 1), + strings.Replace(matching, `"interval":"month"`, `"interval":"day"`, 1), + strings.Replace(matching, `"type":"email"`, `"type":"unknown"`, 1), + strings.Replace(matching, `"ops@example.test"`, `"other@example.test"`, 1), + strings.Replace(matching, `"ops@example.test","finance@example.test"`, `"finance@example.test","finance@example.test"`, 1), + strings.Replace(matching, `"subject_prefix":null`, `"subject_prefix":""`, 1), + strings.Replace(matching, `,"subject_prefix":null`, ``, 1), + strings.Replace(matching, `"threshold_amount":12345`, `"threshold_amount":12345,"threshold_amount":0`, 1), + } { + require.Error(t, limitsMutationConfirmed(gjson.Parse(response), change, false)) + } + empty := "" + change.SubjectPrefix = &empty + require.NoError(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(matching, `"subject_prefix":null`, `"subject_prefix":""`, 1)), change, false)) + change.SubjectPrefixSet = false + require.NoError(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(matching, `,"subject_prefix":null`, ``, 1)), change, false)) + rateChange := limitsChange{Kind: "rate_update", ID: "rate_demo", RateValues: map[string]int64{"max_requests_per_1_minute": 9007199254740993}} + rate := strings.Replace(limitsTestRate, `"max_requests_per_1_minute":100`, `"max_requests_per_1_minute":9007199254740993`, 1) + require.NoError(t, limitsMutationConfirmed(gjson.Parse(rate), rateChange, true)) + require.Error(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(rate, "9007199254740993", "9007199254740992", 1)), rateChange, true)) + require.Error(t, limitsMutationConfirmed(gjson.Parse(limitsTestRate), rateChange, true)) +} + +func TestLimitsValueChecksPreserveCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, limitsUniqueJSONContext(ctx, gjson.Parse(limitsTestLimit)), context.Canceled) + require.ErrorIs(t, limitsEditableBaselineContext(ctx, gjson.Parse(limitsTestLimit), "limit_set"), context.Canceled) + require.ErrorIs(t, limitsValidateChangeContext(ctx, limitsChange{Kind: "limit_set", Amount: 100, BeforeMissing: true}), context.Canceled) + _, err := limitsCanonicalJSONContext(ctx, gjson.Parse(limitsTestLimit)) + require.ErrorIs(t, err, context.Canceled) + _, err = limitsCostTotal(ctx, nil, 86400, 172800, "") + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, limitsMutationConfirmedContext(ctx, gjson.Parse(limitsTestLimit), limitsChange{Kind: "limit_set", Amount: 10000}, false), context.Canceled) +} diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go new file mode 100644 index 00000000..24b6d5fd --- /dev/null +++ b/pkg/custom/limits_workspace.go @@ -0,0 +1,851 @@ +package custom + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "charm.land/bubbles/v2/viewport" + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/colorprofile" + "github.com/charmbracelet/x/ansi" + "github.com/charmbracelet/x/term" + "github.com/openai/openai-cli/internal/clihelp" + clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-go/v3" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +type limitsWorkResult struct { + snapshot limitsSnapshot + value gjson.Result + err error + mutation bool + operation string +} +type limitsMenuItem struct { + label, kind string + value gjson.Result +} + +type limitsWorkspace struct { + detailsHeading string + previewExecutable string + invocation string + lastSavedOperation, workerSavedOperation string + mutationError error + workerSaved gjson.Result + ctx context.Context + cancel context.CancelFunc + backend limitsBackend + output *listNavigationOutput + snapshot limitsSnapshot + width, height, section, selection, focus int + page, note, formError string + details string + busy, quitting, interrupted, locked, apply bool + refreshRequired, refreshingDraft, color, dark bool + lastSaved gjson.Result + draft limitsDraft + change limitsChange + viewport viewport.Model + requestArgs []string + omitPreview bool + shell string + work sync.WaitGroup + mu sync.Mutex + workerSnapshot *limitsSnapshot + workerError error + mutationPending bool +} + +type limitsInterrupt struct{ error } + +func (*limitsInterrupt) ExitCode() int { return 130 } +func (e *limitsInterrupt) Unwrap() error { return e.error } + +func runLimitsWorkspace(ctx context.Context, cmd *cli.Command, backend limitsBackend, opts ShowJSONOpts) error { + file := opts.Stdout.(*os.File) + width, height, err := term.GetSize(file.Fd()) + if err != nil { + return err + } + programContext, stop := context.WithCancel(ctx) + defer stop() + m := &limitsWorkspace{ctx: programContext, cancel: stop, backend: backend, width: width, height: height, page: "home", + color: os.Getenv("NO_COLOR") == "", dark: true, + viewport: viewport.New(viewport.WithWidth(max(1, width)), viewport.WithHeight(max(1, height-5)))} + m.viewport.SoftWrap, m.viewport.FillHeight = false, false + m.requestArgs, m.omitPreview = fileReceiptRequestOptions(cmd.Root()) + m.shell = imagePickerParentShell(ctx) + m.invocation = clihelp.DisplayInvocation(cmd.Root()) + m.previewExecutable, err = os.Executable() + if err != nil || strings.HasPrefix(m.invocation, "go run ") { + m.omitPreview = true + } + m.output = &listNavigationOutput{terminalOutputWriter: terminalOutputWriter{ + Writer: clioutput.Writer{Context: context.Background(), Sink: opts.Stdout}, file: file}, stop: stop} + options := []tea.ProgramOption{tea.WithInput(cmd.Root().Reader), tea.WithOutput(m.output), tea.WithContext(programContext), tea.WithWindowSize(width, height)} + if !m.color { + options = append(options, tea.WithColorProfile(colorprofile.NoTTY)) + } + program := tea.NewProgram(m, options...) + _, runErr := program.Run() + stop() + m.work.Wait() + return finishLimitsWorkspace(ctx, m, runErr, opts) +} + +// Finalization retains independent operation, cancellation, and output failures. +// A broken output pipe must not hide an uncertain mutation or failed source. +func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error, opts ShowJSONOpts) error { + m.mu.Lock() + workerErr, pending, mutationErr := m.workerError, m.mutationPending, m.mutationError + if m.workerSnapshot != nil { + m.snapshot = *m.workerSnapshot + } + if m.workerSaved.Exists() { + m.lastSaved, m.lastSavedOperation = m.workerSaved, m.workerSavedOperation + } + m.mu.Unlock() + var causes []error + removedLimitErr := m.removedLimitFailure() + ignored := []error{removedLimitErr} + if ctx.Err() == nil && !m.interrupted { + ignored = append(ignored, context.Canceled) + } + sourcesIncomplete := false + for _, failure := range m.snapshot.Errors { + if !limitsOnlyErrors(failure.Err, ignored...) { + sourcesIncomplete = true + causes = append(causes, failure.Err) + } + } + // Load returns an aggregate of the recorded source errors. Keep each source + // once, and do not reintroduce cancellation caused by normal workspace exit. + accounted := append([]error{}, ignored...) + for _, failure := range m.snapshot.Errors { + accounted = append(accounted, failure.Err) + } + if limitsOnlyErrors(workerErr, accounted...) { + workerErr = nil + } + // Closing the workspace cancels its worker. That internal cancellation is + // not a failed read. Preserve the caller's cancellation separately below. + if workerErr != nil { + causes = append(causes, workerErr) + } + if m.refreshRequired || pending || m.locked { + causes = append(causes, mutationErr) + } + incomplete := m.refreshRequired || sourcesIncomplete || workerErr != nil + message := "" + if pending || m.locked { + message = "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + } else if m.refreshRequired { + message = limitsPreflightHint(mutationErr, m.invocation) + if m.lastSaved.Exists() { + message += "\nA preceding change was confirmed. The latest change was not sent." + } + } else if incomplete { + message = "The limits overview is incomplete. Refresh before another change." + if m.lastSaved.Exists() { + message = "A change was confirmed, but the overview refresh failed. Inspect the resource before another change." + } + } + var outputErr error + if m.output != nil { + outputErr = m.output.Err() + } + interrupted := m.interrupted || errors.Is(runErr, tea.ErrInterrupted) + if ctx.Err() != nil { + causes = append(causes, ctx.Err()) + if message == "" { + message = "The limits workspace was canceled or timed out." + if m.lastSaved.Exists() { + message += " A preceding change was confirmed. Inspect the resource before another change." + } + } + } + if interrupted { + causes = append(causes, context.Canceled) + if message == "" { + message = "Limits workspace interrupted." + if m.lastSaved.Exists() { + message += " A preceding change was confirmed. Inspect the resource before another change." + } + } + } + // The output guard stops Tea after a sink failure. Preserve independent + // program failures, without treating that expected stop as another failure. + programErr := runErr + var expectedStops []error + if m.quitting || interrupted || outputErr != nil || ctx.Err() != nil { + expectedStops = append(expectedStops, tea.ErrProgramKilled, context.Canceled) + } + if interrupted { + expectedStops = append(expectedStops, tea.ErrInterrupted) + } + if limitsOnlyErrors(programErr, expectedStops...) { + programErr = nil + } + if programErr != nil { + causes = append(causes, programErr) + } + operationErr := errors.Join(causes...) + if message == "" && operationErr != nil { + message = "The limits workspace could not complete. Inspect the current settings before another change." + } + if outputErr != nil || interrupted || ctx.Err() != nil || pending || m.locked || programErr != nil { + if message != "" { + operationErr = &limitsWorkflowFailure{message: message, cause: operationErr} + } + failure := errors.Join(operationErr, outputErr) + if interrupted { + return &limitsInterrupt{failure} + } + return failure + } + var receiptErr error + if m.lastSaved.Exists() { + opts.Operation, opts.OutputKind = m.lastSavedOperation, OutputResponse + opts.Title = "Last confirmed result" + receiptErr = ShowJSON(m.lastSaved, opts) + } + if incomplete { + return errors.Join(&limitsWorkflowFailure{message: message, cause: operationErr}, receiptErr) + } + if receiptErr != nil { + return receiptErr + } + if removedLimitErr != nil && resolvedOutputFormat(opts) == "text" { + _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString("Removal confirmed. Current hard-limit configuration was not returned (HTTP 404).\n") + return err + } + if m.lastSaved.Exists() { + return nil + } + _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString("Limits workspace closed.\n") + return err +} + +func (m *limitsWorkspace) readRecovery() string { + invocation := m.invocation + if invocation == "" { + invocation = "openai" + } + resource := "admin organization " + if m.snapshot.ProjectID != "" { + resource += "projects " + } + switch m.change.Kind { + case "alert_create", "alert_update", "alert_delete": + resource += "spend-alerts list" + case "rate_update": + resource += "rate-limits list-rate-limits" + default: + resource += "spend-limit retrieve" + } + return "Read options: " + invocation + " help " + resource + "\nKeep the original connection, authentication, and scope options." +} + +func (m *limitsWorkspace) Init() tea.Cmd { return tea.Batch(m.load(), tea.RequestBackgroundColor) } + +func (m *limitsWorkspace) launch(change *limitsChange) tea.Cmd { + m.busy = true + result := make(chan limitsWorkResult, 1) + operation := "" + if change != nil { + operation = limitsChangeOperation(change.Kind, m.snapshot.ProjectID != "") + } + m.work.Add(1) + if change != nil { + m.mu.Lock() + m.mutationPending = true + m.mu.Unlock() + } + go func() { + defer m.work.Done() + var r limitsWorkResult + if change == nil { + r.snapshot, r.err = m.backend.Load(m.ctx) + } else { + r.mutation, r.operation = true, operation + r.value, r.err = m.backend.Mutate(m.ctx, *change) + } + m.mu.Lock() + m.workerError = r.err + if !r.mutation { + m.workerSnapshot = &r.snapshot + } + if r.mutation && r.err == nil { + m.workerSaved, m.workerSavedOperation = r.value, r.operation + } + if r.mutation { + m.mutationError = r.err + } + var rejected *limitsMutationFailure + if r.mutation && (r.err == nil || errors.As(r.err, &rejected) && !rejected.Attempted) { + m.mutationPending = false + } + m.mu.Unlock() + result <- r + }() + return func() tea.Msg { return <-result } +} + +func (m *limitsWorkspace) load() tea.Cmd { return m.launch(nil) } +func (m *limitsWorkspace) small() bool { return m.width < 40 || m.height < 12 } + +func (m *limitsWorkspace) reconcileDraft() { + change := m.draft.change + var current gjson.Result + switch change.Kind { + case "limit_set", "limit_delete": + current = m.snapshot.Limit + if change.Kind == "limit_set" && (m.snapshot.LimitAbsent || m.snapshot.LimitUnconfirmed) && m.limitCanBeSet() { + change.Before, change.BeforeMissing = gjson.Result{}, true + m.draft.change, m.refreshRequired, m.formError = change, false, "Baseline refreshed. Review the before and after values." + return + } + case "alert_create": + m.refreshRequired, m.formError = false, "Review the draft before creating the alert." + return + case "alert_update", "alert_delete": + for _, record := range m.snapshot.Alerts { + if record.Get("id").String() == change.ID { + if current.Exists() { + m.formError = "Duplicate alert IDs. Inspect with the API command." + return + } + current = record + } + } + case "rate_update": + for _, record := range m.snapshot.Rates { + if record.Get("id").String() == change.ID { + if current.Exists() { + m.formError = "Duplicate rate IDs. Inspect with the API command." + return + } + current = record + } + } + } + if !current.Exists() || limitsEditableBaseline(current, change.Kind) != nil { + m.formError = "Cannot refresh this draft. Inspect the resource with the API command." + return + } + change.Before, change.BeforeMissing = current, false + m.draft.change, m.refreshRequired, m.formError = change, false, "Baseline refreshed. Review the before and after values." +} + +func (m *limitsWorkspace) menu() []limitsMenuItem { + var rows []limitsMenuItem + switch m.section { + case 0: + if m.limitCanBeSet() { + rows = append(rows, limitsMenuItem{label: "Set hard spend limit", kind: "limit_set", value: m.snapshot.Limit}) + if m.snapshot.Limit.Exists() { + rows = append(rows, limitsMenuItem{label: "Remove hard spend limit", kind: "limit_delete", value: m.snapshot.Limit}) + } + } + case 1: + rows = append(rows, limitsMenuItem{label: "Create spend alert", kind: "alert_create"}) + for _, alert := range m.snapshot.Alerts { + rows = append(rows, limitsMenuItem{label: "Edit " + alert.Get("id").String() + " " + limitsThresholdLabel(alert), kind: "alert_update", value: alert}) + } + case 2: + for _, rate := range m.snapshot.Rates { + rows = append(rows, limitsMenuItem{label: rate.Get("model").String() + " " + rate.Get("id").String(), kind: "rate_update", value: rate}) + } + } + return rows +} + +func (m *limitsWorkspace) begin(kind string, value gjson.Result) { + if m.locked { + m.note = "Writes are locked after a failed save. Inspect the resource before reopening." + return + } + for _, failure := range m.snapshot.Errors { + if failure.Resource == "project" { + m.note = "Project scope unavailable. Refresh: r" + return + } + } + change := limitsChange{Kind: kind, ID: value.Get("id").String(), Before: value, BeforeMissing: kind == "limit_set" && (m.snapshot.LimitAbsent || m.snapshot.LimitUnconfirmed)} + if kind == "limit_set" || kind == "limit_delete" { + change.ID = "" + } + draft, err := newLimitsDraft(change) + if err != nil { + m.note = "This record cannot be edited safely. Use i to inspect the original fields." + return + } + m.draft, m.change, m.focus, m.formError = draft, change, 0, "" + if len(draft.inputs) == 0 { + m.review() + return + } + m.page = "form" + m.setFocus(0) +} + +func (m *limitsWorkspace) setFocus(next int) { + for i := range m.draft.inputs { + m.draft.inputs[i].Blur() + } + count := len(m.draft.inputs) + 1 + if len(m.draft.inputs) == 3 { + count++ + } + m.focus = (next + count) % count + if m.focus < len(m.draft.inputs) { + m.draft.inputs[m.focus].Focus() + } +} + +func (m *limitsWorkspace) review() { + if m.refreshRequired { + m.formError = "Refresh with r before reviewing this draft." + return + } + change, err := m.draft.validate() + if err != nil { + m.formError = err.Error() + return + } + m.change, m.page, m.apply, m.formError = change, "review", false, "" + m.setDetails(limitsReview(change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable)) +} + +func (m *limitsWorkspace) setDetails(content string) { + m.details = content + m.viewport.SetWidth(max(1, m.width-2)) + m.viewport.SetHeight(max(1, m.height-5)) + m.viewport.SetContent(ansi.Hardwrap(content, max(1, m.width-2), true)) + m.viewport.GotoTop() +} + +func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { + if m.quitting { + return m, tea.Quit + } + switch msg := message.(type) { + case tea.BackgroundColorMsg: + m.dark = msg.IsDark() + return m, nil + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + m.apply = false + if m.page == "review" { + m.setDetails(limitsReview(m.change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable)) + } + if m.page == "details" { + m.setDetails(m.details) + } + m.viewport.SetWidth(max(1, msg.Width-2)) + m.viewport.SetHeight(max(1, msg.Height-5)) + return m, nil + case limitsWorkResult: + m.busy = false + if msg.mutation { + if msg.err != nil { + var rejected *limitsMutationFailure + m.locked = !(errors.As(msg.err, &rejected) && !rejected.Attempted) + m.refreshRequired, m.apply, m.page = !m.locked, false, "form" + m.formError = "Save not confirmed. Press ? for recovery. Exit before another change." + if !m.locked { + m.formError = limitsPreflightHint(rejected.Err, m.invocation) + } + return m, nil + } + m.lastSaved, m.lastSavedOperation = msg.value, msg.operation + m.page, m.note, m.selection = "home", "Saved. Refreshing the overview.", 0 + return m, m.load() + } + m.snapshot = msg.snapshot + m.note = "Website controls and source errors: ?" + if m.lastSaved.Exists() { + m.note = "Change confirmed. Last result prints when you exit." + } + if msg.err != nil || len(msg.snapshot.Errors) != 0 { + m.note = "Some sources failed. Press ? for recovery; r to refresh." + if expected := m.removedLimitFailure(); expected != nil && len(msg.snapshot.Errors) == 1 && limitsOnlyErrors(msg.err, expected) { + m.note = "Removal confirmed. Current limit not returned." + } + } + if m.refreshingDraft { + m.refreshingDraft = false + m.reconcileDraft() + } + m.selection = min(m.selection, max(0, len(m.menu())-1)) + return m, nil + case tea.KeyPressMsg: + key := msg.String() + if key == "ctrl+c" || (m.page == "home" && (key == "q" || key == "esc")) { + m.interrupted, m.quitting = key == "ctrl+c", true + m.cancel() + return m, tea.Quit + } + if m.busy { + return m, nil + } + if key == "esc" { + m.page, m.apply = "home", false + return m, nil + } + if m.small() { + return m, nil + } + if key == "f1" || key == "?" && m.page != "form" || key == "?" && m.locked { + m.page, m.detailsHeading = "details", "Limits workspace help" + m.setDetails(m.help()) + return m, nil + } + switch m.page { + case "home": + switch key { + + case "r": + m.note = "Refreshing…" + return m, m.load() + case "left", "right", "tab", "shift+tab": + count, delta := 2, 1 + if m.snapshot.ProjectID != "" { + count = 3 + } + if key == "left" || key == "shift+tab" { + delta = -1 + } + m.section, m.selection = (m.section+count+delta)%count, 0 + case "up": + m.selection = max(0, m.selection-1) + case "down": + m.selection = min(max(0, len(m.menu())-1), m.selection+1) + case "enter", "d", "i": + rows := m.menu() + if len(rows) == 0 { + return m, nil + } + row := rows[m.selection] + if key == "i" { + if row.value.Exists() { + m.page, m.detailsHeading = "details", "Original API record" + m.setDetails(limitsSafeJSON(row.value)) + } + } else if key == "d" { + if row.kind == "alert_update" { + m.begin("alert_delete", row.value) + } + } else { + m.begin(row.kind, row.value) + } + } + case "form": + if key == "r" && m.refreshRequired { + m.refreshingDraft = true + return m, m.load() + } + switch key { + case "tab": + m.setFocus(m.focus + 1) + return m, nil + case "shift+tab": + m.setFocus(m.focus - 1) + return m, nil + case "enter": + if m.focus == len(m.draft.inputs) && len(m.draft.inputs) == 3 { + m.draft.prefixMode = (m.draft.prefixMode + 1) % 3 + } else if m.focus >= len(m.draft.inputs) { + if !m.locked { + m.review() + } + } else { + m.setFocus(m.focus + 1) + } + return m, nil + } + case "review": + if key == "tab" || key == "shift+tab" { + if m.viewport.AtBottom() { + m.apply = !m.apply + } + return m, nil + } + if key == "enter" { + if !m.apply { + m.page = "form" + m.setFocus(0) + return m, nil + } + if m.viewport.AtBottom() && !m.locked && m.ctx.Err() == nil && (m.output == nil || m.output.Err() == nil) { + m.apply = false + change := m.change + return m, m.launch(&change) + } + return m, nil + } + } + } + if m.page == "review" || m.page == "details" { + var command tea.Cmd + m.viewport, command = m.viewport.Update(message) + if !m.viewport.AtBottom() { + m.apply = false + } + return m, command + } + if m.page == "form" && m.focus < len(m.draft.inputs) && !m.small() { + var command tea.Cmd + m.draft.inputs[m.focus], command = m.draft.inputs[m.focus].Update(message) + return m, command + } + return m, nil +} + +func limitsThresholdLabel(value gjson.Result) string { + amount, err := limitsInteger(value.Get("threshold_amount")) + if err == nil && limitsUniqueJSON(value) == nil && value.Get("currency").String() == "USD" && value.Get("interval").String() == "month" { + return "USD " + formatLimitsDollars(amount) + " / month" + } + return "Inspect raw units" +} + +func (m *limitsWorkspace) heading() string { + if m.page == "form" || m.page == "review" { + return limitsChangeTitle(m.change.Kind) + } + if m.page == "details" { + if m.detailsHeading != "" { + return m.detailsHeading + } + return "Original API record" + } + return "Limits" +} + +func (m *limitsWorkspace) scope() string { + if m.snapshot.ProjectID != "" { + return "Project " + m.snapshot.ProjectID + } + return "Organization" +} + +func (m *limitsWorkspace) sourceSummary() string { + switch m.section { + case 0: + if m.snapshot.LimitUnconfirmed { + if m.removedLimitFailure() != nil { + return "Removal confirmed. Current limit unknown." + } + return "Hard limit unknown. Review before replacing." + } + if !m.limitReadable() { + return "Hard spend limit unavailable. Refresh: r" + } + if m.snapshot.LimitAbsent { + return "No hard spend limit configured." + } + if m.snapshot.Limit.Exists() { + return "Hard limit: " + limitsThresholdLabel(m.snapshot.Limit) + } + return "Hard spend limit unavailable. Refresh to retry." + case 1: + return fmt.Sprintf("%d alerts loaded. Complete: %t", len(m.snapshot.Alerts), m.snapshot.AlertsComplete) + default: + return fmt.Sprintf("%d rate records loaded. Complete: %t", len(m.snapshot.Rates), m.snapshot.RatesComplete) + } +} + +func (m *limitsWorkspace) limitReadable() bool { + for _, failure := range m.snapshot.Errors { + if failure.Resource == "spend_limit" || failure.Resource == "project" { + return false + } + } + return true +} + +func limitsSafeJSON(value gjson.Result) string { return imagePickerLine(value.Raw) } + +func limitsPreflightHint(err error, invocation ...string) string { + bin := "openai" + if len(invocation) != 0 && invocation[0] != "" { + bin = invocation[0] + } + var apiErr *openai.Error + if errors.As(err, &apiErr) { + switch apiErr.StatusCode { + case 401: + return "Admin authentication failed. No write sent.\nKey setup: " + bin + " help setup admin" + case 403: + return "The API denied access. No write sent.\nCheck Admin-key permissions and organization or project access.\nAccess setup: " + bin + " help setup admin" + case 404: + return "The requested resource was not returned. No write sent.\nCheck the API address, scope, and resource ID. Press r to refresh." + case 429: + switch apiErr.Code { + case "organization_spend_limit_exceeded": + return "The API reports an organization spend limit was reached. No write sent.\nLimit options: " + bin + " help admin organization spend-limit retrieve" + case "project_spend_limit_exceeded": + return "The API reports a project spend limit was reached. No write sent.\nLimit options: " + bin + " help admin organization projects spend-limit retrieve" + case "insufficient_quota", "billing_hard_limit_reached", "credit_balance_exhausted", "organization_usage_limit_exceeded": + return "The API reports a billing or usage limit. No write sent.\nCheck billing and approved usage: https://platform.openai.com/settings/organization/limits" + default: + return "The API returned HTTP 429. No write sent.\nWait before refreshing. Check API error details if this continues." + } + } + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "The check was interrupted. No write sent. Press r to refresh before trying again." + } + if err == nil { + return "No change sent. Press r to refresh this draft's baseline." + } + // Only exact locally defined messages select specific recovery guidance. + switch err.Error() { + case "Rate-limit editing requires a complete list. Reopen with a larger --max-items value or -1.": + return "No write sent. Reopen with --max-items -1 to check every rate record." + case "The configuration changed. Refresh before applying this change.": + return "Record changed. No write sent. Press r to refresh and review the new baseline." + default: + return "No change sent. Press r to refresh this draft's baseline." + } +} + +func (m *limitsWorkspace) limitCanBeSet() bool { + for _, failure := range m.snapshot.Errors { + if failure.Resource == "project" || failure.Resource == "spend_limit" && !m.snapshot.LimitUnconfirmed { + return false + } + } + return m.snapshot.Limit.Exists() || m.snapshot.LimitAbsent || m.snapshot.LimitUnconfirmed +} + +func (m *limitsWorkspace) costsSummary() string { + if m.snapshot.CostsTotal == "" { + return "Current-month Costs unavailable. ?: details" + } + status := "complete" + if !m.snapshot.CostsComplete { + status = "incomplete" + } + return "Costs: USD " + m.snapshot.CostsTotal + " (" + status + ")" +} + +func (m *limitsWorkspace) help() string { + invocation := m.invocation + if invocation == "" { + invocation = "openai" + } + text := "Spend manages hard monthly USD limits.\nAlerts notify recipients; they do not enforce caps.\nRates manages six project rate fields.\n\nEvery change requires a before/after review.\nRead to the end, then Tab to Apply.\nBack remains the default action.\n\nUse r to refresh. Use i to inspect original fields.\nUse --max-items -1 to load complete lists.\n\n" + if m.locked { + m.mu.Lock() + failure := m.mutationError + m.mu.Unlock() + text = "The latest save was not confirmed. It may have succeeded.\nWrites remain locked, including after refresh.\n" + m.readRecovery() + "\n" + limitsMutationHint(failure, invocation) + "\n\n" + text + } else if m.refreshRequired { + m.mu.Lock() + failure := m.mutationError + m.mu.Unlock() + text = limitsPreflightHint(failure, invocation) + "\n\n" + text + } + for _, failure := range m.snapshot.Errors { + // Source names come from our loader, never response text. + text += "Source unavailable: " + failure.Resource + "\n" + limitsReadHint(failure.Err, invocation) + "\n\n" + } + if len(m.snapshot.Errors) != 0 { + route := "admin limits show" + if m.snapshot.ProjectID != "" { + route = "projects limits show" + } + text += "Overview options: " + invocation + " help " + route + "\nKeep the original connection, authentication, and scope options.\nUse --format json to inspect available records and source_errors.\n\n" + } + if m.snapshot.LimitUnconfirmed { + text += "The current hard limit could not be confirmed.\nSet hard spend limit requires review and can create or replace a limit.\n\n" + } + if m.snapshot.CostsStart != 0 && m.snapshot.CostsEnd != 0 { + text += m.costsSummary() + "\nQueried period: " + time.Unix(m.snapshot.CostsStart, 0).UTC().Format(time.RFC3339) + " to " + time.Unix(m.snapshot.CostsEnd, 0).UTC().Format(time.RFC3339) + " (exclusive).\nCosts can lag and do not determine enforcement state.\n\n" + } + text += "Soft budgets, independent enforcement settings, account tiers, credits, and organization rate catalogs need the website:\nhttps://platform.openai.com/settings/organization/limits\n\nThe public API exposes absolute-value alerts, not website percentage alerts or triggered status.\nRemoving a hard limit stops that cap; inspect soft budgets on the website.\n\nHistorical Costs command:\n" + invocation + " help admin usage costs\n\nThe workspace checks current state before saving. This check is not atomic.\nThe workspace sends each confirmed mutation once, without automatic retries.\nDirect API commands omit these safeguards and can retry.\n" + return text +} + +func limitsReadHint(err error, invocation string) string { + // Read failures have no pending mutation. Reuse the classification without + // presenting a failed read as a failed save. + var summary *limitsCostSummaryError + if errors.As(err, &summary) { + return summary.Error() + "\nCosts options: " + invocation + " help admin usage costs" + } + var apiErr *openai.Error + if errors.As(err, &apiErr) && (apiErr.StatusCode == 401 || apiErr.StatusCode == 403 || apiErr.StatusCode == 404 || apiErr.StatusCode == 429) { + return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "No change was attempted.") + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "The read was interrupted. Press r to refresh." + } + return "The source could not be read. Check the API address and access, then refresh." +} + +func limitsMutationHint(err error, invocation string) string { + var apiErr *openai.Error + if errors.As(err, &apiErr) && (apiErr.StatusCode == 401 || apiErr.StatusCode == 403 || apiErr.StatusCode == 404 || apiErr.StatusCode == 429) { + return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "The change is not confirmed.") + } + if err != nil { + switch err.Error() { + case "The API returned different values from the reviewed change. Inspect the returned record before another change.": + return "The API returned different values from the reviewed change. Read the current record before another change." + case "The API returned an incomplete confirmation. Refresh and inspect the resource before another change.": + return "The API response did not confirm the change. Read the current record before another change." + } + } + return "The response did not confirm the change. Check the API address and connection before reading its current state." +} + +func (m *limitsWorkspace) removedLimitFailure() error { + if m.lastSavedOperation != limitsChangeOperation("limit_delete", m.snapshot.ProjectID != "") || m.lastSaved.Get("deleted").Type != gjson.True || !m.snapshot.LimitUnconfirmed { + return nil + } + for _, failure := range m.snapshot.Errors { + if failure.Resource == "spend_limit" && limitsHTTPStatus(failure.Err, 404) { + return failure.Err + } + } + return nil +} + +// Inspect each branch; errors.Is alone also matches joins with unrelated errors. +func limitsOnlyErrors(err error, expected ...error) bool { + if err == nil { + return false + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + children := joined.Unwrap() + if len(children) == 0 { + return false + } + for _, child := range children { + if !limitsOnlyErrors(child, expected...) { + return false + } + } + return true + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + return limitsOnlyErrors(wrapped.Unwrap(), expected...) + } + for _, target := range expected { + if target != nil && errors.Is(err, target) { + return true + } + } + return false +} diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go new file mode 100644 index 00000000..abc6fc28 --- /dev/null +++ b/pkg/custom/limits_workspace_test.go @@ -0,0 +1,727 @@ +package custom + +import ( + "bytes" + "context" + "errors" + "io" + "math" + "reflect" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "charm.land/bubbles/v2/viewport" + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/x/ansi" + clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-go/v3" + "github.com/tidwall/gjson" +) + +const limitsControllerAlert = `{"object":"organization.spend_alert","id":"alert_synthetic","threshold_amount":10000,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["synthetic@example.invalid"]}}` +const limitsControllerRate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":100,"max_tokens_per_1_minute":200,"max_images_per_1_minute":0}` + +type limitsControllerBackend struct { + snapshot limitsSnapshot + loadErr, mutationErr error + release <-chan struct{} + changes chan limitsChange + writes, reads atomic.Int32 +} + +func (b *limitsControllerBackend) Load(ctx context.Context) (limitsSnapshot, error) { + b.reads.Add(1) + if err := ctx.Err(); err != nil { + return limitsSnapshot{}, err + } + return b.snapshot, b.loadErr +} + +func (b *limitsControllerBackend) Mutate(ctx context.Context, change limitsChange) (gjson.Result, error) { + b.writes.Add(1) + b.changes <- change + if b.release != nil { + select { + case <-b.release: + case <-ctx.Done(): + return gjson.Result{}, ctx.Err() + } + } + return gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month"}`), b.mutationErr +} + +func newLimitsController(t *testing.T, width, height int) (*limitsWorkspace, *limitsControllerBackend) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + b := &limitsControllerBackend{snapshot: limitsSnapshot{LimitAbsent: true, AlertsComplete: true}, changes: make(chan limitsChange, 8)} + m := &limitsWorkspace{ctx: ctx, cancel: cancel, backend: b, snapshot: b.snapshot, width: width, height: height, page: "home", shell: "bash", + viewport: viewport.New(viewport.WithWidth(width-2), viewport.WithHeight(height-5))} + m.viewport.SoftWrap, m.viewport.FillHeight = false, false + t.Cleanup(func() { cancel(); m.work.Wait() }) + return m, b +} + +func limitsControllerKey(m *limitsWorkspace, code rune) tea.Cmd { + _, command := m.Update(tea.KeyPressMsg{Code: code}) + return command +} + +func limitsControllerReview(t *testing.T, m *limitsWorkspace) { + t.Helper() + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" || len(m.draft.inputs) != 1 { + t.Fatalf("limit form did not open: page=%s", m.page) + } + m.draft.inputs[0].SetValue("100.01") + limitsControllerKey(m, tea.KeyTab) + limitsControllerKey(m, tea.KeyEnter) + if m.page != "review" || m.apply || m.busy { + t.Fatalf("review did not start on Back: page=%s apply=%t busy=%t", m.page, m.apply, m.busy) + } +} + +func limitsControllerRun(t *testing.T, command tea.Cmd) limitsWorkResult { + t.Helper() + if command == nil { + t.Fatal("missing asynchronous operation") + } + result, ok := command().(limitsWorkResult) + if !ok { + t.Fatal("operation returned an unexpected message") + } + return result +} + +func limitsControllerStarted(t *testing.T, m *limitsWorkspace, b *limitsControllerBackend) limitsChange { + t.Helper() + select { + case change := <-b.changes: + return change + case <-m.ctx.Done(): + t.Fatal("confirmed mutation did not reach the backend") + return limitsChange{} + } +} + +func TestLimitsWorkspaceReviewRequiresExplicitApply(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + limitsControllerReview(t, m) + m.work.Wait() + if b.writes.Load() != 0 { + t.Fatal("opening or reviewing a draft wrote to the backend") + } + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" || b.writes.Load() != 0 { + t.Fatal("default Back action submitted the draft") + } + m.setFocus(len(m.draft.inputs)) + limitsControllerKey(m, tea.KeyEnter) + m.viewport.GotoTop() + if m.viewport.AtBottom() { + t.Fatal("fixture must require scrolling through its review") + } + limitsControllerKey(m, tea.KeyTab) + if m.apply { + t.Fatal("Apply became active before the review reached its end") + } + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + if !m.apply || b.writes.Load() != 0 { + t.Fatal("selecting Apply must not submit") + } + result := limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter)) + change := limitsControllerStarted(t, m, b) + if b.writes.Load() != 1 || change.Kind != "limit_set" || change.Amount != 10001 || !change.BeforeMissing { + t.Fatalf("incorrect confirmed mutation: %+v; writes=%d", change, b.writes.Load()) + } + _, refresh := m.Update(result) + m.Update(limitsControllerRun(t, refresh)) + if m.page != "home" || m.busy || b.reads.Load() != 1 || b.writes.Load() != 1 { + t.Fatal("successful save did not refresh once without repeating the mutation") + } +} + +func TestLimitsWorkspaceResizeClearsApplyAndBlocksTinySubmission(t *testing.T) { + for _, size := range [][2]int{{80, 24}, {40, 12}, {39, 12}, {40, 11}} { + m, b := newLimitsController(t, 80, 24) + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + m.Update(tea.WindowSizeMsg{Width: size[0], Height: size[1]}) + if m.apply { + t.Errorf("resize to %v retained Apply selection", size) + } + if m.small() { + if view := m.View().Content; !strings.Contains(view, "Resize") || !strings.Contains(view, "Ctrl+C") { + t.Errorf("tiny screen lacks resize/cancel guidance: %q", view) + } + m.apply = true // A hidden action must remain blocked even with stale focus. + if command := limitsControllerKey(m, tea.KeyEnter); command != nil || m.busy { + t.Errorf("tiny screen %v submitted an invisible action", size) + } + } else { + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" { + t.Errorf("resize to %v did not reset confirmation to Back", size) + } + } + m.work.Wait() + if b.writes.Load() != 0 { + t.Errorf("resize to %v caused a write", size) + } + } +} + +func TestLimitsWorkspaceBusyAndFailedSavePreventRepeatedWrites(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + release := make(chan struct{}) + b.release, b.mutationErr = release, errors.New("synthetic uncertain write failure") + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + command := limitsControllerKey(m, tea.KeyEnter) + limitsControllerStarted(t, m, b) + for _, key := range []rune{tea.KeyEnter, tea.KeyTab, tea.KeyEnter, 'r', tea.KeyEscape} { + if next := limitsControllerKey(m, key); next != nil { + t.Error("busy workspace started another operation") + } + } + if b.writes.Load() != 1 || !m.busy { + t.Fatal("pending write lost its submission guard") + } + close(release) + m.Update(limitsControllerRun(t, command)) + if !m.locked || m.apply || m.page != "form" || m.draft.inputs[0].Value() != "100.01" || !strings.Contains(m.formError, "inspect") { + t.Fatalf("failed save discarded draft or unlocked writes: page=%s locked=%t error=%q", m.page, m.locked, m.formError) + } + m.setFocus(len(m.draft.inputs)) + limitsControllerKey(m, tea.KeyEnter) + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, tea.KeyEnter) + m.work.Wait() + if b.writes.Load() != 1 || m.busy || !m.locked { + t.Fatal("failed save permitted an unconfirmed repeat") + } +} + +func TestLimitsWorkspaceControlCFromEveryState(t *testing.T) { + for _, page := range []string{"home", "form", "review", "details", "busy"} { + t.Run(page, func(t *testing.T) { + m, b := newLimitsController(t, 40, 12) + var pending tea.Cmd + if page == "busy" { + b.release = make(chan struct{}) + change := limitsChange{Kind: "limit_set", Amount: 10001, BeforeMissing: true} + pending = m.launch(&change) + limitsControllerStarted(t, m, b) + } else { + m.page = page + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + if quit == nil || !m.quitting || !m.interrupted || !errors.Is(m.ctx.Err(), context.Canceled) || m.View().Content != "" { + t.Fatal("Ctrl+C did not stop work and clear the owned view") + } + if pending != nil && !errors.Is(limitsControllerRun(t, pending).err, context.Canceled) { + t.Fatal("Ctrl+C did not cancel the pending backend request") + } + m.work.Wait() + }) + } +} + +func TestLimitsWorkspacePreflightFailureRequiresRefresh(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + b.mutationErr = &limitsMutationFailure{Err: errors.New("synthetic preflight conflict"), Attempted: false} + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + m.Update(limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter))) + limitsControllerStarted(t, m, b) + if m.locked || !m.refreshRequired || m.page != "form" || m.draft.inputs[0].Value() != "100.01" { + t.Fatal("preflight conflict did not retain a refreshable draft") + } + m.setFocus(len(m.draft.inputs)) + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" || m.busy || b.writes.Load() != 1 { + t.Fatal("preflight conflict allowed review before refresh") + } + b.snapshot.LimitAbsent = false + b.snapshot.Limit = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":20000,"currency":"USD","interval":"month","enforcement":{"status":"enforcing"}}`) + m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) + if m.refreshRequired || m.locked || m.draft.inputs[0].Value() != "100.01" || m.draft.change.Before.Raw != b.snapshot.Limit.Raw || m.draft.change.BeforeMissing { + t.Fatal("refresh discarded user input or failed to reconcile the baseline") + } + limitsControllerKey(m, tea.KeyEnter) + if m.page != "review" || m.apply || m.change.Before.Raw != b.snapshot.Limit.Raw || m.change.Amount != 10001 { + t.Fatal("refreshed review lost old/new distinction or selected Apply") + } + b.mutationErr = nil + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter)) + change := limitsControllerStarted(t, m, b) + if b.writes.Load() != 2 || change.Before.Raw != b.snapshot.Limit.Raw || change.Amount != 10001 { + t.Fatal("explicit retry used a stale baseline or changed the draft") + } +} + +func TestLimitsWorkspaceAttemptedFailureStaysLockedAfterRefresh(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + b.mutationErr = &limitsMutationFailure{Err: errors.New("synthetic uncertain response"), Attempted: true} + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + m.Update(limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter))) + limitsControllerStarted(t, m, b) + if !m.locked || m.refreshRequired { + t.Fatal("attempted write failure became a safe preflight rejection") + } + limitsControllerKey(m, tea.KeyEscape) + m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) + limitsControllerKey(m, tea.KeyEnter) + m.work.Wait() + if !m.locked || m.page != "home" || b.writes.Load() != 1 { + t.Fatal("refresh silently unlocked an uncertain write") + } +} + +func TestLimitsWorkspaceRefreshClearsStaleStatusAndRewrapsDetails(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.Update(limitsWorkResult{snapshot: limitsSnapshot{Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: errors.New("synthetic failure")}}}, err: errors.New("synthetic failure")}) + if !strings.Contains(m.note, "failed") { + t.Fatal("partial load did not report failure") + } + m.Update(limitsWorkResult{snapshot: limitsSnapshot{LimitAbsent: true, AlertsComplete: true}}) + if strings.Contains(m.note, "failed") { + t.Fatal("successful refresh retained stale failure status") + } + m.page = "details" + m.setDetails(strings.Repeat("界", 60)) + m.Update(tea.WindowSizeMsg{Width: 40, Height: 12}) + view := ansi.Strip(m.View().Content) + for _, line := range strings.Split(view, "\n") { + if ansi.StringWidth(line) > 40 { + t.Errorf("resized details overflowed display cells: %q", line) + } + } + if !strings.Contains(view, "Ctrl+C") { + t.Fatal("resized details hid cancellation") + } +} + +func TestLimitsWorkspaceNoLimitPreservesOtherEditBaselines(t *testing.T) { + for _, kind := range []string{"alert_update", "alert_delete", "rate_update"} { + t.Run(kind, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + value := gjson.Parse(limitsControllerAlert) + if kind == "rate_update" { + value = gjson.Parse(limitsControllerRate) + } + m.begin(kind, value) + if m.change.Kind != kind || m.change.BeforeMissing || m.change.Before.Raw != value.Raw || m.change.ID != value.Get("id").Str { + t.Fatalf("absent hard limit corrupted %s baseline: %+v", kind, m.change) + } + }) + } +} + +func TestLimitsDraftExactAmountsAndPrefixPresence(t *testing.T) { + for _, tc := range []struct { + kind, amount string + want int64 + valid bool + }{ + {"limit_set", "92233720368547758.07", math.MaxInt64, true}, + {"limit_set", "0.01", 1, true}, {"limit_set", "0", 0, false}, + {"alert_create", "0", 0, true}, {"alert_create", "1.001", 0, false}, + {"limit_set", "92233720368547758.08", 0, false}, + } { + draft, err := newLimitsDraft(limitsChange{Kind: tc.kind, BeforeMissing: true}) + if err != nil { + t.Fatal(err) + } + draft.inputs[0].SetValue(tc.amount) + if tc.kind == "alert_create" { + draft.inputs[1].SetValue("synthetic@example.invalid") + } + change, err := draft.validate() + if (err == nil) != tc.valid || tc.valid && change.Amount != tc.want { + t.Errorf("%s amount %q: cents=%d error=%v", tc.kind, tc.amount, change.Amount, err) + } + } + for _, tc := range []struct { + name, original, value string + mode int + set, null bool + }{ + {name: "omitted remains omitted"}, + {name: "null remains null", original: `,"subject_prefix":null`, set: true, null: true}, + {name: "empty remains empty", original: `,"subject_prefix":""`, set: true}, + {name: "preserve string", original: `,"subject_prefix":"Existing"`, value: "Existing", set: true}, + {name: "replace empty", original: `,"subject_prefix":"Existing"`, mode: 1, set: true}, + {name: "clear null", original: `,"subject_prefix":"Existing"`, mode: 2, set: true, null: true}, + {name: "edited value", value: "New prefix", set: true}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := strings.Replace(limitsControllerAlert, `["synthetic@example.invalid"]}`, `["synthetic@example.invalid"]`+tc.original+`}`, 1) + draft, err := newLimitsDraft(limitsChange{Kind: "alert_update", ID: "alert_synthetic", Before: gjson.Parse(raw)}) + if err != nil { + t.Fatal(err) + } + draft.prefixMode = tc.mode + draft.inputs[2].SetValue(tc.value) + change, err := draft.validate() + if err != nil || change.SubjectPrefixSet != tc.set || tc.set && (change.SubjectPrefix == nil) != tc.null { + t.Fatalf("prefix presence changed: %+v; error=%v", change, err) + } + if tc.set && !tc.null && *change.SubjectPrefix != tc.value { + t.Errorf("prefix content changed: %q", *change.SubjectPrefix) + } + }) + } +} + +func TestLimitsDraftRatesSendOnlyExplicitChanges(t *testing.T) { + draft, err := newLimitsDraft(limitsChange{Kind: "rate_update", ID: "rl_synthetic", Before: gjson.Parse(limitsControllerRate)}) + if err != nil { + t.Fatal(err) + } + if draft.inputs[2].Value() != "" || draft.inputs[3].Value() != "0" || draft.inputs[4].Value() != "" { + t.Fatal("draft conflated missing rates with explicit zero") + } + draft.inputs[0].SetValue("") // Blank retains the existing value. + draft.inputs[1].SetValue("201") + draft.inputs[4].SetValue("0") // Explicit zero remains a real change. + change, err := draft.validate() + want := map[string]int64{"max_tokens_per_1_minute": 201, "max_audio_megabytes_per_1_minute": 0} + if err != nil || !reflect.DeepEqual(change.RateValues, want) { + t.Fatalf("rate changes=%v error=%v; want %v", change.RateValues, err, want) + } + for _, tc := range []struct { + text string + value int64 + }{{"-1", -1}, {"-9223372036854775808", math.MinInt64}} { + draft.inputs[1].SetValue(tc.text) + change, err := draft.validate() + if err != nil || change.RateValues["max_tokens_per_1_minute"] != tc.value { + t.Errorf("signed rate %q was not preserved: %v, %v", tc.text, change.RateValues, err) + } + } + for _, invalid := range []string{"-", "1.5", "1e3", "9223372036854775808", "-9223372036854775809"} { + draft.inputs[1].SetValue(invalid) + if _, err := draft.validate(); err == nil { + t.Errorf("rate draft accepted %q", invalid) + } + } +} + +func TestLimitsWorkspaceActionVisibleAtMinimumSize(t *testing.T) { + for _, kind := range []string{"limit_set", "alert_create", "rate_update"} { + t.Run(kind, func(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + value := gjson.Result{} + if kind == "rate_update" { + value = gjson.Parse(limitsControllerRate) + } + m.begin(kind, value) + for focus := 0; focus <= len(m.draft.inputs); focus++ { + m.setFocus(focus) + view := ansi.Strip(m.View().Content) + if !strings.Contains(view, "Review change") || !strings.Contains(view, "Ctrl+C") { + t.Errorf("%s focus %d hides action or cancel: %q", kind, focus, view) + } + if lines := strings.Split(view, "\n"); len(lines) > 12 { + t.Errorf("minimum view uses %d rows", len(lines)) + } else { + for _, line := range lines { + if ansi.StringWidth(line) > 40 { + t.Errorf("minimum view exceeds 40 columns: %q", line) + } + } + } + } + if kind == "rate_update" { + m.draft.inputs[0].SetValue("101") + } else { + m.draft.inputs[0].SetValue("100.01") + if kind == "alert_create" { + m.draft.inputs[1].SetValue("synthetic@example.invalid") + } + } + m.review() + if m.page != "review" { + t.Fatalf("fixture could not enter review: %s", m.formError) + } + m.viewport.GotoBottom() + view := ansi.Strip(m.View().Content) + if !strings.Contains(view, "> Back") || !strings.Contains(view, "Apply change") || !strings.Contains(view, "Ctrl+C") { + t.Errorf("minimum review hides confirmation: %q", view) + } + }) + } +} + +func TestLimitsDraftCommandPreviewPreservesPublicRoutes(t *testing.T) { + for _, tc := range []struct { + change limitsChange + wants []string + }{ + {limitsChange{Kind: "limit_set", Amount: 10001, BeforeMissing: true}, []string{"spend-limit update", "--currency USD", "--threshold-amount 10001"}}, + {limitsChange{Kind: "alert_update", ID: "alert_synthetic", Amount: 10001, Recipients: []string{"synthetic@example.invalid"}}, []string{"spend-alerts update", "--alert-id alert_synthetic", "--currency USD"}}, + {limitsChange{Kind: "rate_update", ID: "rl_synthetic", RateValues: map[string]int64{"max_requests_per_1_minute": 0}}, []string{"rate-limits update-rate-limit", "--rate-limit-id rl_synthetic", "--max-requests-per-1-minute 0"}}, + } { + preview := limitsReview(tc.change, "proj_synthetic", nil, false, "bash") + for _, want := range append(tc.wants, "openai admin organization projects", "--project-id proj_synthetic") { + if !strings.Contains(preview, want) { + t.Errorf("preview missing %q: %q", want, preview) + } + } + if tc.change.Kind != "rate_update" && limitsChangeBody(tc.change)["currency"] != "USD" { + t.Error("review body changed API currency spelling") + } + } +} + +func TestLimitsDraftPreviewNamesDirectCommandDifferences(t *testing.T) { + change := limitsChange{Kind: "limit_set", Amount: 10001, BeforeMissing: true} + for _, shell := range []string{"bash", "zsh", "fish", "pwsh"} { + t.Run(shell, func(t *testing.T) { + preview := limitsReview(change, "", nil, false, shell, "/synthetic path/openai") + for _, want := range []string{"Direct API command", "skips the workspace preflight", "can retry automatically", "creates or replaces"} { + if !strings.Contains(preview, want) { + t.Errorf("preview omitted %q", want) + } + } + if strings.Contains(preview, "Equivalent command") || strings.Contains(preview, "Not configured") { + t.Fatal("preview claimed unsupported equivalence or confirmed absence") + } + command := strings.SplitN(strings.SplitN(preview, "Direct API command\n", 2)[1], "\n", 2)[0] + if shell == "pwsh" && !strings.HasPrefix(command, "& '/synthetic path/openai' ") { + t.Fatalf("PowerShell preview cannot invoke the quoted executable: %q", command) + } + }) + } + deletion := limitsReview(limitsChange{Kind: "limit_delete"}, "", nil, true, "bash") + if strings.Contains(deletion, "does not preserve a disabled budget") || !strings.Contains(deletion, "does not guarantee soft-budget behavior") { + t.Fatal("hard-limit removal claimed unverified soft-budget semantics") + } +} + +func TestLimitsWorkspaceUnknownLimitRequiresExplicitReview(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + m.snapshot = limitsSnapshot{LimitUnconfirmed: true, Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: &openai.Error{StatusCode: 404}}}} + if rows := m.menu(); len(rows) != 1 || rows[0].kind != "limit_set" || !strings.Contains(m.sourceSummary(), "unknown") { + t.Fatal("unknown limit lost its explicit create-or-replace path or became confirmed absence") + } + limitsControllerReview(t, m) + if !m.change.BeforeMissing || m.apply || !strings.Contains(m.details, "The current limit could not be confirmed.") || b.writes.Load() != 0 { + t.Fatal("unknown-baseline change bypassed its warning and Back default") + } + m.page = "home" + m.snapshot.Errors = append(m.snapshot.Errors, limitsSourceFailure{Resource: "project", Err: errors.New("synthetic project failure")}) + if len(m.menu()) != 0 { + t.Fatal("unverified project scope allowed a limit change") + } + m.snapshot = limitsSnapshot{Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: &openai.Error{StatusCode: 401}}}} + if len(m.menu()) != 0 { + t.Fatal("an authentication failure allowed an assumed-absent limit change") + } +} + +func TestLimitsPreflightGuidancePreservesFailureScope(t *testing.T) { + for _, tc := range []struct { + status int + code, want, reject string + }{ + {401, "", "help setup admin", "help setup\n"}, + {403, "", "Check Admin-key permissions", "invalid key"}, + {404, "", "was not returned", "not configured"}, + {429, "organization_spend_limit_exceeded", "help admin organization spend-limit retrieve", "Wait before"}, + {429, "project_spend_limit_exceeded", "help admin organization projects spend-limit retrieve", "Wait before"}, + {429, "credit_balance_exhausted", "billing or usage limit", "Rate limited"}, + {429, "unknown_synthetic_code", "Wait before refreshing", "spend limit was reached"}, + } { + err := &openai.Error{StatusCode: tc.status, Code: tc.code, Message: "private synthetic response prose"} + got := limitsPreflightHint(err, "custom-openai") + if !strings.Contains(got, tc.want) || strings.Contains(got, tc.reject) || strings.Contains(got, err.Message) || !strings.Contains(got, "No write sent.") { + t.Errorf("status=%d code=%s yielded unsafe or incorrect recovery: %q", tc.status, tc.code, got) + } + } +} + +func TestLimitsWorkspaceRecoveryAccessibleFromLockedForm(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + m.begin("alert_create", gjson.Result{}) + m.locked = true + m.change.Kind = "alert_create" + m.formError = "Save not confirmed. Press ? for recovery. Exit before another change." + if !strings.Contains(m.View().Content, "F1: help") { + t.Fatal("minimum-size form hid the recovery action") + } + limitsControllerKey(m, tea.KeyF1) + if m.page != "details" || !strings.Contains(m.details, "It may have succeeded") || !strings.Contains(m.details, "help admin organization spend-alerts list") { + t.Fatal("recovery help did not explain uncertainty and a safe read") + } + if !strings.Contains(m.View().Content, "Ctrl+C") { + t.Fatal("recovery help hid cancellation") + } +} + +type limitsFailingWriter struct{ err error } + +func (w limitsFailingWriter) Write([]byte) (int, error) { return 0, w.err } + +func TestLimitsWorkspaceFinalizationPreservesIndependentFailures(t *testing.T) { + for _, scenario := range []string{"read and terminal output", "mutation and terminal output", "cancellation and terminal output", "refresh and receipt output", "program and terminal output"} { + t.Run(scenario, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + ctx := t.Context() + original := errors.New("private synthetic operation failure") + writeErr := &clioutput.WriteError{Err: syscall.EPIPE} + opts := ShowJSONOpts{Context: ctx, Stdout: io.Discard, Stderr: io.Discard, Format: "text"} + var runErr error + switch scenario { + case "program and terminal output": + runErr = errors.Join(original, tea.ErrProgramKilled) + m.output = &listNavigationOutput{err: writeErr} + case "read and terminal output": + m.workerError = original + m.output = &listNavigationOutput{err: writeErr} + case "mutation and terminal output": + m.mutationPending = true + m.mutationError = original + m.output = &listNavigationOutput{err: writeErr} + case "cancellation and terminal output": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + original = context.Canceled + m.interrupted = true + m.output = &listNavigationOutput{err: writeErr} + case "refresh and receipt output": + m.snapshot.Errors = []limitsSourceFailure{{Resource: "spend_alerts", Err: original}} + m.lastSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month"}`) + m.lastSavedOperation = limitsChangeOperation("limit_set", false) + opts.Stdout = limitsFailingWriter{err: syscall.EPIPE} + } + err := finishLimitsWorkspace(ctx, m, runErr, opts) + if !errors.Is(err, original) || !errors.Is(err, syscall.EPIPE) || clioutput.IsBrokenPipe(err) { + t.Fatalf("output failure hid an independent outcome: %v", err) + } + var safe *limitsWorkflowFailure + if !errors.As(err, &safe) || strings.Contains(safe.Error(), "private synthetic") || strings.Contains(safe.Error(), "shown above") { + t.Fatal("finalization lost safe outcome guidance or asserted an undisplayed receipt") + } + if scenario == "cancellation and terminal output" { + var interrupt *limitsInterrupt + if !errors.As(err, &interrupt) || interrupt.ExitCode() != 130 { + t.Fatal("output failure changed the interrupt status") + } + } + }) + } +} + +func TestLimitsWorkspaceReceiptUsesMutationProjection(t *testing.T) { + for _, format := range []string{"text", "json"} { + t.Run(format, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month","enforcement":{"status":"inactive"}}`) + m.workerSavedOperation = limitsChangeOperation("limit_set", false) + var out bytes.Buffer + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: format, Stdout: &out, Stderr: io.Discard}) + if err != nil { + t.Fatal(err) + } + if format == "text" { + if !strings.Contains(out.String(), "USD 100.01") || !strings.Contains(out.String(), "inactive") { + t.Fatalf("receipt lost threshold units or returned state: %q", out.String()) + } + } else if !strings.Contains(out.String(), `"threshold_amount": 10001`) || strings.Contains(out.String(), "spend_threshold") { + t.Fatalf("machine receipt changed threshold data: %q", out.String()) + } + }) + } +} + +func TestLimitsWorkspaceCostsAndEnforcementStayDistinct(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.snapshot.Limit = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month","enforcement":{"status":"inactive"}}`) + m.snapshot.LimitAbsent = false + m.snapshot.CostsTotal, m.snapshot.CostsComplete = "123.000000000000000001", false + m.snapshot.CostsStart, m.snapshot.CostsEnd = 1790812800, 1792368000 + view := m.View().Content + if !strings.Contains(view, "Costs: USD 123.000000000000000001 (incomplete)") || !strings.Contains(view, "Returned enforcement: inactive") { + t.Fatalf("workspace conflated Costs and returned enforcement: %q", view) + } + help := m.help() + if !strings.Contains(help, "Queried period:") || !strings.Contains(help, "Costs can lag and do not determine enforcement state") { + t.Fatal("Costs view omitted its coverage and enforcement limitations") + } + m.Update(tea.WindowSizeMsg{Width: 40, Height: 12}) + for _, line := range strings.Split(ansi.Strip(m.View().Content), "\n") { + if ansi.StringWidth(line) > 40 { + t.Fatalf("minimum-size overview exceeded its width: %q", line) + } + } +} + +func TestLimitsWorkspaceConfirmedRemovalPreservesReceiptAfter404(t *testing.T) { + for _, otherFailure := range []bool{false, true} { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + missing := &openai.Error{StatusCode: 404} + m.lastSaved = gjson.Parse(`{"object":"organization.spend_limit.deleted","deleted":true}`) + m.lastSavedOperation = limitsChangeOperation("limit_delete", false) + m.snapshot = limitsSnapshot{LimitUnconfirmed: true, AlertsComplete: true, CostsComplete: true, + Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: missing}}} + m.workerError = missing + independent := errors.New("synthetic Costs failure") + if otherFailure { + m.snapshot.Errors = append(m.snapshot.Errors, limitsSourceFailure{Resource: "costs", Err: independent}) + m.workerError = errors.Join(missing, independent) + } + var out bytes.Buffer + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard}) + if otherFailure { + if !errors.Is(err, independent) { + t.Fatal("confirmed removal hid an independent refresh failure") + } + } else if err != nil || !strings.Contains(out.String(), "Removal confirmed. Current hard-limit configuration was not returned (HTTP 404).") { + t.Fatalf("expected post-removal 404 overturned a confirmed removal: %v, %q", err, out.String()) + } + if !strings.Contains(out.String(), "deleted") || strings.Contains(out.String(), "No hard spend limit configured") { + t.Fatalf("receipt disappeared or 404 became confirmed absence: %q", out.String()) + } + } +} + +func TestLimitsWorkspaceShutdownKeepsCompletedSourceFailure(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + original := errors.New("synthetic source failure before shutdown") + snapshot := limitsSnapshot{Errors: []limitsSourceFailure{ + {Resource: "spend_limit", Err: original}, {Resource: "costs", Err: context.Canceled}, + }} + m.workerSnapshot = &snapshot + m.workerError = errors.Join(original, context.Canceled) + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) + if !errors.Is(err, original) || errors.Is(err, context.Canceled) { + t.Fatalf("normal shutdown hid the read failure or became an interruption: %v", err) + } +} + +func TestLimitsWorkspaceAmbiguousEnforcementStaysUnknown(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.snapshot.LimitAbsent = false + m.snapshot.Limit = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month","enforcement":{"status":"inactive","status":"enforcing"}}`) + view := ansi.Strip(m.View().Content) + if !strings.Contains(view, "Returned enforcement: ambiguous; inspect original") || strings.Contains(view, "Returned enforcement: inactive") { + t.Fatalf("workspace selected one conflicting enforcement value: %q", view) + } +} diff --git a/pkg/custom/limits_workspace_view.go b/pkg/custom/limits_workspace_view.go new file mode 100644 index 00000000..446e3855 --- /dev/null +++ b/pkg/custom/limits_workspace_view.go @@ -0,0 +1,135 @@ +package custom + +import ( + "strings" + + tea "charm.land/bubbletea/v2" + "charm.land/lipgloss/v2" + "github.com/charmbracelet/x/ansi" + "github.com/tidwall/gjson" +) + +func (m *limitsWorkspace) View() tea.View { + view := tea.NewView("") + view.AltScreen = true + if m.quitting { + return view + } + if m.small() { + view.Content = "Resize to at least 40 x 12.\nCtrl+C: quit" + return view + } + width := m.width - 2 + accent := lipgloss.NewStyle().Bold(true) + if m.color { + color := "#3159BC" + if m.dark { + color = "#8AA8FF" + } + accent = accent.Foreground(lipgloss.Color(color)) + } + line := func(s string) string { return ansi.Truncate(imagePickerLine(s), width, "…") } + mark := func(s string, selected bool) string { + if selected { + return accent.Render("> " + line(s)) + } + return " " + line(s) + } + header := accent.Render(line(m.heading() + " · " + m.scope())) + var body []string + footer := "Esc: back Ctrl+C: quit" + switch m.page { + case "home": + sections := []string{"Spend", "Alerts"} + if m.snapshot.ProjectID != "" { + sections = append(sections, "Rates") + } + for i := range sections { + if i == m.section { + sections[i] = "[" + sections[i] + "]" + } + } + body = append(body, strings.Join(sections, " "), line(m.sourceSummary()), line(m.costsSummary())) + if m.section == 0 && m.snapshot.Limit.Exists() { + state := m.snapshot.Limit.Get("enforcement.status") + status := "not reported" + if limitsUniqueJSON(m.snapshot.Limit) != nil { + status = "ambiguous; inspect original" + } else if state.Type == gjson.String { + status = state.String() + } + body = append(body, line("Returned enforcement: "+status)) + } + rows := m.menu() + room := max(1, m.height-6-len(body)) + if m.locked { + room = max(1, room-1) + } + start := max(0, m.selection-room+1) + for i := start; i < min(len(rows), start+room); i++ { + body = append(body, mark(rows[i].label, i == m.selection)) + } + if len(rows) == 0 { + body = append(body, "No editable records loaded.") + } + body = append(body, line(m.note)) + if m.locked { + body = append(body, "Writes locked. Inspect before retrying.") + } + footer = "←/→: section ↑/↓: select Enter: edit\nr: refresh i: inspect d: delete alert\nq: quit ?: help Ctrl+C: interrupt" + case "form": + body = append(body, "Tab: next field · Shift+Tab: previous") + count := len(m.draft.inputs) + start := max(0, min(m.focus, count-1)-max(0, (m.height-9)/2)) + end := min(count, start+max(1, (m.height-7)/2)) + for i := start; i < end; i++ { + m.draft.inputs[i].SetWidth(max(1, width-4)) + marker := " " + if i == m.focus { + marker = "> " + } + body = append(body, line(m.draft.labels[i]), marker+ansi.Truncate(m.draft.inputs[i].View(), width, "…")) + } + if count == 3 { + modes := []string{"Preserve original", "Replace (empty is allowed)", "Clear to null"} + body = append(body, mark("Prefix: "+modes[m.draft.prefixMode], m.focus == count)) + } + body = append(body, mark("Review change", m.focus >= count && !(count == 3 && m.focus == count))) + if m.formError != "" { + body = append(body, line(strings.SplitN(m.formError, "\n", 2)[0])) + } + footer = "Enter: next / select\nF1: help Esc: back Ctrl+C: quit" + case "review", "details": + body = append(body, m.viewport.View()) + if m.page == "review" { + back, apply := "> Back", " Apply change" + if m.apply { + back, apply = " Back", "> Apply change" + } + if !m.viewport.AtBottom() { + apply = " Read to the end before applying" + } + body = append(body, line(back+" "+apply)) + footer = "↑/↓: scroll Tab: action Enter: select\nEsc: back Ctrl+C: quit" + } else { + footer = "↑/↓: scroll Esc: back Ctrl+C: quit" + } + } + if m.busy { + status := "Loading…" + if m.page == "review" { + status = "Saving… Do not repeat this change." + } + footer = line(status) + "\nCtrl+C: interrupt" + } + // The action row and controls occupy reserved rows. Long data stays in the viewport. + footerRows := strings.Count(footer, "\n") + 1 + available := max(0, m.height-2-footerRows) + text := strings.Join(body, "\n") + rows := strings.Split(text, "\n") + if len(rows) > available { + rows = rows[:available] + } + view.Content = header + "\n" + strings.Join(rows, "\n") + strings.Repeat("\n", max(1, available-len(rows)+1)) + footer + return view +} diff --git a/pkg/custom/local_errors.go b/pkg/custom/local_errors.go index f885591d..646cf6e6 100644 --- a/pkg/custom/local_errors.go +++ b/pkg/custom/local_errors.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" "net/url" "os" @@ -23,11 +24,16 @@ import ( // Local diagnostics can contain rejected values, file names, and credentialed // URLs. Classify them into local prose; terminal escaping is not redaction. func localErrorMessage(root *cli.Command, failure error) string { - command := root - var contextual *commandError - if errors.As(failure, &contextual) && contextual.command != nil { - command = contextual.command - } + return localErrorMessageForDisplay(root, failure, false) +} + +// Keep human recovery separate from existing machine fallback messages. +func readableLocalErrorMessage(root *cli.Command, failure error) string { + return localErrorMessageForDisplay(root, failure, true) +} + +func localErrorMessageForDisplay(root *cli.Command, failure error, human bool) string { + command := commandErrorContext(root, failure) var timeout net.Error var networkError *url.Error var connectionError *net.OpError @@ -51,8 +57,11 @@ func localErrorMessage(root *cli.Command, failure error) string { var navigationFailure *listNavigationError var modelsListFailure *modelsListError var batchFailure *batchWorkflowError + var limitsFailure *limitsWorkflowFailure var finiteListFailure *clioutput.FiniteJSONListError switch { + case errors.As(failure, &limitsFailure): + return limitsFailure.Error() case errors.As(failure, &batchFailure): return batchFailure.message case errors.As(failure, &adminSetupFailure): @@ -133,6 +142,11 @@ func localErrorMessage(root *cli.Command, failure error) string { } // Transport wrappers can contain a vetted local configuration error, such // as the mTLS proxy restriction. Keep that guidance before this fallback. + if human && (errors.As(failure, &networkError) || errors.As(failure, &connectionError) || errors.Is(failure, io.ErrUnexpectedEOF)) { + if message := readableSpendMutationFailure(root, command); message != "" { + return message + } + } if errors.As(failure, &networkError) || errors.As(failure, &connectionError) { return "Could not connect to the API. Check your connection, proxy, and --base-url setting." } diff --git a/pkg/custom/readable_errors.go b/pkg/custom/readable_errors.go index aaf9851c..24c334ea 100644 --- a/pkg/custom/readable_errors.go +++ b/pkg/custom/readable_errors.go @@ -109,17 +109,25 @@ func showCommandError(ctx context.Context, root *cli.Command, failure error, out isAPI := errors.As(failure, &apierr) var streamerr *ssestream.StreamError isStream := !errors.Is(failure, context.Canceled) && errors.As(failure, &streamerr) + var limitsFailure *limitsWorkflowFailure + if errors.As(failure, &limitsFailure) { + // Aggregate source failures retain their causes, but expose only the + // workflow's safe outcome. Individual API payloads are not the result. + isAPI, isStream = false, false + } if format == "text" && root.String("transform-error") == "" { message := "" var batchFailure *batchWorkflowError - if errors.As(failure, &batchFailure) { + if limitsFailure != nil { + message = limitsFailure.Error() + } else if errors.As(failure, &batchFailure) { message = batchFailure.message } else if isAPI && !errors.Is(failure, context.Canceled) { message = readableAPIErrorMessage(root, failure, apierr) } else if isStream { message = readableStreamErrorText } else { - message = localErrorMessage(root, failure) + message = readableLocalErrorMessage(root, failure) } return readable.WriteText(out, message) } @@ -161,10 +169,17 @@ func readableAPIErrorMessage(root *cli.Command, failure error, apierr *openai.Er switch apierr.StatusCode { case http.StatusUnauthorized: message = "Check API key, organization and project.\nKey setup: " + invocation + " help setup" + if required, _ := commandErrorContext(root, failure).Metadata[adminCredentialsRequiredMetadata].(bool); required { + message = "Check --admin-api-key or OPENAI_ADMIN_KEY and any Authorization override.\n" + + "Check the organization and API address.\nKey setup: " + invocation + " help setup admin" + } case http.StatusForbidden: message = "Check your key's permissions.\nCheck project access to this resource." case http.StatusNotFound: message = "Check the resource or model ID.\nCheck your key's access to it." + if guidance := readableSpendNotFoundMessage(root, failure); guidance != "" { + message = guidance + } case http.StatusBadRequest, http.StatusUnprocessableEntity: var contextual *commandError var command *cli.Command @@ -175,7 +190,11 @@ func readableAPIErrorMessage(root *cli.Command, failure error, apierr *openai.Er case http.StatusTooManyRequests: message = "Wait before trying again." switch apierr.Code { - case "insufficient_quota", "billing_hard_limit_reached", "credit_balance_exhausted", "organization_spend_limit_exceeded", "project_spend_limit_exceeded", "organization_usage_limit_exceeded": + case "organization_spend_limit_exceeded": + message = readableOrganizationSpendLimitMessage(root) + case "organization_usage_limit_exceeded": + message = "Organization usage limit reached.\nCheck your organization's billing and usage limits." + case "insufficient_quota", "billing_hard_limit_reached", "credit_balance_exhausted", "project_spend_limit_exceeded": message = "API usage or billing limit reached.\nCheck your project's billing and limits." } case http.StatusConflict: diff --git a/pkg/custom/spend_error_guidance.go b/pkg/custom/spend_error_guidance.go new file mode 100644 index 00000000..3838719b --- /dev/null +++ b/pkg/custom/spend_error_guidance.go @@ -0,0 +1,100 @@ +package custom + +import ( + "errors" + + "github.com/openai/openai-cli/internal/clihelp" + localcommands "github.com/openai/openai-cli/internal/commands" + "github.com/urfave/cli/v3" +) + +func commandErrorContext(root *cli.Command, failure error) *cli.Command { + var contextual *commandError + if errors.As(failure, &contextual) && contextual.command != nil { + return contextual.command + } + return root +} + +const spendRecoveryContext = "Keep the original API address, authentication, organization, and project options for any check." + +// A status alone cannot establish configuration at an arbitrary API address. +// Recovery uses declared command metadata, never request URLs or response prose. +func readableSpendNotFoundMessage(root *cli.Command, failure error) string { + command := commandErrorContext(root, failure) + invocation := clihelp.DisplayInvocation(root) + message := "" + switch localcommands.ResourceName(command) { + case "admin:organization:spend-limit": + if command.Name == "retrieve" || command.Name == "delete" { + message = "The organization spend limit was not returned.\n" + + "Check the API address and organization. The limit may not be configured.\n" + + "Limit options: " + invocation + " help admin organization spend-limit update" + } + case "admin:organization:projects:spend-limit": + if command.Name == "retrieve" || command.Name == "delete" { + message = "The project or its spend limit could not be found.\n" + + "Check the project ID and access first.\n" + + "Project lookup: " + invocation + " help admin organization projects retrieve\n" + + "If the project exists, the limit may not be configured.\n" + + "Limit options: " + invocation + " help admin organization projects spend-limit update" + } + case "admin:organization:spend-alerts", "admin:organization:projects:spend-alerts": + if command.Name == "retrieve" || command.Name == "update" || command.Name == "delete" { + path := "admin organization spend-alerts list" + message = "The spend alert could not be found at this API address.\nCheck the alert ID against current organization alerts." + if localcommands.ResourceName(command) == "admin:organization:projects:spend-alerts" { + path = "admin organization projects spend-alerts list" + message = "The project spend alert could not be found at this API address.\nCheck the project ID, access, and current project alerts." + } + message += "\nList options: " + invocation + " help " + path + } + } + if message != "" { + message += "\n" + spendRecoveryContext + } + return message +} + +func readableOrganizationSpendLimitMessage(root *cli.Command) string { + invocation := clihelp.DisplayInvocation(root) + return "The API reports an organization spend limit was reached.\n" + + "Ask an organization owner to inspect that limit.\n" + + "An admin API key is required for the OpenAI limit lookup.\n" + + "Limit lookup: " + invocation + " help admin organization spend-limit retrieve\n" + + "Admin key setup: " + invocation + " help setup admin\n" + + spendRecoveryContext +} + +func readableSpendMutationFailure(root, command *cli.Command) string { + path := "" + switch localcommands.ResourceName(command) { + case "admin:organization:spend-limit": + if command.Name == "update" || command.Name == "delete" { + path = "admin organization spend-limit retrieve" + } + case "admin:organization:projects:spend-limit": + if command.Name == "update" || command.Name == "delete" { + path = "admin organization projects spend-limit retrieve" + } + case "admin:organization:spend-alerts", "admin:organization:projects:spend-alerts": + prefix := "admin organization spend-alerts " + if localcommands.ResourceName(command) == "admin:organization:projects:spend-alerts" { + prefix = "admin organization projects spend-alerts " + } + switch command.Name { + case "create": + path = prefix + "list" + case "update", "delete": + path = prefix + "retrieve" + } + } + if path == "" { + return "" + } + return "The API response did not confirm the spend change.\n" + + "The API may have received the change.\n" + + "Check its current state before repeating the command.\n" + + "Read options: " + clihelp.DisplayInvocation(root) + " help " + path + "\n" + + spendRecoveryContext +} diff --git a/pkg/custom/spend_error_guidance_test.go b/pkg/custom/spend_error_guidance_test.go new file mode 100644 index 00000000..52582cd8 --- /dev/null +++ b/pkg/custom/spend_error_guidance_test.go @@ -0,0 +1,164 @@ +package custom + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "testing" + + "github.com/openai/openai-go/v3" + "github.com/stretchr/testify/require" + "github.com/urfave/cli/v3" +) + +func spendGuidanceTestCommand(resource, operation string) *cli.Command { + return &cli.Command{Name: operation, Metadata: map[string]any{ + "openai-resource-command": resource, adminCredentialsRequiredMetadata: true, + }} +} + +func TestSpendErrorGuidanceAdminMetadata(t *testing.T) { + root := readableErrorTestCommand(t) + root.Metadata["help-invocation"] = "'/tmp/synthetic app/openai'" + for _, required := range []bool{false, true} { + command := &cli.Command{Name: "retrieve", Metadata: map[string]any{adminCredentialsRequiredMetadata: required}} + failure := &openai.Error{StatusCode: http.StatusUnauthorized} + message := readableAPIErrorMessage(root, withCommandError(command, failure), failure) + want := "Key setup: '/tmp/synthetic app/openai' help setup" + if required { + want += " admin" + require.Contains(t, message, "OPENAI_ADMIN_KEY") + require.Contains(t, message, "Authorization override") + } + require.Contains(t, strings.Split(message, "\n"), want) + } +} + +func TestSpendErrorGuidanceNotFoundPreservesUncertainty(t *testing.T) { + root := readableErrorTestCommand(t) + for _, test := range []struct{ resource, operation, want string }{ + {"admin:organization:spend-limit", "retrieve", "may not be configured"}, + {"admin:organization:projects:spend-limit", "delete", "Check the project ID and access first"}, + {"admin:organization:spend-alerts", "update", "help admin organization spend-alerts list"}, + {"admin:organization:projects:spend-alerts", "retrieve", "help admin organization projects spend-alerts list"}, + } { + command := spendGuidanceTestCommand(test.resource, test.operation) + failure := &openai.Error{StatusCode: http.StatusNotFound, Message: "synthetic-private-secret\x1b[2J"} + message := readableAPIErrorMessage(root, withCommandError(command, failure), failure) + require.Contains(t, message, test.want) + require.Contains(t, message, spendRecoveryContext) + require.NotContains(t, message, "No hard spend limit configured") + require.NotContains(t, message, "synthetic-private") + require.NotContains(t, message, "\x1b") + } + for _, command := range []*cli.Command{ + spendGuidanceTestCommand("models", "retrieve"), + spendGuidanceTestCommand("admin:organization:spend-limit", "update"), + spendGuidanceTestCommand("admin:organization:spend-alerts", "create"), + } { + require.Empty(t, readableSpendNotFoundMessage(root, withCommandError(command, errors.New("synthetic")))) + } +} + +func TestSpendErrorGuidanceTransportPrecedenceAndMachineFallback(t *testing.T) { + command := spendGuidanceTestCommand("admin:organization:spend-limit", "update") + network := &url.Error{Op: "Post", URL: "https://synthetic-private.invalid/?token=synthetic-private", Err: io.EOF} + for _, test := range []struct { + name string + err error + want string + changed bool + }{ + {"lost response", network, "The API may have received the change", true}, + {"truncated response", fmt.Errorf("synthetic-private: %w", io.ErrUnexpectedEOF), "The API may have received the change", true}, + {"cancel", errors.Join(network, context.Canceled), "Request canceled.", false}, + {"timeout", errors.Join(network, context.DeadlineExceeded), "The request timed out.", false}, + {"configuration", &url.Error{Op: "Post", URL: "synthetic-private", Err: errors.New("mTLS does not support HTTPS proxies")}, "mTLS does not support HTTPS proxies", false}, + } { + t.Run(test.name, func(t *testing.T) { + failure := withCommandError(command, test.err) + root := readableErrorTestCommand(t) + baseline := localErrorMessage(root, failure) + var human bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &human)) + require.Contains(t, human.String(), test.want) + require.NotContains(t, human.String(), "synthetic-private") + if test.changed { + require.Contains(t, human.String(), "help admin organization spend-limit retrieve") + require.NotContains(t, human.String(), "Could not connect") + } else { + require.Equal(t, baseline+"\n", human.String()) + } + root = readableErrorTestCommand(t, "--format-error", "json") + var machine bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &machine)) + var value map[string]any + require.NoError(t, json.Unmarshal(machine.Bytes(), &value)) + require.Equal(t, map[string]any{"message": baseline}, value) + root = readableErrorTestCommand(t, "--format-error", "text", "--transform-error", "message") + machine.Reset() + require.NoError(t, ShowCommandError(root, failure, &machine)) + require.Equal(t, baseline+"\n", machine.String()) + }) + } + root := readableErrorTestCommand(t) + for _, command := range []*cli.Command{ + spendGuidanceTestCommand("models", "delete"), + spendGuidanceTestCommand("admin:organization:spend-limit", "retrieve"), + } { + failure := withCommandError(command, network) + require.Equal(t, localErrorMessage(root, failure), readableLocalErrorMessage(root, failure)) + } +} + +func TestSpendErrorGuidanceMutationReadRoutes(t *testing.T) { + root := readableErrorTestCommand(t) + for _, test := range []struct{ resource, operation, read string }{ + {"admin:organization:projects:spend-limit", "delete", "admin organization projects spend-limit retrieve"}, + {"admin:organization:spend-alerts", "create", "admin organization spend-alerts list"}, + {"admin:organization:projects:spend-alerts", "create", "admin organization projects spend-alerts list"}, + {"admin:organization:projects:spend-alerts", "update", "admin organization projects spend-alerts retrieve"}, + } { + require.Contains(t, readableSpendMutationFailure(root, spendGuidanceTestCommand(test.resource, test.operation)), "help "+test.read) + } +} + +func TestSpendErrorGuidanceLimitCodesStayScoped(t *testing.T) { + root := readableErrorTestCommand(t) + for _, test := range []struct{ code, want, absent string }{ + {"organization_spend_limit_exceeded", "admin API key", "project's billing"}, + {"organization_usage_limit_exceeded", "organization's billing", "help admin organization spend-limit"}, + {"project_spend_limit_exceeded", "project's billing", "organization spend limit"}, + {"rate_limit_exceeded", "Wait before trying again", "admin API key"}, + {"synthetic-private-code", "Wait before trying again", "spend limit"}, + } { + failure := &openai.Error{StatusCode: http.StatusTooManyRequests, Code: test.code} + message := readableAPIErrorMessage(root, failure, failure) + require.Contains(t, message, test.want) + require.NotContains(t, message, test.absent) + require.NotContains(t, message, "synthetic-private") + } +} + +func TestSpendErrorGuidanceAggregateKeepsSafeMessageAndCause(t *testing.T) { + apierr := &openai.Error{StatusCode: http.StatusForbidden, Message: "synthetic-private"} + failure := &limitsWorkflowFailure{message: "The overview is incomplete. Retry the failed source.", cause: errors.Join(apierr, io.ErrClosedPipe)} + for _, args := range [][]string{nil, {"--format-error", "json"}, {"--transform-error", "message"}} { + root := readableErrorTestCommand(t, args...) + var out bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &out)) + require.Contains(t, out.String(), failure.message) + require.NotContains(t, out.String(), "synthetic-private") + require.NotContains(t, out.String(), "HTTP 403") + require.ErrorIs(t, failure, io.ErrClosedPipe) + var preserved *openai.Error + require.ErrorAs(t, failure, &preserved) + require.Same(t, apierr, preserved) + } +} diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md new file mode 100644 index 00000000..6b127582 --- /dev/null +++ b/scripts/demos/limits-workspace/README.md @@ -0,0 +1,78 @@ +# Limits workspace terminal replay + +This recipe drives an immutable CLI binary against a synthetic loopback API. +It makes no live API calls or billing changes. +The recorder uses the shared `capture_and_render.sh` lifecycle and existing PTY harness. +No Go build occurs during recording. + +Obtain the source-build and PTY reservations through the current coordinator. +Read `/Users/vguvvala/code/cli-work/DESIGN.md`, `STATUS.md`, and `PRE_PUSH_REVIEW.md` before presentation changes or publication. +Preserve generated ownership and obtain independent adversarial review before every feature push. + +Use existing Python, asciinema, agg, ffmpeg, ffprobe, and Menlo installations. +Supply a verified binary and its full source commit. +Keep each output directory empty and outside the repository. +Set `DEMO_SOURCE_STATE` when recording an uncommitted candidate based on that source commit. +Binary hashes and source manifests establish recording identity; they do not establish build provenance alone. + +```sh +# Run only during the granted PTY reservation. +DEMO_WIDTH=80 DEMO_NO_COLOR=1 \ + bash scripts/demos/limits-workspace/record.sh --run-authorized-pty-slot \ + /absolute/path/to/immutable/openai SOURCE_SHA /absolute/path/to/evidence/limits-80 + +DEMO_WIDTH=40 DEMO_NO_COLOR=1 \ + bash scripts/demos/limits-workspace/record.sh --run-authorized-pty-slot \ + /absolute/path/to/immutable/openai SOURCE_SHA /absolute/path/to/evidence/limits-40 +``` + +Width 80 uses a 24-row workspace. Width 40 uses a 12-row workspace. +The outer recording reserves four additional rows for the evidence label and shell prompt. +`DEMO_NO_COLOR=0` permits a separate color run; monochrome remains the default. +The isolated child environment includes only fake credentials and the fixture endpoint. + +| Scene | Scope | Confirmed fixture action | +| --- | --- | --- | +| `org-limit` | Organization | Replace the hard limit with USD 125.25 per month | +| `org-limit-delete` | Organization | Remove the hard limit and retain its confirmed receipt after a missing-record refresh | +| `project-limit` | Project | Replace the hard limit with USD 125.25 per month | +| `alert-create` | Organization | Create a USD 75.50 monthly alert with synthetic email settings | +| `alert-edit` | Project | Change the alert to USD 150.25 and clear its prefix to null | +| `alert-delete` | Organization | Delete the selected synthetic alert | +| `rate-update` | Project | Change requests per minute to 250; preserve all other dimensions | +| `uncertain` | Organization | Apply a synthetic write, lose its response, and verify that repetition stays blocked | +| `slow-cancel` | Organization | Cancel a held read with Ctrl+C; perform no mutation | + +Each mutation scene opens a form or deletion review. +The driver reads to the review's end before selecting Apply. +It checks that the fixture receives no mutation before confirmation. +The uncertain scene checks one mutation, retained failure guidance, and exit status 1. +The cancellation scene waits for the fixture's request marker and requires exit status 130. +Successful scenes require exit status 0 and one mutation each. + +The fixture stores only synthetic request bodies, response hashes, and method/path/status records. +It never logs request headers or credentials. +Shutdown releases held work and joins the server worker. +The scene driver restores terminal output processing and closes every owned child PTY. + +The recorder preserves raw child bytes, child casts, outer casts, request logs, hashes, and actual process statuses. +The validator checks byte identity across all three capture forms. +It checks exact mutation bodies and guards against automatic retry after an uncertain write. +Terminal captures merge stdout and stderr. They do not prove separate stream routing. + +Planned screenshot evidence comes from recorded checkpoints: + +- Overview: scope, reported current-month Costs, spend units, returned enforcement, and keyboard controls. +- Form: entered values, retained optional-field semantics, and visible Review action. +- Review top: selected resource and original state. +- Review Apply: explicit final action after scrolling through the complete review. +- Saved state: refreshed resource and confirmed outcome. +- Uncertain state: retained draft and blocked retry guidance. +- Loading state: cancellation remains available while a read is pending. +- Final scene frame: restored shell, actual result, and no stale confirmation question. + +Inspect every full-terminal PNG and relevant GIF frame at both sizes. +Check clipping, stale rows, focus markers, controls, and final state after exit. +Generated artifacts are terminal replays, not native terminal appearance evidence. +Do not commit binary media or claim these planned scenes ran before execution. +The earlier spend-threshold recorder remains the separate before/after comparison. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh new file mode 100755 index 00000000..248529f1 --- /dev/null +++ b/scripts/demos/limits-workspace/record.sh @@ -0,0 +1,79 @@ +#!/bin/bash +set -euo pipefail + +if [ "$#" -ne 4 ] || [ "$1" != --run-authorized-pty-slot ]; then + echo 'usage: record.sh --run-authorized-pty-slot CLI_BINARY SOURCE_SHA OUTPUT_DIR' >&2 + exit 2 +fi +shift +record_source="$(cd "$(dirname "$0")" && pwd -P)" +record_root="$(cd "$record_source/../../.." && pwd -P)" +record_width="${DEMO_WIDTH:-80}" +record_no_color="${DEMO_NO_COLOR:-1}" +case "$record_width" in 80) record_height=24;; 40) record_height=12;; *) echo 'DEMO_WIDTH must be 80 or 40.' >&2; exit 2;; esac +case "$record_no_color" in 0|1) ;; *) echo 'DEMO_NO_COLOR must be 0 or 1.' >&2; exit 2;; esac +source "$record_source/../capture_and_render.sh" +# This workflow has one candidate binary. Passing it twice reuses the shared lifecycle. +demo_prepare_capture "$record_root" "$1" "$1" "$2" "$2" "$3" "$record_source/server.py" +demo_python="$(command -v python3)" +record_digest="$(shasum -a 256 "$demo_after" | cut -d ' ' -f 1)" +demo_start_api "$demo_output/requests.jsonl" "$demo_output/fixtures.json" "$demo_runtime/gates" +cat > "$demo_runtime/scene.sh" <<'SCENE' +#!/bin/bash +set -euo pipefail +test -t 0 && test -t 1 && test -t 2 || exit 99 +printf '\033[2J\033[H' +printf '%s\n' 'Terminal replay. Synthetic data.' "$DEMO_SCENE_LABEL" +exec "$DEMO_PYTHON" -I -B "$DEMO_DRIVER" +SCENE +cp "$demo_runtime/scene.sh" "$demo_output/scene.sh" +{ + echo 'feature: supported limits workspace workflows' + echo "candidate source commit: $demo_after_sha" + echo "candidate source state: ${DEMO_SOURCE_STATE:-committed source supplied by caller}" + echo 'baseline: no before comparison; this records the supplied workspace binary only' + echo 'scope: synthetic loopback API; Bash PTY replay; no native graphical terminal or live API claim' + echo "child terminal: ${record_width}x${record_height}; outer terminal: ${record_width}x$((record_height+4))" + echo "NO_COLOR: $record_no_color; GOMAXPROCS=2; isolated HOME; CI unset" + echo 'render: Menlo 18px, asciinema theme, shared capture_and_render lifecycle' + demo_capture_metadata + "$demo_python" --version +} > "$demo_output/metadata.txt" +shasum -a 256 "$record_source/record.sh" "$record_source/server.py" "$record_source/scene.py" \ + "$record_source/validate.py" "$record_source/README.md" \ + "$record_root/scripts/demos/capture_and_render.sh" "$record_root/scripts/image_picker_harness.py" \ + "$record_root/pkg/custom/limits_commands.go" "$record_root/pkg/custom/limits_service.go" \ + "$record_root/pkg/custom/limits_values.go" "$record_root/pkg/custom/limits_draft.go" \ + "$record_root/pkg/custom/limits_workspace.go" "$record_root/pkg/custom/limits_workspace_view.go" \ + "$record_root/internal/output/transform/project_limits_overview.go" > "$demo_output/source-sha256.txt" +demo_window_size="${record_width}x$((record_height+4))" +demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ + --theme asciinema --fps-cap 15 --last-frame-duration 2) +record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel) +for record_scene in "${record_scenes[@]}"; do + demo_capture_scene "$record_scene" 0 "$demo_runtime/after" "$demo_api_url/$record_scene/v1" \ + "$record_scene: ${record_width}x${record_height}" \ + "DEMO_PYTHON=$demo_python" "DEMO_ASCIINEMA=$demo_asciinema" "DEMO_DRIVER=$record_source/scene.py" \ + "DEMO_SCENE_NAME=$record_scene" "DEMO_EVIDENCE_BASE=$demo_output/$record_scene-evidence" \ + "DEMO_WIDTH=$record_width" "DEMO_HEIGHT=$record_height" "DEMO_NO_COLOR=$record_no_color" \ + "DEMO_GATE_DIRECTORY=$demo_runtime/gates" "DEMO_REQUEST_LOG=$demo_output/requests.jsonl" +done +demo_stop_api +test "$record_digest" = "$(shasum -a 256 "$demo_after" | cut -d ' ' -f 1)" +shasum -a 256 -c "$demo_output/source-sha256.txt" > "$demo_output/source-validation.txt" +"$demo_python" -I -B "$record_source/validate.py" "$demo_output" > "$demo_output/validation.txt" +demo_assemble_capture 150 "${record_scenes[@]}" +# Select checkpoint frames from the exact child captures. No UI text is reconstructed. +for record_scene in "${record_scenes[@]}"; do + while IFS=$'\t' read -r record_name record_seconds; do + "$demo_agg" --quiet "${demo_render_options[@]}" --select "$record_seconds" \ + "$demo_output/$record_scene-evidence.child.cast" "$demo_output/$record_scene-$record_name-frame.gif" + "$demo_ffmpeg" -hide_banner -loglevel error -y -i "$demo_output/$record_scene-$record_name-frame.gif" \ + -frames:v 1 "$demo_output/$record_scene-$record_name.png" + done < <("$demo_python" -I -B -c 'import json,sys; data=json.load(open(sys.argv[1])); [print(item["name"]+"\t"+str(item["seconds"])) for item in data["checkpoints"]]' "$demo_output/$record_scene-evidence.json") +done +( + cd "$demo_output" + shasum -a 256 -- *.cast *.gif *.png *.txt *.json *.jsonl *.sh > SHA256SUMS +) +printf 'Recorded limits workspace terminal replay in %s\n' "$demo_output" diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py new file mode 100755 index 00000000..655aa984 --- /dev/null +++ b/scripts/demos/limits-workspace/scene.py @@ -0,0 +1,269 @@ +#!/usr/bin/env python3 +"""Drive the real workspace with deterministic synthetic input and retain raw bytes.""" + +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import termios +import time + + +spec = importlib.util.spec_from_file_location("limits_demo_terminal", Path(__file__).resolve().parents[2] / "image_picker_harness.py") +harness = importlib.util.module_from_spec(spec) +spec.loader.exec_module(harness) + + +def write_text(path, value): + with Path(path).open("x", encoding="utf-8") as destination: + if destination.write(value) != len(value): + raise OSError("incomplete scene evidence write") + + +class RelayTerminal(harness.Terminal): + def read(self, duration=0.05): + offset = len(self.raw) + super().read(duration) + chunk = self.raw[offset:] + if sys.stdout.buffer.write(chunk) != len(chunk): + raise OSError("short terminal relay write") + sys.stdout.buffer.flush() + + def send(self, value): + if os.write(self.master, value) != len(value): + raise OSError("short workspace key write") + + def finish(self, expected): + deadline = time.monotonic() + 8 + while self.child.poll() is None: + self.read() + if time.monotonic() >= deadline: + raise AssertionError("workspace did not exit") + drain(self, 0.15) + if self.child.returncode != expected: + raise AssertionError(f"workspace exit={self.child.returncode}, expected={expected}; tail={self.text()[-900:]}") + restored, initial = termios.tcgetattr(self.slave), self.initial.copy() + if sys.platform == "darwin": + restored[3] &= ~termios.PENDIN + initial[3] &= ~termios.PENDIN + if restored != initial: + raise AssertionError("workspace did not restore terminal input") + if self.raw.rfind(b"\x1b[?1049l") <= self.raw.rfind(b"\x1b[?1049h"): + raise AssertionError("workspace did not leave its alternate screen") + if self.raw.rfind(b"\x1b[?25h") <= self.raw.rfind(b"\x1b[?25l"): + raise AssertionError("workspace did not restore cursor visibility") + + +def drain(terminal, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + terminal.read(min(0.03, max(0, deadline - time.monotonic()))) + + +def screen_text(terminal): + with tempfile.TemporaryDirectory(prefix="limits-demo-screen-") as temporary: + recording, transcript = Path(temporary) / "screen.cast", Path(temporary) / "screen.txt" + header = {"version": 2, "width": terminal.initial_size[0], "height": terminal.initial_size[1]} + write_text(recording, "\n".join(json.dumps(item) for item in [header, *terminal.events]) + "\n") + subprocess.run([os.environ["DEMO_ASCIINEMA"], "convert", "-f", "txt", str(recording), str(transcript)], + capture_output=True, check=True, timeout=10, + env={"PATH": "/usr/bin:/bin", "HOME": temporary, + "ASCIINEMA_STATE_HOME": temporary + "/state", "ASCIINEMA_CONFIG_HOME": temporary + "/config"}) + return transcript.read_text() + + +def wait_screen(terminal, required, forbidden=(), timeout=10): + deadline, screen = time.monotonic() + timeout, "" + while time.monotonic() < deadline: + drain(terminal, 0.15) + screen = screen_text(terminal) + if all(text in screen for text in required) and not any(text in screen for text in forbidden): + return screen + if terminal.child.poll() is not None: + break + raise AssertionError(f"workspace state missing {required!r}; screen={screen!r}") + + +def requests(scene): + entries = [json.loads(line) for line in Path(os.environ["DEMO_REQUEST_LOG"]).read_text().splitlines() if line] + return [entry for entry in entries if entry["scene"] == scene] + + +def mutations(scene): + return [item for item in requests(scene) if item["method"] in {"POST", "DELETE"}] + + +def checkpoint(terminal, evidence, output, name): + screen = screen_text(terminal) + write_text(Path(str(output) + f".{name}.txt"), screen) + evidence["checkpoints"].append({"name": name, "seconds": time.monotonic() - terminal.started}) + drain(terminal, 0.6) + + +def key(terminal, evidence, label, value): + if terminal.child.poll() is not None: + raise AssertionError("workspace exited before its scripted action") + evidence["keys"].append({"key": label, "seconds": time.monotonic() - terminal.started}) + terminal.send(value) + drain(terminal, 0.08) + + +def replace(terminal, evidence, label, value): + key(terminal, evidence, label, b"\x01\x0b\x1b[200~" + value.encode() + b"\x1b[201~") + + +def review(terminal, evidence, scene, output): + wait_screen(terminal, ["> Back", "Enter: select"]) + checkpoint(terminal, evidence, output, "review-top") + for _ in range(30): + screen = screen_text(terminal) + if "Apply change" in screen and "Read to the end" not in screen: + break + key(terminal, evidence, "PageDown", b"\x1b[6~") + else: + raise AssertionError("review never exposed its final action") + if mutations(scene): + raise AssertionError("fixture received a mutation before explicit confirmation") + key(terminal, evidence, "Tab to Apply", b"\t") + wait_screen(terminal, ["> Apply change"]) + checkpoint(terminal, evidence, output, "review-apply") + key(terminal, evidence, "Apply", b"\r") + + +def run_scene(terminal, scene, evidence, output): + if scene == "slow-cancel": + gate = Path(os.environ["DEMO_GATE_DIRECTORY"]) / "slow-cancel.requested" + deadline = time.monotonic() + 8 + while not gate.exists(): + terminal.read() + if time.monotonic() >= deadline: + raise AssertionError("slow request did not reach fixture") + wait_screen(terminal, ["Loading", "Ctrl+C"]) + checkpoint(terminal, evidence, output, "loading") + key(terminal, evidence, "Ctrl+C", b"\x03") + terminal.finish(130) + write_text(Path(os.environ["DEMO_GATE_DIRECTORY"]) / "slow-cancel.release", "child canceled and exited\n") + if mutations(scene): + raise AssertionError("canceling a load caused a mutation") + return + wait_screen(terminal, ["Set hard spend limit", "Ctrl+C"], ["Loading"]) + checkpoint(terminal, evidence, output, "overview") + if scene.startswith("alert-"): + key(terminal, evidence, "Alerts", b"\x1b[C") + wait_screen(terminal, ["Create spend alert"]) + if scene != "alert-create": + key(terminal, evidence, "Select alert", b"\x1b[B") + if scene == "alert-delete": + key(terminal, evidence, "Delete selected alert", b"d") + else: + key(terminal, evidence, "Edit alert", b"\r") + wait_screen(terminal, ["Monthly USD", "Review change"]) + replace(terminal, evidence, "Monthly USD", "75.50" if scene == "alert-create" else "150.25") + key(terminal, evidence, "Recipients", b"\t") + if scene == "alert-create": + replace(terminal, evidence, "Synthetic recipient", "finance@example.test") + key(terminal, evidence, "Subject prefix", b"\t") + if scene == "alert-create": + replace(terminal, evidence, "Subject prefix", "Demo") + key(terminal, evidence, "Prefix mode", b"\t") + if scene == "alert-edit": + key(terminal, evidence, "Replace prefix mode", b"\r") + key(terminal, evidence, "Clear prefix to null", b"\r") + key(terminal, evidence, "Review action", b"\t") + checkpoint(terminal, evidence, output, "form") + key(terminal, evidence, "Review", b"\r") + elif scene == "org-limit-delete": + key(terminal, evidence, "Select removal", b"\x1b[B") + wait_screen(terminal, ["Remove hard spend limit"]) + key(terminal, evidence, "Review removal", b"\r") + elif scene == "rate-update": + key(terminal, evidence, "Alerts", b"\x1b[C") + key(terminal, evidence, "Rates", b"\x1b[C") + wait_screen(terminal, ["rl_demo"]) + key(terminal, evidence, "Edit rate", b"\r") + wait_screen(terminal, ["Requests / minute", "Review change"]) + replace(terminal, evidence, "Requests per minute", "250") + for _ in range(6): + key(terminal, evidence, "Next rate field", b"\t") + checkpoint(terminal, evidence, output, "form") + key(terminal, evidence, "Review", b"\r") + else: + key(terminal, evidence, "Edit hard limit", b"\r") + wait_screen(terminal, ["Monthly USD", "Review change"]) + replace(terminal, evidence, "Monthly USD", "125.25") + key(terminal, evidence, "Review action", b"\t") + checkpoint(terminal, evidence, output, "form") + key(terminal, evidence, "Review", b"\r") + review(terminal, evidence, scene, output) + if scene == "uncertain": + wait_screen(terminal, ["Save not confirmed", "Review change"]) + checkpoint(terminal, evidence, output, "uncertain") + key(terminal, evidence, "Blocked repeat Enter", b"\r\r") + if len(mutations(scene)) != 1: + raise AssertionError("uncertain write repeated automatically") + key(terminal, evidence, "Back", b"\x1b") + wait_screen(terminal, ["Writes locked"]) + key(terminal, evidence, "Quit", b"q") + terminal.finish(1) + else: + wait_screen(terminal, ["Removal confirmed" if scene == "org-limit-delete" else "Saved successfully"], ["Loading", "Saving"]) + checkpoint(terminal, evidence, output, "saved") + key(terminal, evidence, "Quit", b"q") + terminal.finish(0) + if len(mutations(scene)) != 1: + raise AssertionError("scene did not produce exactly one confirmed mutation") + + +def main(): + scene, output = os.environ["DEMO_SCENE_NAME"], Path(os.environ["DEMO_EVIDENCE_BASE"]) + width, height = int(os.environ["DEMO_WIDTH"]), int(os.environ["DEMO_HEIGHT"]) + if (width, height) not in {(80, 24), (40, 12)} or not all(os.isatty(fd) for fd in (0, 1, 2)): + raise RuntimeError("scene requires a supported terminal size") + binary = shutil.which("openai") + if binary is None: + raise RuntimeError("immutable CLI binary is unavailable") + project = scene in {"project-limit", "alert-edit", "rate-update"} + args = ["projects", "limits", "--project-id", "proj_demo"] if project else ["admin", "limits"] + evidence = {"scene": scene, "width": width, "height": height, "command": ["openai", *args], + "keys": [], "checkpoints": [], "binary_sha256": hashlib.sha256(Path(binary).read_bytes()).hexdigest()} + print("$ openai " + " ".join(args), flush=True) + original = termios.tcgetattr(1) + relay = termios.tcgetattr(1) + relay[1] &= ~termios.OPOST + try: + termios.tcsetattr(1, termios.TCSANOW, relay) + with tempfile.TemporaryDirectory(prefix="limits-demo-home-") as temporary: + env = {"PATH": "/usr/bin:/bin", "HOME": temporary, "TERM": "xterm-256color", "LANG": "en_US.UTF-8", + "GOMAXPROCS": "2", "PAGER": "cat", "OPENAI_API_KEY": "synthetic-demo-key", + "OPENAI_ADMIN_KEY": "synthetic-demo-admin-key", "OPENAI_BASE_URL": os.environ["OPENAI_BASE_URL"]} + if os.environ.get("DEMO_NO_COLOR", "1") == "1": + env.update(NO_COLOR="1", FORCE_COLOR="0") + terminal = None + try: + terminal = RelayTerminal(binary, args, env, width=width, height=height, controlling_terminal=True) + run_scene(terminal, scene, evidence, output) + evidence.update(exit_status=terminal.child.returncode, relayed_bytes=len(terminal.raw), + relayed_sha256=hashlib.sha256(terminal.raw).hexdigest()) + with Path(str(output) + ".tty").open("xb") as destination: + if destination.write(terminal.raw) != len(terminal.raw): + raise OSError("incomplete raw terminal write") + header = {"version": 2, "width": width, "height": height, "title": "Limits workspace; synthetic terminal replay"} + write_text(Path(str(output) + ".child.cast"), "\n".join(json.dumps(item) for item in [header, *terminal.events]) + "\n") + write_text(Path(str(output) + ".exit.txt"), screen_text(terminal)) + finally: + if terminal is not None: + terminal.close() + finally: + termios.tcsetattr(1, termios.TCSANOW, original) + write_text(Path(str(output) + ".json"), json.dumps(evidence, indent=2) + "\n") + print("\n$ ", end="", flush=True) + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/limits-workspace/server.py b/scripts/demos/limits-workspace/server.py new file mode 100755 index 00000000..f4dceaaf --- /dev/null +++ b/scripts/demos/limits-workspace/server.py @@ -0,0 +1,199 @@ +#!/usr/bin/env python3 +"""Stateful synthetic limits API. Bind only to loopback; never log credentials.""" + +import copy +import hashlib +import http.server +import json +from pathlib import Path +import signal +import socket +import sys +import threading +import time +from urllib.parse import urlsplit, parse_qs + + +SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", + "alert-delete", "rate-update", "uncertain", "slow-cancel") + + +def write_text(path, value): + with Path(path).open("x", encoding="utf-8") as destination: + if destination.write(value) != len(value): + raise OSError("incomplete fixture evidence write") + + +def initial_state(project): + scope = "project" if project else "organization" + return { + "limit": {"object": scope + ".spend_limit", "threshold_amount": 10000, + "currency": "USD", "interval": "month", "enforcement": {"status": "enforcing"}}, + "alerts": [{"object": scope + ".spend_alert", "id": "alert_demo", "threshold_amount": 10000, + "currency": "USD", "interval": "month", "notification_channel": { + "type": "email", "recipients": ["finance@example.test"], "subject_prefix": "Demo"}}], + "rates": [{"object": "project.rate_limit", "id": "rl_demo", "model": "synthetic-model", + "max_requests_per_1_minute": 100, "max_tokens_per_1_minute": 20000, + "max_images_per_1_minute": 5, "max_requests_per_1_day": 1000, + "max_audio_megabytes_per_1_minute": 10, "batch_1_day_max_input_tokens": 50000}], + } + + +def page(records): + return {"object": "list", "data": records, "has_more": False, + "first_id": records[0]["id"] if records else None, + "last_id": records[-1]["id"] if records else None} + + +def main(): + if len(sys.argv) != 5: + raise SystemExit("usage: server.py ADDRESS_FILE REQUEST_LOG FIXTURE_METADATA GATE_DIRECTORY") + gate = Path(sys.argv[4]) + gate.mkdir() + states = {name: initial_state(name in {"project-limit", "alert-edit", "rate-update"}) for name in SCENES} + write_text(sys.argv[3], json.dumps({"data": "synthetic only", "initial": states}, indent=2) + "\n") + stop, lock, failures = threading.Event(), threading.Lock(), [] + with open(sys.argv[2], "x", encoding="utf-8") as log: + def record(entry): + line = json.dumps(entry, separators=(",", ":")) + "\n" + with lock: + if log.write(line) != len(line): + raise OSError("incomplete request log write") + log.flush() + + class Fixture(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def handle_request(self): + if self.headers.get("Authorization") != "Bearer synthetic-demo-admin-key": + raise AssertionError("unexpected fixture authentication") + parts = urlsplit(self.path).path.strip("/").split("/") + if len(parts) < 4 or parts[0] not in states or parts[1] != "v1" or parts[2] != "organization": + raise AssertionError("unexpected fixture route") + scene, project, tail = parts[0], False, parts[3:] + if tail[:1] == ["projects"]: + if tail[1:2] != ["proj_demo"]: + raise AssertionError("unexpected synthetic project") + project, tail = True, tail[2:] + state, scope = states[scene], "project" if project else "organization" + payload = None + if self.command == "POST": + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + status, result = 200, None + if self.command == "GET" and tail == ["costs"]: + query = parse_qs(urlsplit(self.path).query) + expected_project = ["proj_demo"] if scene in {"project-limit", "alert-edit", "rate-update"} else [] + if query.get("project_ids[]", []) != expected_project: + raise AssertionError("Costs scope differs from the selected workspace") + start = int(query["start_time"][0]) + result = {"object": "page", "data": [{"object": "bucket", "start_time": start, + "end_time": start + 86400, "results": [{"object": "organization.costs.result", + "amount": {"value": 37.25, "currency": "usd"}, + "project_id": "proj_demo" if scene in {"project-limit", "alert-edit", "rate-update"} else None, + "line_item": None}]}], "has_more": False, "next_page": None} + elif self.command == "GET" and not tail and project: + result = {"id": "proj_demo", "object": "organization.project", "name": "Synthetic project", "status": "active"} + elif tail == ["spend_limit"]: + if self.command == "GET": + if scene == "slow-cancel": + record({"scene": scene, "method": "GET", "resource": "spend_limit", "status": "held"}) + write_text(gate / "slow-cancel.requested", "synthetic request received\n") + deadline = time.monotonic() + 30 + while not (gate / "slow-cancel.release").exists(): + if stop.wait(0.05): + return + if time.monotonic() >= deadline: + raise TimeoutError("slow scene did not confirm cancellation") + return + result = state["limit"] + if result is None: + status, result = 404, {"error": {"message": "Synthetic limit is absent"}} + elif self.command == "POST": + if set(payload) != {"threshold_amount", "currency", "interval"} or payload["currency"] != "USD" or payload["interval"] != "month": + raise AssertionError("unexpected limit mutation body") + state["limit"] = dict(payload, object=scope + ".spend_limit", enforcement={"status": "enforcing"}) + result = state["limit"] + elif self.command == "DELETE": + state["limit"] = None + result = {"object": scope + ".spend_limit.deleted", "deleted": True} + elif tail[:1] == ["spend_alerts"]: + alert_id = tail[1] if len(tail) == 2 else None + alert = next((value for value in state["alerts"] if value["id"] == alert_id), None) + if self.command == "GET": + result = page(state["alerts"]) if alert_id is None else alert + elif self.command == "POST": + if set(payload) != {"threshold_amount", "currency", "interval", "notification_channel"} or payload["currency"] != "USD" or payload["interval"] != "month": + raise AssertionError("unexpected alert mutation body") + result = dict(payload, object=scope + ".spend_alert", id=alert_id or "alert_created") + if alert_id: + if alert is None: + raise AssertionError("edit targeted an unknown synthetic alert") + state["alerts"][state["alerts"].index(alert)] = result + else: + state["alerts"].append(result) + elif self.command == "DELETE" and alert is not None: + state["alerts"].remove(alert) + result = {"object": scope + ".spend_alert.deleted", "id": alert_id, "deleted": True} + elif tail[:1] == ["rate_limits"] and project: + if self.command == "GET": + result = page(state["rates"]) + elif self.command == "POST" and tail[1:] == ["rl_demo"]: + if payload != {"max_requests_per_1_minute": 250}: + raise AssertionError("rate update must change only the selected dimension") + state["rates"][0].update(payload) + result = state["rates"][0] + if result is None: + raise AssertionError("unsupported synthetic request") + body = json.dumps(copy.deepcopy(result), separators=(",", ":")).encode() + entry = {"scene": scene, "method": self.command, "path": urlsplit(self.path).path, + "status": status, "response_sha256": hashlib.sha256(body).hexdigest()} + if payload is not None: + entry["body"] = payload + if scene == "uncertain" and self.command == "POST": + entry["status"] = "response_lost_after_write" + record(entry) + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + return + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("x-should-retry", "false") + self.end_headers() + if self.wfile.write(body) != len(body): + raise OSError("incomplete fixture response") + self.wfile.flush() + record(entry) + + do_GET = handle_request + do_POST = handle_request + do_DELETE = handle_request + + class Server(http.server.ThreadingHTTPServer): + daemon_threads = False + + def handle_error(self, *_): + with lock: + failures.append("fixture handler failed") + + server = Server(("127.0.0.1", 0), Fixture) + signal.signal(signal.SIGTERM, lambda *_: stop.set()) + signal.signal(signal.SIGINT, lambda *_: stop.set()) + worker = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + worker.start() + try: + write_text(sys.argv[1], f"http://127.0.0.1:{server.server_port}\n") + stop.wait() + finally: + stop.set() + server.shutdown() + server.server_close() + worker.join() + if failures: + raise RuntimeError("synthetic fixture failed; evidence is incomplete") + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py new file mode 100755 index 00000000..6a20ce84 --- /dev/null +++ b/scripts/demos/limits-workspace/validate.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +"""Check recorded bytes, actual child statuses, and synthetic mutation bodies.""" + +import hashlib +import json +from pathlib import Path +import sys + + +SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", + "alert-delete", "rate-update", "uncertain", "slow-cancel") + + +def main(): + if len(sys.argv) != 2: + raise SystemExit("usage: validate.py OUTPUT_DIRECTORY") + root = Path(sys.argv[1]) + requests = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] + for scene in SCENES: + evidence = json.loads((root / (scene + "-evidence.json")).read_text()) + expected_status = 130 if scene == "slow-cancel" else 1 if scene == "uncertain" else 0 + assert evidence["exit_status"] == expected_status, (scene, "incorrect child exit") + assert (evidence["width"], evidence["height"]) in {(80, 24), (40, 12)} + raw = (root / (scene + "-evidence.tty")).read_bytes() + child = [json.loads(line) for line in (root / (scene + "-evidence.child.cast")).read_text().splitlines()] + child_bytes = "".join(event[2] for event in child[1:] if event[1] == "o").encode() + outer = [json.loads(line) for line in (root / (scene + ".cast")).read_text().splitlines()] + outer_bytes = "".join(event[2] for event in outer[1:] if event[1] == "o").encode() + marker = ("$ " + " ".join(evidence["command"]) + "\r\n").encode() + assert outer_bytes.count(marker) == 1, (scene, "ambiguous command boundary") + relay = outer_bytes.split(marker, 1)[1] + prompt = b"\r\n$ " + assert relay.endswith(prompt), (scene, "missing restored prompt") + assert relay[:-len(prompt)] == raw == child_bytes, (scene, "capture changed CLI bytes") + assert hashlib.sha256(raw).hexdigest() == evidence["relayed_sha256"] + mutations = [entry for entry in requests if entry["scene"] == scene and entry["method"] in {"POST", "DELETE"}] + assert len(mutations) == (0 if scene == "slow-cancel" else 1), (scene, "unexpected mutation count") + if scene == "slow-cancel": + assert any(entry["scene"] == scene and entry["status"] == "held" for entry in requests) + continue + request = mutations[0] + if scene in {"org-limit", "project-limit", "uncertain"}: + assert request["body"] == {"currency": "USD", "interval": "month", "threshold_amount": 12525} + elif scene in {"alert-create", "alert-edit"}: + assert request["body"] == {"currency": "USD", "interval": "month", + "threshold_amount": 7550 if scene == "alert-create" else 15025, + "notification_channel": {"type": "email", "recipients": ["finance@example.test"], + "subject_prefix": "Demo" if scene == "alert-create" else None}} + elif scene == "org-limit-delete": + assert request["method"] == "DELETE" and request["path"].endswith("/spend_limit") + assert request["body"] is None + elif scene == "alert-delete": + assert request["method"] == "DELETE" and request["path"].endswith("/spend_alerts/alert_demo") + elif scene == "rate-update": + assert request["body"] == {"max_requests_per_1_minute": 250} + if scene == "uncertain": + assert request["status"] == "response_lost_after_write" + else: + assert request["status"] == 200 + assert "review-apply" in {point["name"] for point in evidence["checkpoints"]} + assert [event["key"] for event in evidence["keys"]].count("Apply") == 1 + final_screen = (root / (scene + "-evidence.exit.txt")).read_text() + assert "> Apply change" not in final_screen, (scene, "stale confirmation after exit") + print("PASS: nine real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") + print("PASS: one mutation per confirmed action; no retry after uncertain outcome; slow load canceled without mutation.") + print("LIMIT: inspect all checkpoint images and GIF frames before claiming visual acceptance.") + + +if __name__ == "__main__": + main() From 56915557f5ed6fef90beac0deb2553a643e6f1a3 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 10:51:56 -0700 Subject: [PATCH 05/44] test: align limits replay with confirmed outcomes --- scripts/demos/limits-workspace/scene.py | 2 +- scripts/demos/limits-workspace/validate.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index 655aa984..a6de8458 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -211,7 +211,7 @@ def run_scene(terminal, scene, evidence, output): key(terminal, evidence, "Quit", b"q") terminal.finish(1) else: - wait_screen(terminal, ["Removal confirmed" if scene == "org-limit-delete" else "Saved successfully"], ["Loading", "Saving"]) + wait_screen(terminal, ["Removal confirmed" if scene == "org-limit-delete" else "Change confirmed."], ["Loading", "Saving"]) checkpoint(terminal, evidence, output, "saved") key(terminal, evidence, "Quit", b"q") terminal.finish(0) diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 6a20ce84..62a63ab3 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -48,7 +48,7 @@ def main(): "subject_prefix": "Demo" if scene == "alert-create" else None}} elif scene == "org-limit-delete": assert request["method"] == "DELETE" and request["path"].endswith("/spend_limit") - assert request["body"] is None + assert request.get("body") is None elif scene == "alert-delete": assert request["method"] == "DELETE" and request["path"].endswith("/spend_alerts/alert_demo") elif scene == "rate-update": From cd195fc896d8965851c74c01eaee9b488e1ea503 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 10:58:46 -0700 Subject: [PATCH 06/44] test: correct Limits receipts and execute shell previews --- pkg/custom/limits_workspace_test.go | 79 +++++++++++++++++++++++++++-- 1 file changed, 76 insertions(+), 3 deletions(-) diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index abc6fc28..c460e1be 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -3,10 +3,14 @@ package custom import ( "bytes" "context" + "encoding/json" "errors" "io" "math" + "os" + "path/filepath" "reflect" + "runtime" "strings" "sync/atomic" "syscall" @@ -194,7 +198,7 @@ func TestLimitsWorkspaceBusyAndFailedSavePreventRepeatedWrites(t *testing.T) { } close(release) m.Update(limitsControllerRun(t, command)) - if !m.locked || m.apply || m.page != "form" || m.draft.inputs[0].Value() != "100.01" || !strings.Contains(m.formError, "inspect") { + if !m.locked || m.apply || m.page != "form" || m.draft.inputs[0].Value() != "100.01" || !strings.Contains(m.formError, "Press ? for recovery") { t.Fatalf("failed save discarded draft or unlocked writes: page=%s locked=%t error=%q", m.page, m.locked, m.formError) } m.setFocus(len(m.draft.inputs)) @@ -205,6 +209,75 @@ func TestLimitsWorkspaceBusyAndFailedSavePreventRepeatedWrites(t *testing.T) { if b.writes.Load() != 1 || m.busy || !m.locked { t.Fatal("failed save permitted an unconfirmed repeat") } + limitsControllerKey(m, '?') + if m.page != "details" || !strings.Contains(m.details, "It may have succeeded") || !strings.Contains(m.details, "help admin organization spend-limit retrieve") { + t.Fatal("failed save did not provide uncertainty guidance and a safe read") + } +} + +func TestLimitsDraftCommandPreviewNativeShellPreservesArguments(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Bash/Zsh preview execution uses a POSIX recorder; native Windows requires separate coverage") + } + for _, shell := range []string{"bash", "zsh"} { + t.Run(shell, func(t *testing.T) { + binary, directory := audioPreviewExecutable(t, "openai limits' recorder") + marker := filepath.Join(t.TempDir(), "MUST_NOT_EXIST") + subject := "Owner's \"alert\"; $(touch " + marker + "); `touch " + marker + "` $HOME \\path\nnext line" + channel, err := json.Marshal(map[string]any{"type": "email", "recipients": []string{"finance@example.test"}, "subject_prefix": subject}) + if err != nil { + t.Fatal(err) + } + requestArgs := []string{"--organization=org owner's", "--project=proj root's", "--base-url=https://example.invalid/proxy/v1"} + cases := []struct { + name string + projectID string + change limitsChange + want []string + }{ + {name: "organization-limit", change: limitsChange{Kind: "limit_set", Amount: 12525}, + want: []string{"admin", "organization", "spend-limit", "update", "--currency", "USD", "--interval", "month", "--threshold-amount", "12525"}}, + {name: "project-limit", projectID: "proj target's", change: limitsChange{Kind: "limit_set", Amount: 12525}, + want: []string{"admin", "organization", "projects", "spend-limit", "update", "--project-id", "proj target's", "--currency", "USD", "--interval", "month", "--threshold-amount", "12525"}}, + {name: "organization-limit-delete", change: limitsChange{Kind: "limit_delete"}, + want: []string{"admin", "organization", "spend-limit", "delete"}}, + {name: "project-limit-delete", projectID: "proj target's", change: limitsChange{Kind: "limit_delete"}, + want: []string{"admin", "organization", "projects", "spend-limit", "delete", "--project-id", "proj target's"}}, + {name: "alert-create", change: limitsChange{Kind: "alert_create", Amount: 7550, Recipients: []string{"finance@example.test"}, SubjectPrefixSet: true, SubjectPrefix: &subject}, + want: []string{"admin", "organization", "spend-alerts", "create", "--currency", "USD", "--interval", "month", "--threshold-amount", "7550", "--notification-channel", string(channel)}}, + {name: "alert-update", projectID: "proj target's", change: limitsChange{Kind: "alert_update", ID: "alert owner's", Amount: 15025, Recipients: []string{"finance@example.test"}, SubjectPrefixSet: true, SubjectPrefix: &subject}, + want: []string{"admin", "organization", "projects", "spend-alerts", "update", "--project-id", "proj target's", "--alert-id", "alert owner's", "--currency", "USD", "--interval", "month", "--threshold-amount", "15025", "--notification-channel", string(channel)}}, + {name: "alert-delete", projectID: "proj target's", change: limitsChange{Kind: "alert_delete", ID: "alert owner's"}, + want: []string{"admin", "organization", "projects", "spend-alerts", "delete", "--project-id", "proj target's", "--alert-id", "alert owner's"}}, + {name: "rate-update", projectID: "proj target's", change: limitsChange{Kind: "rate_update", ID: "rl owner's", RateValues: map[string]int64{"max_requests_per_1_minute": -1}}, + want: []string{"admin", "organization", "projects", "rate-limits", "update-rate-limit", "--project-id", "proj target's", "--rate-limit-id", "rl owner's", "--max-requests-per-1-minute", "-1"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + review := limitsReview(tc.change, tc.projectID, requestArgs, false, shell, binary) + _, tail, found := strings.Cut(review, "\nDirect API command\n") + if !found { + t.Fatalf("review omitted its copyable command: %q", review) + } + command, guidance, found := strings.Cut(tail, "\n\n") + if !found || command == "" || !strings.HasPrefix(guidance, "This command skips the workspace preflight and can retry automatically.") { + t.Fatalf("review did not separate the executable command from its guidance: %q", tail) + } + capture := executeAudioPreview(t, shell, command, directory) + want := append(append([]string{}, requestArgs...), tc.want...) + if !reflect.DeepEqual(capture.args, want) { + t.Fatalf("printed command changed arguments:\n got: %q\nwant: %q", capture.args, want) + } + if len(capture.stdin) != 0 || capture.protected != "unset" { + t.Fatal("printed command unexpectedly supplied stdin or changed its environment") + } + if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("printed command executed subject text: %v", err) + } + }) + } + }) + } } func TestLimitsWorkspaceControlCFromEveryState(t *testing.T) { @@ -634,7 +707,7 @@ func TestLimitsWorkspaceReceiptUsesMutationProjection(t *testing.T) { m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month","enforcement":{"status":"inactive"}}`) m.workerSavedOperation = limitsChangeOperation("limit_set", false) var out bytes.Buffer - err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: format, Stdout: &out, Stderr: io.Discard}) + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: format, ExplicitFormat: true, Stdout: &out, Stderr: io.Discard}) if err != nil { t.Fatal(err) } @@ -642,7 +715,7 @@ func TestLimitsWorkspaceReceiptUsesMutationProjection(t *testing.T) { if !strings.Contains(out.String(), "USD 100.01") || !strings.Contains(out.String(), "inactive") { t.Fatalf("receipt lost threshold units or returned state: %q", out.String()) } - } else if !strings.Contains(out.String(), `"threshold_amount": 10001`) || strings.Contains(out.String(), "spend_threshold") { + } else if !gjson.Valid(out.String()) || gjson.Get(out.String(), "threshold_amount").Raw != "10001" || strings.Contains(out.String(), "spend_threshold") { t.Fatalf("machine receipt changed threshold data: %q", out.String()) } }) From c1f589981a97dfa4cf5250859eb2cd179daae45b Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 11:05:31 -0700 Subject: [PATCH 07/44] test: import atomic counters for Limits Costs coverage --- cmd/openai/limits_workflow_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go index 6baee588..9f3f706f 100644 --- a/cmd/openai/limits_workflow_test.go +++ b/cmd/openai/limits_workflow_test.go @@ -9,6 +9,7 @@ import ( "strconv" "strings" "sync" + "sync/atomic" "testing" "github.com/tidwall/gjson" From 55374ff697dd60e37a0c446cacbebd22cba73bf9 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 11:18:27 -0700 Subject: [PATCH 08/44] test: exercise real shell previews in Limits demos --- scripts/demos/limits-workspace/record.sh | 4 +- scripts/demos/limits-workspace/scene.py | 71 +++++++++++++++++++--- scripts/demos/limits-workspace/validate.py | 19 +++++- 3 files changed, 84 insertions(+), 10 deletions(-) diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh index 248529f1..6ac48eaf 100755 --- a/scripts/demos/limits-workspace/record.sh +++ b/scripts/demos/limits-workspace/record.sh @@ -14,7 +14,8 @@ case "$record_width" in 80) record_height=24;; 40) record_height=12;; *) echo 'D case "$record_no_color" in 0|1) ;; *) echo 'DEMO_NO_COLOR must be 0 or 1.' >&2; exit 2;; esac source "$record_source/../capture_and_render.sh" # This workflow has one candidate binary. Passing it twice reuses the shared lifecycle. -demo_prepare_capture "$record_root" "$1" "$1" "$2" "$2" "$3" "$record_source/server.py" +# The shared cleanup also owns synthetic executable paths shown in command previews. +TMPDIR=/private/tmp demo_prepare_capture "$record_root" "$1" "$1" "$2" "$2" "$3" "$record_source/server.py" demo_python="$(command -v python3)" record_digest="$(shasum -a 256 "$demo_after" | cut -d ' ' -f 1)" demo_start_api "$demo_output/requests.jsonl" "$demo_output/fixtures.json" "$demo_runtime/gates" @@ -56,6 +57,7 @@ for record_scene in "${record_scenes[@]}"; do "DEMO_PYTHON=$demo_python" "DEMO_ASCIINEMA=$demo_asciinema" "DEMO_DRIVER=$record_source/scene.py" \ "DEMO_SCENE_NAME=$record_scene" "DEMO_EVIDENCE_BASE=$demo_output/$record_scene-evidence" \ "DEMO_WIDTH=$record_width" "DEMO_HEIGHT=$record_height" "DEMO_NO_COLOR=$record_no_color" \ + "DEMO_RUNTIME_DIRECTORY=$demo_runtime" \ "DEMO_GATE_DIRECTORY=$demo_runtime/gates" "DEMO_REQUEST_LOG=$demo_output/requests.jsonl" done demo_stop_api diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index a6de8458..b9b30d01 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -18,6 +18,14 @@ harness = importlib.util.module_from_spec(spec) spec.loader.exec_module(harness) +# Trapped signals retain their default disposition in the foreground CLI child. +BASH_WORKSPACE = r'''trap ':' HUP INT TERM +"$@" +limits_scene_status=$? +printf '%s\n' "$limits_scene_status" > "$DEMO_CLI_EXIT" || exit 74 +exit "$limits_scene_status" +''' + def write_text(path, value): with Path(path).open("x", encoding="utf-8") as destination: @@ -98,8 +106,11 @@ def mutations(scene): return [item for item in requests(scene) if item["method"] in {"POST", "DELETE"}] -def checkpoint(terminal, evidence, output, name): +def checkpoint(terminal, evidence, output, name, required=()): screen = screen_text(terminal) + compact = "".join(line.rstrip() for line in screen.splitlines()) + if not all(text in compact for text in required): + raise AssertionError(f"checkpoint {name} omitted its required preview: {required!r}") write_text(Path(str(output) + f".{name}.txt"), screen) evidence["checkpoints"].append({"name": name, "seconds": time.monotonic() - terminal.started}) drain(terminal, 0.6) @@ -120,13 +131,26 @@ def replace(terminal, evidence, label, value): def review(terminal, evidence, scene, output): wait_screen(terminal, ["> Back", "Enter: select"]) checkpoint(terminal, evidence, output, "review-top") - for _ in range(30): + preview_seen = False + for _ in range(40): screen = screen_text(terminal) + compact = "".join(line.rstrip() for line in screen.splitlines()) + if "Command printing needs" in compact or "Command preview omitted" in compact: + raise AssertionError("review omitted the real command preview") + if not preview_seen and "Direct API command" in screen and evidence["staged_binary"]["path"] in compact: + checkpoint(terminal, evidence, output, "review-command", + ("Direct API command", evidence["staged_binary"]["path"])) + preview_seen = True if "Apply change" in screen and "Read to the end" not in screen: break - key(terminal, evidence, "PageDown", b"\x1b[6~") + if not preview_seen and "Direct API command" in screen: + key(terminal, evidence, "Show command preview", b"\x1b[B") + else: + key(terminal, evidence, "PageDown", b"\x1b[6~") else: raise AssertionError("review never exposed its final action") + if not preview_seen: + raise AssertionError("review reached Apply without a command preview checkpoint") if mutations(scene): raise AssertionError("fixture received a mutation before explicit confirmation") key(terminal, evidence, "Tab to Apply", b"\t") @@ -227,26 +251,54 @@ def main(): binary = shutil.which("openai") if binary is None: raise RuntimeError("immutable CLI binary is unavailable") + binary = Path(binary).resolve(strict=True) + runtime = Path(os.environ["DEMO_RUNTIME_DIRECTORY"]).resolve(strict=True) + if runtime.parent != Path("/private/tmp") or not runtime.name.startswith("cli-demo."): + raise AssertionError("the recorder runtime must use an owned synthetic /private/tmp directory") project = scene in {"project-limit", "alert-edit", "rate-update"} args = ["projects", "limits", "--project-id", "proj_demo"] if project else ["admin", "limits"] evidence = {"scene": scene, "width": width, "height": height, "command": ["openai", *args], - "keys": [], "checkpoints": [], "binary_sha256": hashlib.sha256(Path(binary).read_bytes()).hexdigest()} + "keys": [], "checkpoints": [], "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "parent_shell": "/bin/bash", "recorder_runtime": str(runtime)} print("$ openai " + " ".join(args), flush=True) original = termios.tcgetattr(1) relay = termios.tcgetattr(1) relay[1] &= ~termios.OPOST try: termios.tcsetattr(1, termios.TCSANOW, relay) - with tempfile.TemporaryDirectory(prefix="limits-demo-home-") as temporary: - env = {"PATH": "/usr/bin:/bin", "HOME": temporary, "TERM": "xterm-256color", "LANG": "en_US.UTF-8", + with tempfile.TemporaryDirectory(prefix="openai-limits-demo-", dir=runtime) as temporary: + staged = Path(temporary) / "openai" + shutil.copy2(binary, staged) + staged_hash = hashlib.sha256(staged.read_bytes()).hexdigest() + if staged_hash != evidence["binary_sha256"]: + raise AssertionError("staged CLI differs from the supplied binary") + evidence["staged_binary"] = {"path": str(staged), "sha256_before": staged_hash} + home = Path(temporary) / "home" + home.mkdir(mode=0o700) + exit_receipt = Path(temporary) / "cli-exit-status" + env = {"PATH": temporary + ":/usr/bin:/bin", "HOME": str(home), "TERM": "xterm-256color", "LANG": "en_US.UTF-8", "GOMAXPROCS": "2", "PAGER": "cat", "OPENAI_API_KEY": "synthetic-demo-key", - "OPENAI_ADMIN_KEY": "synthetic-demo-admin-key", "OPENAI_BASE_URL": os.environ["OPENAI_BASE_URL"]} + "OPENAI_ADMIN_KEY": "synthetic-demo-admin-key", "OPENAI_BASE_URL": os.environ["OPENAI_BASE_URL"], + "DEMO_CLI_EXIT": str(exit_receipt)} if os.environ.get("DEMO_NO_COLOR", "1") == "1": env.update(NO_COLOR="1", FORCE_COLOR="0") terminal = None try: - terminal = RelayTerminal(binary, args, env, width=width, height=height, controlling_terminal=True) + # Post-command status handling keeps Bash as the CLI's real parent. + # Both processes inherit the existing PTY session and process group. + shell_args = ["--noprofile", "--norc", "-c", BASH_WORKSPACE, "bash", str(staged), *args] + terminal = RelayTerminal("/bin/bash", shell_args, env, width=width, height=height, controlling_terminal=True) run_scene(terminal, scene, evidence, output) + status_text = exit_receipt.read_text() + if status_text != str(terminal.child.returncode) + "\n": + raise AssertionError("Bash did not preserve the actual CLI exit status") + evidence["cli_exit_status"] = int(status_text) + evidence["staged_binary"]["sha256_after"] = hashlib.sha256(staged.read_bytes()).hexdigest() + if (evidence["staged_binary"]["sha256_after"] != evidence["binary_sha256"] + or hashlib.sha256(binary.read_bytes()).hexdigest() != evidence["binary_sha256"]): + raise AssertionError("supplied or staged CLI changed during capture") + if str(binary).encode() in terminal.raw: + raise AssertionError("capture exposed the private binary source path") evidence.update(exit_status=terminal.child.returncode, relayed_bytes=len(terminal.raw), relayed_sha256=hashlib.sha256(terminal.raw).hexdigest()) with Path(str(output) + ".tty").open("xb") as destination: @@ -258,6 +310,9 @@ def main(): finally: if terminal is not None: terminal.close() + evidence["staged_binary"]["removed"] = not Path(temporary).exists() + if not evidence["staged_binary"]["removed"]: + raise AssertionError("owned staged CLI directory remains after cleanup") finally: termios.tcsetattr(1, termios.TCSANOW, original) write_text(Path(str(output) + ".json"), json.dumps(evidence, indent=2) + "\n") diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 62a63ab3..9fa3aa14 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -20,6 +20,16 @@ def main(): evidence = json.loads((root / (scene + "-evidence.json")).read_text()) expected_status = 130 if scene == "slow-cancel" else 1 if scene == "uncertain" else 0 assert evidence["exit_status"] == expected_status, (scene, "incorrect child exit") + assert evidence["cli_exit_status"] == evidence["exit_status"], (scene, "Bash changed the CLI exit status") + assert evidence["parent_shell"] == "/bin/bash", (scene, "missing native Bash parent") + staged = evidence["staged_binary"] + staged_path = Path(staged["path"]) + runtime = Path(evidence["recorder_runtime"]) + assert runtime.parent == Path("/private/tmp") and runtime.name.startswith("cli-demo.") + assert staged_path.name == "openai" and staged_path.parent.parent == runtime + assert staged_path.parent.name.startswith("openai-limits-demo-") + assert staged["sha256_before"] == staged["sha256_after"] == evidence["binary_sha256"] + assert staged["removed"] is True and not staged_path.parent.exists(), (scene, "staged CLI cleanup incomplete") assert (evidence["width"], evidence["height"]) in {(80, 24), (40, 12)} raw = (root / (scene + "-evidence.tty")).read_bytes() child = [json.loads(line) for line in (root / (scene + "-evidence.child.cast")).read_text().splitlines()] @@ -57,12 +67,19 @@ def main(): assert request["status"] == "response_lost_after_write" else: assert request["status"] == 200 - assert "review-apply" in {point["name"] for point in evidence["checkpoints"]} + checkpoints = {point["name"]: point for point in evidence["checkpoints"]} + assert "review-command" in checkpoints and "review-apply" in checkpoints + assert checkpoints["review-command"]["seconds"] < checkpoints["review-apply"]["seconds"] + preview = (root / (scene + "-evidence.review-command.txt")).read_text() + preview = "".join(line.rstrip() for line in preview.splitlines()) + assert "Direct API command" in preview and staged["path"] in preview, (scene, "missing real command preview") + assert "Command printing needs" not in preview and "Command preview omitted" not in preview assert [event["key"] for event in evidence["keys"]].count("Apply") == 1 final_screen = (root / (scene + "-evidence.exit.txt")).read_text() assert "> Apply change" not in final_screen, (scene, "stale confirmation after exit") print("PASS: nine real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") print("PASS: one mutation per confirmed action; no retry after uncertain outcome; slow load canceled without mutation.") + print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; eight real command previews; staged files removed.") print("LIMIT: inspect all checkpoint images and GIF frames before claiming visual acceptance.") From b07ef80e311a650aecaf240d878e39223019b267 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 11:30:00 -0700 Subject: [PATCH 09/44] fix: preserve safe limits validation recovery --- pkg/custom/limits_commands.go | 4 +- pkg/custom/limits_workspace.go | 32 +++++-- pkg/custom/limits_workspace_test.go | 129 ++++++++++++++++++++++++++++ pkg/custom/readable_errors.go | 3 + 4 files changed, 161 insertions(+), 7 deletions(-) diff --git a/pkg/custom/limits_commands.go b/pkg/custom/limits_commands.go index 5fca45b6..4bc0da30 100644 --- a/pkg/custom/limits_commands.go +++ b/pkg/custom/limits_commands.go @@ -16,7 +16,9 @@ import ( // Aggregate workflows retain their causes without presenting private source errors. type limitsWorkflowFailure struct { message string - cause error + // Only the text error presenter includes this additional recovery guidance. + humanDetail string + cause error } func (failure *limitsWorkflowFailure) Error() string { return failure.message } diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index d2f508d1..8c1beb63 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -146,9 +146,12 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error causes = append(causes, mutationErr) } incomplete := m.refreshRequired || sourcesIncomplete || workerErr != nil - message := "" + message, humanDetail := "", "" if pending || m.locked { message = "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + if mutationErr != nil { + humanDetail = limitsMutationHint(mutationErr, m.invocation) + } } else if m.refreshRequired { message = limitsPreflightHint(mutationErr, m.invocation) if m.lastSaved.Exists() { @@ -205,7 +208,7 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error } if outputErr != nil || interrupted || ctx.Err() != nil || pending || m.locked || programErr != nil { if message != "" { - operationErr = &limitsWorkflowFailure{message: message, cause: operationErr} + operationErr = &limitsWorkflowFailure{message: message, humanDetail: humanDetail, cause: operationErr} } failure := errors.Join(operationErr, outputErr) if interrupted { @@ -787,8 +790,15 @@ func limitsReadHint(err error, invocation string) string { return summary.Error() + "\nCosts options: " + invocation + " help admin usage costs" } var apiErr *openai.Error - if errors.As(err, &apiErr) && (apiErr.StatusCode == 401 || apiErr.StatusCode == 403 || apiErr.StatusCode == 404 || apiErr.StatusCode == 429) { - return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "No change was attempted.") + if errors.As(err, &apiErr) { + switch apiErr.StatusCode { + case 401, 403, 404, 429: + return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "No change was attempted.") + case 400, 422: + return fmt.Sprintf("The API returned HTTP %d. No change was attempted.\nRecheck the request fields and scope, then refresh.", apiErr.StatusCode) + default: + return fmt.Sprintf("The API returned HTTP %d. No change was attempted.\nPress r to refresh. If this continues, check API availability and access.", apiErr.StatusCode) + } } if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { return "The read was interrupted. Press r to refresh." @@ -798,8 +808,18 @@ func limitsReadHint(err error, invocation string) string { func limitsMutationHint(err error, invocation string) string { var apiErr *openai.Error - if errors.As(err, &apiErr) && (apiErr.StatusCode == 401 || apiErr.StatusCode == 403 || apiErr.StatusCode == 404 || apiErr.StatusCode == 429) { - return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "The change is not confirmed.") + if errors.As(err, &apiErr) { + switch apiErr.StatusCode { + case 401, 403, 404, 429: + return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "The change is not confirmed.") + case 400, 422: + return fmt.Sprintf("The API returned HTTP %d. The change is not confirmed.\nRecheck the reviewed fields. Read the current record before another change.", apiErr.StatusCode) + default: + return fmt.Sprintf("The API returned HTTP %d. The change is not confirmed.\nRead the current record before another change.", apiErr.StatusCode) + } + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "The response was interrupted. The change is not confirmed.\nRead the current record before another change." } if err != nil { switch err.Error() { diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index c460e1be..651a2de3 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -5,8 +5,10 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" "math" + "net/http" "os" "path/filepath" "reflect" @@ -22,6 +24,7 @@ import ( "github.com/charmbracelet/x/ansi" clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" "github.com/tidwall/gjson" ) @@ -622,6 +625,132 @@ func TestLimitsPreflightGuidancePreservesFailureScope(t *testing.T) { if !strings.Contains(got, tc.want) || strings.Contains(got, tc.reject) || strings.Contains(got, err.Message) || !strings.Contains(got, "No write sent.") { t.Errorf("status=%d code=%s yielded unsafe or incorrect recovery: %q", tc.status, tc.code, got) } + for _, hint := range []string{limitsReadHint(err, "custom-openai"), limitsMutationHint(err, "custom-openai")} { + if !strings.Contains(hint, tc.want) || strings.Contains(hint, tc.reject) || strings.Contains(hint, err.Message) || strings.Contains(hint, "No write sent.") { + t.Errorf("status=%d code=%s lost its read or mutation classification: %q", tc.status, tc.code, hint) + } + } + } +} + +func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { + for _, status := range []int{400, 422, 409, 503} { + for _, operation := range []string{"read", "mutation"} { + t.Run(fmt.Sprintf("%s/%d", operation, status), func(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet && operation == "mutation" { + if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitsTestWrite(t, w, limitsTestLimit) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + return + } + if r.Method != http.MethodGet { + writes.Add(1) + } + w.WriteHeader(status) + limitsTestWrite(t, w, `{"error":{"message":"synthetic-private-message\u001b[2J","param":"synthetic-private-param\nline","code":"synthetic-private-code\u001b]0;title\u0007"}}`) + }) + m, _ := newLimitsController(t, 80, 24) + m.backend = backend + var result limitsWorkResult + if operation == "read" { + result = limitsControllerRun(t, m.load()) + } else { + backend.client = openai.NewClient(append(backend.client.Options, option.WithMaxRetries(3))...) + m.snapshot.LimitAbsent, m.snapshot.Limit = false, gjson.Parse(limitsTestLimit) + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + result = limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter)) + var attempted *limitsMutationFailure + if !errors.As(result.err, &attempted) || !attempted.Attempted || writes.Load() != 1 { + t.Fatal("failed mutation lost its attempted state or sent more than once") + } + } + m.Update(result) + limitsControllerKey(m, tea.KeyF1) + if !strings.Contains(m.details, fmt.Sprintf("HTTP %d", status)) || strings.Contains(m.details, "synthetic-private") || strings.Contains(m.details, "\x1b") { + t.Fatal("recovery omitted the status or exposed response data") + } + if status == 400 || status == 422 { + fields := "request fields and scope" + if operation == "mutation" { + fields = "reviewed fields" + } + if !strings.Contains(m.details, fields) { + t.Fatal("validation recovery did not identify fields to recheck") + } + } + if operation == "read" { + if writes.Load() != 0 || !strings.Contains(m.details, "No change was attempted.") || strings.Contains(m.details, "It may have succeeded") { + t.Fatal("read failure was presented as an attempted mutation") + } + return + } + if !m.locked || !strings.Contains(m.details, "It may have succeeded") || strings.Contains(m.details, "No write sent.") { + t.Fatal("HTTP failure removed mutation uncertainty or the write lock") + } + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, tea.KeyEnter) + m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) + limitsControllerKey(m, tea.KeyEnter) + if !m.locked || writes.Load() != 1 { + t.Fatal("Enter or refresh repeated a failed mutation") + } + m.output = &listNavigationOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) + if !errors.Is(failure, result.err) || !errors.Is(failure, syscall.EPIPE) || clioutput.IsBrokenPipe(failure) { + t.Fatal("finalization discarded the mutation or output failure") + } + baseline := "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + for _, args := range [][]string{nil, {"--format", "json", "--format-error", "text"}, {"--format", "text", "--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { + root := readableErrorTestCommand(t, args...) + var out bytes.Buffer + if err := ShowCommandError(root, failure, &out); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "synthetic-private") || strings.Contains(out.String(), "\x1b") { + t.Fatal("final diagnostic exposed response data") + } + switch { + case root.String("transform-error") != "": + if out.String() != baseline+"\n" { + t.Fatal("extracted error message changed") + } + case errorOutputFormat(root) == "json": + var value map[string]any + if err := json.Unmarshal(out.Bytes(), &value); err != nil || len(value) != 1 || value["message"] != baseline { + t.Fatal("machine error message changed") + } + default: + if !strings.Contains(out.String(), fmt.Sprintf("HTTP %d", status)) || !strings.Contains(out.String(), "may have succeeded") { + t.Fatal("final text lost HTTP status or mutation uncertainty") + } + if (status == 400 || status == 422) && !strings.Contains(out.String(), "reviewed fields") { + t.Fatal("final text omitted validation recovery") + } + } + } + }) + } + } +} + +func TestLimitsWorkspaceCanceledMutationRecovery(t *testing.T) { + for _, cause := range []error{context.Canceled, context.DeadlineExceeded} { + m, _ := newLimitsController(t, 80, 24) + m.locked, m.mutationError = true, &limitsMutationFailure{Err: cause, Attempted: true} + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) + var out bytes.Buffer + if err := ShowCommandError(readableErrorTestCommand(t), failure, &out); err != nil { + t.Fatal(err) + } + if !errors.Is(failure, cause) || !strings.Contains(out.String(), "interrupted") || strings.Contains(out.String(), "Check the API address and connection") || !strings.Contains(out.String(), "may have succeeded") { + t.Fatal("canceled mutation lost its cause or gave connection advice") + } } } diff --git a/pkg/custom/readable_errors.go b/pkg/custom/readable_errors.go index 6af48144..74cb626f 100644 --- a/pkg/custom/readable_errors.go +++ b/pkg/custom/readable_errors.go @@ -121,6 +121,9 @@ func showCommandError(ctx context.Context, root *cli.Command, failure error, out var batchFailure *batchcommands.WorkflowError if limitsFailure != nil { message = limitsFailure.Error() + if limitsFailure.humanDetail != "" { + message += "\n" + limitsFailure.humanDetail + } } else if errors.As(failure, &batchFailure) { message = batchFailure.Error() } else if isAPI && !errors.Is(failure, context.Canceled) { From 662a05c7b564f9133543ac734e8704eacfe27ff7 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 11:34:12 -0700 Subject: [PATCH 10/44] fix: adapt limits output to shared terminal writer --- pkg/custom/limits_workspace.go | 40 +++++++- pkg/custom/limits_workspace_test.go | 137 +++++++++++++++++++++++++++- 2 files changed, 169 insertions(+), 8 deletions(-) diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index 8c1beb63..4ea575d4 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -45,7 +45,7 @@ type limitsWorkspace struct { ctx context.Context cancel context.CancelFunc backend limitsBackend - output *listNavigationOutput + output *limitsWorkspaceOutput snapshot limitsSnapshot width, height, section, selection, focus int page, note, formError string @@ -71,6 +71,40 @@ type limitsInterrupt struct{ error } func (*limitsInterrupt) ExitCode() int { return 130 } func (e *limitsInterrupt) Unwrap() error { return e.error } +// limitsWorkspaceOutput records failures that the terminal renderer may ignore. +// Its output context stays active so shutdown can still restore terminal modes. +type limitsWorkspaceOutput struct { + clioutput.TerminalWriter + stop context.CancelFunc + mu sync.Mutex + err error +} + +var _ term.File = (*limitsWorkspaceOutput)(nil) + +func (w *limitsWorkspaceOutput) Write(data []byte) (int, error) { + n, err := w.Writer.Write(data) + if err != nil { + w.mu.Lock() + if w.err == nil { + w.err = err + } + w.mu.Unlock() + w.stop() + } + return n, err +} + +func (w *limitsWorkspaceOutput) WriteString(data string) (int, error) { + return w.Write([]byte(data)) +} + +func (w *limitsWorkspaceOutput) Err() error { + w.mu.Lock() + defer w.mu.Unlock() + return w.err +} + func runLimitsWorkspace(ctx context.Context, cmd *cli.Command, backend limitsBackend, opts ShowJSONOpts) error { file := opts.Stdout.(*os.File) width, height, err := term.GetSize(file.Fd()) @@ -90,8 +124,8 @@ func runLimitsWorkspace(ctx context.Context, cmd *cli.Command, backend limitsBac if err != nil || strings.HasPrefix(m.invocation, "go run ") { m.omitPreview = true } - m.output = &listNavigationOutput{terminalOutputWriter: terminalOutputWriter{ - Writer: clioutput.Writer{Context: context.Background(), Sink: opts.Stdout}, file: file}, stop: stop} + m.output = &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ + Writer: clioutput.Writer{Context: context.Background(), Sink: opts.Stdout}, File: file}, stop: stop} options := []tea.ProgramOption{tea.WithInput(cmd.Root().Reader), tea.WithOutput(m.output), tea.WithContext(programContext), tea.WithWindowSize(width, height)} if !m.color { options = append(options, tea.WithColorProfile(colorprofile.NoTTY)) diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index 651a2de3..101f05f3 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -700,7 +700,7 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { if !m.locked || writes.Load() != 1 { t.Fatal("Enter or refresh repeated a failed mutation") } - m.output = &listNavigationOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} + m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) if !errors.Is(failure, result.err) || !errors.Is(failure, syscall.EPIPE) || clioutput.IsBrokenPipe(failure) { t.Fatal("finalization discarded the mutation or output failure") @@ -776,6 +776,133 @@ type limitsFailingWriter struct{ err error } func (w limitsFailingWriter) Write([]byte) (int, error) { return 0, w.err } +type limitsWorkspaceWriteFunc func([]byte) (int, error) + +func (write limitsWorkspaceWriteFunc) Write(data []byte) (int, error) { return write(data) } + +func TestLimitsWorkspaceOutputPreservesFailuresAndCleanup(t *testing.T) { + for _, method := range []string{"Write", "WriteString"} { + for _, failure := range []string{"sink error", "short write"} { + t.Run(method+"/"+failure, func(t *testing.T) { + ctx, stop := context.WithCancel(t.Context()) + defer stop() + first, later := errors.New("synthetic output failure"), errors.New("later output failure") + want := first + if failure == "short write" { + want = io.ErrShortWrite + } + calls, cleanup := 0, "" + sink := limitsWorkspaceWriteFunc(func(data []byte) (int, error) { + calls++ + switch calls { + case 1: + if failure == "short write" { + return 2, nil + } + return 2, first + case 2: + cleanup = string(data) + return len(data), nil + default: + return 0, later + } + }) + out := &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ + Writer: clioutput.Writer{Context: context.Background(), Sink: sink}, + }, stop: stop} + write := func(value string) (int, error) { return out.Write([]byte(value)) } + if method == "WriteString" { + write = func(value string) (int, error) { return io.WriteString(out, value) } + } + n, original := write("result") + var writeErr *clioutput.WriteError + if n != 2 || !errors.Is(original, want) || !errors.As(original, &writeErr) { + t.Fatalf("lost partial write or sink classification: n=%d err=%v", n, original) + } + if !errors.Is(ctx.Err(), context.Canceled) || out.Err() != original { + t.Fatal("output failure did not cancel the workspace and retain the original error") + } + const restore = "\x1b[?25h" + if n, err := io.WriteString(out, restore); n != len(restore) || err != nil || cleanup != restore { + t.Fatalf("workspace cancellation prevented terminal cleanup: n=%d err=%v", n, err) + } + if _, err := out.Write([]byte("later")); !errors.Is(err, later) || out.Err() != original { + t.Fatal("a later write replaced the first output failure") + } + }) + } + } +} + +func TestLimitsWorkspaceOutputPreservesCancellation(t *testing.T) { + for _, when := range []string{"before write", "during write"} { + t.Run(when, func(t *testing.T) { + outputContext, cancelOutput := context.WithCancel(t.Context()) + defer cancelOutput() + workspaceContext, stop := context.WithCancel(t.Context()) + defer stop() + calls := 0 + out := &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ + Writer: clioutput.Writer{Context: outputContext, Sink: limitsWorkspaceWriteFunc(func(data []byte) (int, error) { + calls++ + cancelOutput() + return len(data), nil + })}, + }, stop: stop} + wantBytes, wantCalls := len("result"), 1 + if when == "before write" { + cancelOutput() + wantBytes, wantCalls = 0, 0 + } + n, err := io.WriteString(out, "result") + var writeErr *clioutput.WriteError + if n != wantBytes || calls != wantCalls || !errors.Is(err, context.Canceled) || errors.As(err, &writeErr) { + t.Fatalf("cancellation changed byte counts or became a sink failure: n=%d calls=%d err=%v", n, calls, err) + } + if out.Err() != err || !errors.Is(workspaceContext.Err(), context.Canceled) { + t.Fatal("cancellation did not stop the workspace and preserve its cause") + } + }) + } +} + +func TestLimitsWorkspaceOutputPreservesTerminalFile(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "terminal-output") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { file.Close() }) + if _, err := file.WriteString("input"); err != nil { + t.Fatal(err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + var sink bytes.Buffer + out := &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ + Writer: clioutput.Writer{Context: t.Context(), Sink: &sink}, File: file, + }, stop: func() { t.Fatal("successful output canceled the workspace") }} + if out.Fd() != file.Fd() { + t.Fatal("output wrapper changed the descriptor used for terminal detection") + } + if _, err := io.WriteString(out, "output"); err != nil || sink.String() != "output" { + t.Fatal("output wrapper bypassed its guarded sink", err) + } + input := make([]byte, len("input")) + if _, err := io.ReadFull(out, input); err != nil || string(input) != "input" { + t.Fatal("output wrapper changed terminal reads", err) + } + if out.Err() != nil { + t.Fatal("successful writes recorded an error", out.Err()) + } + if err := out.Close(); err != nil { + t.Fatal(err) + } + if _, err := file.Stat(); !errors.Is(err, os.ErrClosed) { + t.Fatal("output wrapper changed terminal file ownership", err) + } +} + func TestLimitsWorkspaceFinalizationPreservesIndependentFailures(t *testing.T) { for _, scenario := range []string{"read and terminal output", "mutation and terminal output", "cancellation and terminal output", "refresh and receipt output", "program and terminal output"} { t.Run(scenario, func(t *testing.T) { @@ -789,21 +916,21 @@ func TestLimitsWorkspaceFinalizationPreservesIndependentFailures(t *testing.T) { switch scenario { case "program and terminal output": runErr = errors.Join(original, tea.ErrProgramKilled) - m.output = &listNavigationOutput{err: writeErr} + m.output = &limitsWorkspaceOutput{err: writeErr} case "read and terminal output": m.workerError = original - m.output = &listNavigationOutput{err: writeErr} + m.output = &limitsWorkspaceOutput{err: writeErr} case "mutation and terminal output": m.mutationPending = true m.mutationError = original - m.output = &listNavigationOutput{err: writeErr} + m.output = &limitsWorkspaceOutput{err: writeErr} case "cancellation and terminal output": var cancel context.CancelFunc ctx, cancel = context.WithCancel(ctx) cancel() original = context.Canceled m.interrupted = true - m.output = &listNavigationOutput{err: writeErr} + m.output = &limitsWorkspaceOutput{err: writeErr} case "refresh and receipt output": m.snapshot.Errors = []limitsSourceFailure{{Resource: "spend_alerts", Err: original}} m.lastSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month"}`) From 7dd327fcb51c9a469c2b4a434d50b6236e30a2f8 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 11:45:03 -0700 Subject: [PATCH 11/44] fix: preserve limits failures after closed stdout pipes --- cmd/openai/limits_workflow_test.go | 8 ++- cmd/openai/limits_workflow_unix_test.go | 96 +++++++++++++++---------- pkg/custom/limits_commands.go | 12 +++- 3 files changed, 75 insertions(+), 41 deletions(-) diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go index 9f3f706f..7b490630 100644 --- a/cmd/openai/limits_workflow_test.go +++ b/cmd/openai/limits_workflow_test.go @@ -287,12 +287,16 @@ func TestMainLimitsWorkflowAmbiguousAndOptionalRecords(t *testing.T) { if got.code != 0 || got.stderr != "" { t.Fatalf("ambiguous records lost read-only access: %+v", got) } - for _, text := range []string{"Threshold amount: 10000", "Threshold amount: 90000", "Currency: USD", "Currency: EUR", "Max requests per 1 minute: 0", "Future: first", "Future: second"} { + // Ambiguous rates keep source keys; numeric segments require quoted labels. + for _, text := range []string{ + "Threshold amount: 10000", "Threshold amount: 90000", "Currency: USD", "Currency: EUR", + `"max_requests_per_1_minute": 0`, `"max_tokens_per_1_minute": 7`, "Future: first", "Future: second", + } { if !strings.Contains(got.stdout, text) { t.Errorf("readable overview lost %q: %q", text, got.stdout) } } - for _, absent := range []string{"Spend threshold:", "Alerts notify;", "not reported in this response", "Max images", "Max audio", "Max requests per day", "Max batch"} { + for _, absent := range []string{"Spend threshold:", "Alerts notify;", "not reported in this response", "Max requests per minute:", "Max tokens per minute:", "Max images", "Max audio", "Max requests per day", "Max batch"} { if strings.Contains(got.stdout, absent) { t.Errorf("ambiguous overview invented %q: %q", absent, got.stdout) } diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go index bef45044..a11cb623 100644 --- a/cmd/openai/limits_workflow_unix_test.go +++ b/cmd/openai/limits_workflow_unix_test.go @@ -15,42 +15,64 @@ import ( ) func TestMainLimitsWorkflowPartialFailureSurvivesClosedPipe(t *testing.T) { - server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { - if r.URL.Path == "/organization/spend_alerts" { - return http.StatusForbidden, `{"error":{"message":"private synthetic source failure"}}` - } - status, body := limitsWorkflowSuccess(t, false, r) - if r.URL.Path == "/organization/spend_limit" { - body = strings.TrimSuffix(body, "}") + `,"future":"` + strings.Repeat("synthetic", 2<<20) + `"}` - } - return status, body - }) - ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) - defer cancel() - child := batchesWorkflowProcess(t, ctx, server, "admin", "limits", "show", "--format", "json") - child.Env = append(child.Env, "OPENAI_ADMIN_KEY=sk-fake-spend-admin") - stdout, err := child.StdoutPipe() - if err != nil { - t.Fatal(err) - } - defer stdout.Close() - var stderr bytes.Buffer - child.Stderr = &stderr - if err := child.Start(); err != nil { - t.Fatal(err) - } - if _, err := io.ReadFull(stdout, make([]byte, 1)); err != nil { - t.Fatal(err) - } - if err := stdout.Close(); err != nil { - t.Fatal(err) - } - err = child.Wait() - exit, ok := err.(*exec.ExitError) - if !ok || exit.ExitCode() != 1 || ctx.Err() != nil || requests.count("/organization/spend_alerts") != 1 { - t.Fatalf("source failure disappeared after EPIPE: error=%v context=%v stderr=%q", err, ctx.Err(), stderr.String()) - } - if !json.Valid(stderr.Bytes()) || !strings.Contains(stderr.String(), "overview is incomplete") || strings.Contains(stderr.String(), "private synthetic") { - t.Fatalf("safe structured recovery missing: %q", stderr.String()) + for _, test := range []struct { + name string + sourceFailure bool + }{{"complete sources", false}, {"partial source failure", true}} { + t.Run(test.name, func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if test.sourceFailure && r.URL.Path == "/organization/spend_alerts" { + return http.StatusForbidden, `{"error":{"message":"private synthetic source failure"}}` + } + status, body := limitsWorkflowSuccess(t, false, r) + if r.URL.Path == "/organization/spend_limit" { + body = strings.TrimSuffix(body, "}") + `,"future":"` + strings.Repeat("synthetic", 2<<20) + `"}` + } + return status, body + }) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + child := batchesWorkflowProcess(t, ctx, server, "admin", "limits", "show", "--format", "json") + child.Env = append(child.Env, "OPENAI_ADMIN_KEY=sk-fake-spend-admin") + stdout, err := child.StdoutPipe() + if err != nil { + t.Fatal(err) + } + defer stdout.Close() + var stderr bytes.Buffer + child.Stderr = &stderr + if err := child.Start(); err != nil { + t.Fatal(err) + } + defer func() { + if child.ProcessState == nil { + _ = child.Process.Kill() + _ = child.Wait() + } + }() + if _, err := io.ReadFull(stdout, make([]byte, 1)); err != nil { + t.Fatal(err) + } + if err := stdout.Close(); err != nil { + t.Fatal(err) + } + err = child.Wait() + if ctx.Err() != nil || requests.count("/organization/spend_alerts") != 1 { + t.Fatalf("closed-pipe command timed out or repeated a source: context=%v stderr=%q", ctx.Err(), stderr.String()) + } + if !test.sourceFailure { + if err != nil || stderr.Len() != 0 { + t.Fatalf("lone output EPIPE changed success: error=%v stderr=%q", err, stderr.String()) + } + return + } + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + t.Fatalf("source failure disappeared after EPIPE: error=%v stderr=%q", err, stderr.String()) + } + if !json.Valid(stderr.Bytes()) || !strings.Contains(stderr.String(), "overview is incomplete") || strings.Contains(stderr.String(), "private synthetic") { + t.Fatalf("safe structured recovery missing: %q", stderr.String()) + } + }) } } diff --git a/pkg/custom/limits_commands.go b/pkg/custom/limits_commands.go index 4bc0da30..c3c42439 100644 --- a/pkg/custom/limits_commands.go +++ b/pkg/custom/limits_commands.go @@ -9,6 +9,7 @@ import ( "github.com/charmbracelet/x/term" "github.com/openai/openai-cli/internal/clihelp" localcommands "github.com/openai/openai-cli/internal/commands" + clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-cli/internal/output/transform" "github.com/urfave/cli/v3" ) @@ -107,8 +108,15 @@ func limitsCommandAction(project, show bool) cli.ActionFunc { return errors.Join(sourceErr, ctx.Err(), err) } } - outputErr := ShowJSON(value, opts) - return errors.Join(sourceErr, ctx.Err(), outputErr) + write := func() error { + outputErr := ShowJSON(value, opts) + return errors.Join(sourceErr, ctx.Err(), outputErr) + } + if file, ok := opts.Stdout.(*os.File); ok && file == os.Stdout { + // Join source failures before classifying a closed stdout pipe. + return clioutput.StreamToStdout(func(*os.File) error { return write() }) + } + return write() } } From 834d8bbde894cc322b1e4fae11ca3af73ec4e7e8 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 11:49:31 -0700 Subject: [PATCH 12/44] test: verify limits endpoint validation and precedence --- cmd/openai/limits_workflow_test.go | 74 ++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go index 7b490630..49d743d2 100644 --- a/cmd/openai/limits_workflow_test.go +++ b/cmd/openai/limits_workflow_test.go @@ -115,6 +115,80 @@ func limitsWorkflowSuccess(t *testing.T, project bool, r *http.Request) (int, st } } +func TestMainLimitsWorkflowBaseURLSecurity(t *testing.T) { + const remote = "http://fake-user:fake-password@remote.example/v1?token=fake-private" + for _, project := range []bool{false, true} { + t.Run(fmt.Sprintf("project=%t", project), func(t *testing.T) { + var proxyRequests atomic.Int32 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + proxyRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + io.WriteString(w, `{"error":{"message":"synthetic proxy rejects all destinations"}}`) + })) + t.Cleanup(proxy.Close) + home := t.TempDir() + env := adminCredentialsProxyEnv(proxy.URL, []string{"HOME=" + home, "XDG_CONFIG_HOME=" + home, + "OPENAI_API_KEY=sk-fake-spend-project", "OPENAI_ADMIN_KEY=sk-fake-spend-admin"}) + for _, source := range []string{"flag", "environment"} { + for _, format := range []string{"text", "json"} { + t.Run(source+"/"+format, func(t *testing.T) { + args := []string{"openai", "--format-error", format} + childEnv := append([]string{}, env...) + wantSource := "OPENAI_BASE_URL" + if source == "flag" { + args = append(args, "--base-url", remote) + wantSource = "--base-url" + } else { + childEnv = append(childEnv, "OPENAI_BASE_URL="+remote) + } + got := runMainDispatchWithEnv(t, "bash", childEnv, append(args, limitsWorkflowArgs(project, true)...)...) + if got.code != 1 || got.stdout != "" || proxyRequests.Load() != 0 { + t.Fatalf("remote HTTP reached the overview: code=%d requests=%d", got.code, proxyRequests.Load()) + } + message := got.stderr + if format == "json" { + var value map[string]string + if err := json.Unmarshal([]byte(got.stderr), &value); err != nil || len(value) != 1 { + t.Fatal("remote HTTP rejection did not preserve a structured diagnostic") + } + message = value["message"] + } + if !strings.Contains(message, wantSource) || !strings.Contains(message, "HTTPS") { + t.Fatal("remote HTTP rejection omitted its setting or recovery") + } + for _, private := range []string{"fake-", "remote.example", "\x1b"} { + if strings.Contains(got.stderr, private) { + t.Fatal("remote HTTP diagnostic exposed URL or credential contents") + } + } + }) + } + } + t.Run("loopback flag overrides remote environment", func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, project, r) + }) + childEnv := append(append([]string{}, env...), "OPENAI_BASE_URL="+remote) + args := append([]string{"openai", "--format", "json", "--base-url", server.URL}, limitsWorkflowArgs(project, true)...) + got := runMainDispatchWithEnv(t, "bash", childEnv, args...) + if got.code != 0 || got.stderr != "" || !json.Valid([]byte(got.stdout)) || proxyRequests.Load() != 0 { + t.Fatalf("loopback override failed: code=%d proxy_requests=%d", got.code, proxyRequests.Load()) + } + paths := []string{spendThresholdPath(project, "spend-limit"), spendThresholdPath(project, "spend-alerts"), "/organization/costs"} + if project { + paths = append(paths, "/organization/projects/proj_synthetic", spendThresholdPath(true, "rate-limits")) + } + for _, path := range paths { + if requests.count(path) != 1 { + t.Errorf("loopback overview requested %s %d times", path, requests.count(path)) + } + } + }) + }) + } +} + func TestMainLimitsWorkflowReadableAndFormats(t *testing.T) { for _, project := range []bool{false, true} { for _, mode := range []string{"bare", "show", "text", "JSON", "jsonl", "raw", "yaml", "pretty", "extraction", "raw-output"} { From d4a48495d9c5e077ca52ef25ce644df250caaade Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 12:08:11 -0700 Subject: [PATCH 13/44] fix: verify terminal restoration and clarify reported Costs --- pkg/custom/limits_workspace.go | 8 +++---- pkg/custom/limits_workspace_test.go | 22 +++++++++++++---- scripts/demos/limits-workspace/scene.py | 28 +++++++++++++++++++--- scripts/demos/limits-workspace/validate.py | 10 ++++++++ 4 files changed, 56 insertions(+), 12 deletions(-) diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index 4ea575d4..1dd2126d 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -771,11 +771,11 @@ func (m *limitsWorkspace) costsSummary() string { if m.snapshot.CostsTotal == "" { return "Current-month Costs unavailable. ?: details" } - status := "complete" + label := "Reported Costs" if !m.snapshot.CostsComplete { - status = "incomplete" + label += " (incomplete)" } - return "Costs: USD " + m.snapshot.CostsTotal + " (" + status + ")" + return label + ": USD " + m.snapshot.CostsTotal } func (m *limitsWorkspace) help() string { @@ -810,7 +810,7 @@ func (m *limitsWorkspace) help() string { text += "The current hard limit could not be confirmed.\nSet hard spend limit requires review and can create or replace a limit.\n\n" } if m.snapshot.CostsStart != 0 && m.snapshot.CostsEnd != 0 { - text += m.costsSummary() + "\nQueried period: " + time.Unix(m.snapshot.CostsStart, 0).UTC().Format(time.RFC3339) + " to " + time.Unix(m.snapshot.CostsEnd, 0).UTC().Format(time.RFC3339) + " (exclusive).\nCosts can lag and do not determine enforcement state.\n\n" + text += m.costsSummary() + "\nQueried period: " + time.Unix(m.snapshot.CostsStart, 0).UTC().Format(time.RFC3339) + " to " + time.Unix(m.snapshot.CostsEnd, 0).UTC().Format(time.RFC3339) + " (exclusive).\nToday's Costs remain partial.\nCosts can lag and do not determine enforcement state.\n\n" } text += "Soft budgets, independent enforcement settings, account tiers, credits, and organization rate catalogs need the website:\nhttps://platform.openai.com/settings/organization/limits\n\nThe public API exposes absolute-value alerts, not website percentage alerts or triggered status.\nRemoving a hard limit stops that cap; inspect soft budgets on the website.\n\nHistorical Costs command:\n" + invocation + " help admin usage costs\n\nThe workspace checks current state before saving. This check is not atomic.\nThe workspace sends each confirmed mutation once, without automatic retries.\nDirect API commands omit these safeguards and can retry.\n" return text diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index 101f05f3..08031f31 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -985,17 +985,29 @@ func TestLimitsWorkspaceCostsAndEnforcementStayDistinct(t *testing.T) { m.snapshot.CostsTotal, m.snapshot.CostsComplete = "123.000000000000000001", false m.snapshot.CostsStart, m.snapshot.CostsEnd = 1790812800, 1792368000 view := m.View().Content - if !strings.Contains(view, "Costs: USD 123.000000000000000001 (incomplete)") || !strings.Contains(view, "Returned enforcement: inactive") { + if !strings.Contains(view, "Reported Costs (incomplete): USD 123.000000000000000001") || !strings.Contains(view, "Returned enforcement: inactive") { t.Fatalf("workspace conflated Costs and returned enforcement: %q", view) } help := m.help() - if !strings.Contains(help, "Queried period:") || !strings.Contains(help, "Costs can lag and do not determine enforcement state") { + if !strings.Contains(help, "Queried period:") || !strings.Contains(help, "Today's Costs remain partial.") || !strings.Contains(help, "Costs can lag and do not determine enforcement state") { t.Fatal("Costs view omitted its coverage and enforcement limitations") } + m.snapshot.CostsComplete = true + view = m.View().Content + if !strings.Contains(view, "Reported Costs: USD 123.000000000000000001") || strings.Contains(view, "(complete)") || strings.Contains(view, "(incomplete)") { + t.Fatalf("complete pagination must not imply final Costs: %q", view) + } m.Update(tea.WindowSizeMsg{Width: 40, Height: 12}) - for _, line := range strings.Split(ansi.Strip(m.View().Content), "\n") { - if ansi.StringWidth(line) > 40 { - t.Fatalf("minimum-size overview exceeded its width: %q", line) + for _, complete := range []bool{false, true} { + m.snapshot.CostsComplete = complete + view = ansi.Strip(m.View().Content) + if !complete && !strings.Contains(view, "Reported Costs (incomplete)") { + t.Fatalf("narrow overview hid incomplete Costs: %q", view) + } + for _, line := range strings.Split(view, "\n") { + if ansi.StringWidth(line) > 40 { + t.Fatalf("minimum-size overview exceeded its width: %q", line) + } } } } diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index b9b30d01..ff526cd3 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -23,6 +23,13 @@ "$@" limits_scene_status=$? printf '%s\n' "$limits_scene_status" > "$DEMO_CLI_EXIT" || exit 74 +"$DEMO_TERMIOS_PYTHON" -I -B -c ' +import json, os, sys, termios +attributes = termios.tcgetattr(0) +attributes[6] = [value.hex() if isinstance(value, bytes) else value for value in attributes[6]] +with open(sys.argv[1], "x", encoding="utf-8") as destination: + json.dump({"parent_pid": os.getppid(), "attributes": attributes}, destination) +' "$DEMO_TERMIOS_SNAPSHOT" || exit 74 exit "$limits_scene_status" ''' @@ -34,6 +41,10 @@ def write_text(path, value): class RelayTerminal(harness.Terminal): + def __init__(self, *args, terminal_snapshot, **kwargs): + self.terminal_snapshot = terminal_snapshot + super().__init__(*args, **kwargs) + def read(self, duration=0.05): offset = len(self.raw) super().read(duration) @@ -55,7 +66,14 @@ def finish(self, expected): drain(self, 0.15) if self.child.returncode != expected: raise AssertionError(f"workspace exit={self.child.returncode}, expected={expected}; tail={self.text()[-900:]}") - restored, initial = termios.tcgetattr(self.slave), self.initial.copy() + # Bash snapshots the modes before exiting its controlling PTY session. + snapshot = json.loads(self.terminal_snapshot.read_text()) + if snapshot["parent_pid"] != self.child.pid: + raise AssertionError("terminal modes were not captured under the live Bash parent") + restored, initial = snapshot["attributes"].copy(), self.initial.copy() + initial[6] = [value.hex() if isinstance(value, bytes) else value for value in initial[6]] + self.terminal_modes = {"initial": initial.copy(), "restored": restored.copy(), + "snapshot_parent_pid": snapshot["parent_pid"], "bash_pid": self.child.pid} if sys.platform == "darwin": restored[3] &= ~termios.PENDIN initial[3] &= ~termios.PENDIN @@ -276,10 +294,12 @@ def main(): home = Path(temporary) / "home" home.mkdir(mode=0o700) exit_receipt = Path(temporary) / "cli-exit-status" + terminal_snapshot = Path(temporary) / "terminal-modes.json" env = {"PATH": temporary + ":/usr/bin:/bin", "HOME": str(home), "TERM": "xterm-256color", "LANG": "en_US.UTF-8", "GOMAXPROCS": "2", "PAGER": "cat", "OPENAI_API_KEY": "synthetic-demo-key", "OPENAI_ADMIN_KEY": "synthetic-demo-admin-key", "OPENAI_BASE_URL": os.environ["OPENAI_BASE_URL"], - "DEMO_CLI_EXIT": str(exit_receipt)} + "DEMO_CLI_EXIT": str(exit_receipt), "DEMO_TERMIOS_PYTHON": sys.executable, + "DEMO_TERMIOS_SNAPSHOT": str(terminal_snapshot)} if os.environ.get("DEMO_NO_COLOR", "1") == "1": env.update(NO_COLOR="1", FORCE_COLOR="0") terminal = None @@ -287,12 +307,14 @@ def main(): # Post-command status handling keeps Bash as the CLI's real parent. # Both processes inherit the existing PTY session and process group. shell_args = ["--noprofile", "--norc", "-c", BASH_WORKSPACE, "bash", str(staged), *args] - terminal = RelayTerminal("/bin/bash", shell_args, env, width=width, height=height, controlling_terminal=True) + terminal = RelayTerminal("/bin/bash", shell_args, env, width=width, height=height, + controlling_terminal=True, terminal_snapshot=terminal_snapshot) run_scene(terminal, scene, evidence, output) status_text = exit_receipt.read_text() if status_text != str(terminal.child.returncode) + "\n": raise AssertionError("Bash did not preserve the actual CLI exit status") evidence["cli_exit_status"] = int(status_text) + evidence["terminal_modes"] = terminal.terminal_modes evidence["staged_binary"]["sha256_after"] = hashlib.sha256(staged.read_bytes()).hexdigest() if (evidence["staged_binary"]["sha256_after"] != evidence["binary_sha256"] or hashlib.sha256(binary.read_bytes()).hexdigest() != evidence["binary_sha256"]): diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 9fa3aa14..61381ac7 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -5,6 +5,7 @@ import json from pathlib import Path import sys +import termios SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", @@ -22,6 +23,14 @@ def main(): assert evidence["exit_status"] == expected_status, (scene, "incorrect child exit") assert evidence["cli_exit_status"] == evidence["exit_status"], (scene, "Bash changed the CLI exit status") assert evidence["parent_shell"] == "/bin/bash", (scene, "missing native Bash parent") + modes = evidence["terminal_modes"] + assert modes["snapshot_parent_pid"] == modes["bash_pid"], (scene, "terminal snapshot lacks its live Bash parent") + initial, restored = modes["initial"].copy(), modes["restored"].copy() + assert len(initial) == len(restored) == 7, (scene, "incomplete terminal mode snapshot") + if sys.platform == "darwin": + initial[3] &= ~termios.PENDIN + restored[3] &= ~termios.PENDIN + assert initial == restored, (scene, "terminal input mode was not restored") staged = evidence["staged_binary"] staged_path = Path(staged["path"]) runtime = Path(evidence["recorder_runtime"]) @@ -80,6 +89,7 @@ def main(): print("PASS: nine real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") print("PASS: one mutation per confirmed action; no retry after uncertain outcome; slow load canceled without mutation.") print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; eight real command previews; staged files removed.") + print("PASS: all seven terminal mode fields restored before Bash exits; only Darwin PENDIN normalized.") print("LIMIT: inspect all checkpoint images and GIF frames before claiming visual acceptance.") From 5566e9f78ce1b54afd1186cdfe9ee5419e5d7dbf Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 12:27:58 -0700 Subject: [PATCH 14/44] fix(limits): print complete reviewed commands after terminal exit --- docs/limits-workspace.md | 10 +- pkg/custom/limits_draft.go | 17 +- pkg/custom/limits_workspace.go | 51 +++- pkg/custom/limits_workspace_test.go | 303 +++++++++++++++++++-- pkg/custom/limits_workspace_view.go | 3 + scripts/demos/limits-workspace/README.md | 8 +- scripts/demos/limits-workspace/record.sh | 2 +- scripts/demos/limits-workspace/scene.py | 61 ++++- scripts/demos/limits-workspace/server.py | 2 +- scripts/demos/limits-workspace/validate.py | 55 +++- 10 files changed, 446 insertions(+), 66 deletions(-) diff --git a/docs/limits-workspace.md b/docs/limits-workspace.md index 51d3e35d..a5a6b141 100644 --- a/docs/limits-workspace.md +++ b/docs/limits-workspace.md @@ -54,6 +54,7 @@ Values beyond that precision remain available in `costs.buckets`; the summary re | d | Delete the selected alert | | Escape | Return without saving | | q | Close the overview | +| Ctrl+P | Print the reviewed command and exit without saving | | Ctrl+C | Interrupt from any state | The workspace needs at least 40 columns and 12 rows. @@ -64,7 +65,11 @@ Every change opens a before-and-after review. Scroll to the end before selecting Apply. Back is the initial selection, including after resizing. The review shows a shell command when the CLI can reproduce the request context safely. -It omits command previews that require private options. +Wrapped command rows are for inspection only. +Press Ctrl+P to print the complete command after the terminal screen restores. +Copy the printed command when you want to run it later. +Printing sends no change and preserves any earlier confirmed receipt. +The workspace omits command previews that require private options or unsupported shells. Existing administration commands remain available for scripts. ```sh @@ -82,7 +87,8 @@ Enter decimal dollars, such as `125.50`. The CLI converts dollars to integer cents without floating-point arithmetic. A hard limit must be at least USD 0.01. Enforcement can be delayed after the threshold is reached. -Removing a hard limit removes enforcement. +Removing a hard limit removes that limit’s enforcement. +Parent limits can still apply. Use the Platform website to inspect or edit any retained soft budget. Alerts support creation, editing, and deletion. diff --git a/pkg/custom/limits_draft.go b/pkg/custom/limits_draft.go index 7972a983..ebafdcc8 100644 --- a/pkg/custom/limits_draft.go +++ b/pkg/custom/limits_draft.go @@ -216,8 +216,17 @@ func limitsReview(c limitsChange, projectID string, requestArgs []string, omit b result += "\nThe CLI checks the current record before saving. This check is not atomic.\n" result += "The workspace sends this change once, without automatic retries.\n" result = readable.Text(result) + command, reason := limitsCommand(c, projectID, requestArgs, omit, shell, executable...) + if command == "" { + return result + "\n" + reason + "\n" + } + return result + "\nCommand preview (inspection only)\n" + command + "\n\nCtrl+P prints the complete command and exits.\nThis command skips the workspace preflight and can retry automatically.\nRecheck current settings before running it.\n" +} + +// Keep executable bytes separate from prose and viewport wrapping. +func limitsCommand(c limitsChange, projectID string, requestArgs []string, omit bool, shell string, executable ...string) (string, string) { if omit { - return result + "\nCommand preview omitted: request context cannot be reproduced safely.\n" + return "", "Command preview omitted: request context cannot be reproduced safely." } args := append([]string{}, requestArgs...) args = append(args, "admin", "organization") @@ -256,7 +265,7 @@ func limitsReview(c limitsChange, projectID string, requestArgs []string, omit b if c.SubjectPrefixSet && c.SubjectPrefix != nil { prefix := *c.SubjectPrefix if strings.HasPrefix(prefix, "\\@") { - return result + "\nCommand preview omitted: the request parser cannot preserve this literal prefix.\n" + return "", "Command preview omitted: the request parser cannot preserve this literal prefix." } if strings.HasPrefix(prefix, "@") { channelValues["subject_prefix"] = "\\" + prefix @@ -272,7 +281,7 @@ func limitsReview(c limitsChange, projectID string, requestArgs []string, omit b } command := terminal.FormatCommand(args, shell) if command == "" { - return result + "\nCommand printing needs Bash, zsh, fish or PowerShell 7.\n" + return "", "Command printing needs Bash, zsh, fish or PowerShell 7." } if len(executable) > 0 && executable[0] != "" { command = terminal.ShellQuoter(shell)(executable[0]) + strings.TrimPrefix(command, "openai") @@ -280,7 +289,7 @@ func limitsReview(c limitsChange, projectID string, requestArgs []string, omit b command = "& " + command } } - return result + "\nDirect API command\n" + command + "\n\nThis command skips the workspace preflight and can retry automatically.\nRecheck current settings before running it.\n" + return command, "" } // Use the generated operation identity so successful receipts retain projections. diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index 1dd2126d..83800cd1 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -38,6 +38,7 @@ type limitsMenuItem struct { type limitsWorkspace struct { detailsHeading string previewExecutable string + previewCommand, commandToPrint string invocation string lastSavedOperation, workerSavedOperation string mutationError error @@ -256,15 +257,27 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error opts.Title = "Last confirmed result" receiptErr = ShowJSON(m.lastSaved, opts) } + var overviewErr error if incomplete { - return errors.Join(&limitsWorkflowFailure{message: message, cause: operationErr}, receiptErr) + overviewErr = &limitsWorkflowFailure{message: message, cause: operationErr} + if m.commandToPrint == "" { + return errors.Join(overviewErr, receiptErr) + } } if receiptErr != nil { - return receiptErr + return errors.Join(overviewErr, receiptErr) } if removedLimitErr != nil && resolvedOutputFormat(opts) == "text" { - _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString("Removal confirmed. Current hard-limit configuration was not returned (HTTP 404).\n") - return err + if _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString("Removal confirmed. Current hard-limit configuration was not returned (HTTP 404).\n"); err != nil { + return errors.Join(overviewErr, err) + } + } + if m.commandToPrint != "" { + _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString(m.commandToPrint + "\n") + if err != nil { + err = &limitsWorkflowFailure{message: "Could not print the limits command. Check terminal output and try again.", cause: err} + } + return errors.Join(overviewErr, err) } if m.lastSaved.Exists() { return nil @@ -461,13 +474,25 @@ func (m *limitsWorkspace) review() { return } m.change, m.page, m.apply, m.formError = change, "review", false, "" - m.setDetails(limitsReview(change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable)) + m.setReview() +} + +func (m *limitsWorkspace) setReview() { + m.previewCommand, _ = limitsCommand(m.change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable) + m.setDetails(limitsReview(m.change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable)) +} + +func (m *limitsWorkspace) detailsHeight() int { + if m.page == "review" && m.previewCommand != "" { + return max(1, m.height-6) + } + return max(1, m.height-5) } func (m *limitsWorkspace) setDetails(content string) { m.details = content m.viewport.SetWidth(max(1, m.width-2)) - m.viewport.SetHeight(max(1, m.height-5)) + m.viewport.SetHeight(m.detailsHeight()) m.viewport.SetContent(ansi.Hardwrap(content, max(1, m.width-2), true)) m.viewport.GotoTop() } @@ -484,13 +509,13 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { m.width, m.height = msg.Width, msg.Height m.apply = false if m.page == "review" { - m.setDetails(limitsReview(m.change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable)) + m.setReview() } if m.page == "details" { m.setDetails(m.details) } m.viewport.SetWidth(max(1, msg.Width-2)) - m.viewport.SetHeight(max(1, msg.Height-5)) + m.viewport.SetHeight(m.detailsHeight()) return m, nil case limitsWorkResult: m.busy = false @@ -612,6 +637,14 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { return m, nil } case "review": + if key == "ctrl+p" { + if m.previewCommand != "" && !m.locked && !m.refreshRequired && m.ctx.Err() == nil && (m.output == nil || m.output.Err() == nil) { + m.commandToPrint, m.quitting = m.previewCommand, true + m.cancel() + return m, tea.Quit + } + return m, nil + } if key == "tab" || key == "shift+tab" { if m.viewport.AtBottom() { m.apply = !m.apply @@ -783,7 +816,7 @@ func (m *limitsWorkspace) help() string { if invocation == "" { invocation = "openai" } - text := "Spend manages hard monthly USD limits.\nAlerts notify recipients; they do not enforce caps.\nRates manages six project rate fields.\n\nEvery change requires a before/after review.\nRead to the end, then Tab to Apply.\nBack remains the default action.\n\nUse r to refresh. Use i to inspect original fields.\nUse --max-items -1 to load complete lists.\n\n" + text := "Spend manages hard monthly USD limits.\nAlerts notify recipients; they do not enforce caps.\nRates manages six project rate fields.\n\nEvery change requires a before/after review.\nRead to the end, then Tab to Apply.\nBack remains the default action.\nCtrl+P prints the reviewed command and exits. Printing sends no change.\n\nUse r to refresh. Use i to inspect original fields.\nUse --max-items -1 to load complete lists.\n\n" if m.locked { m.mu.Lock() failure := m.mutationError diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index 08031f31..5db113a2 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -218,13 +218,267 @@ func TestLimitsWorkspaceBusyAndFailedSavePreventRepeatedWrites(t *testing.T) { } } -func TestLimitsDraftCommandPreviewNativeShellPreservesArguments(t *testing.T) { +func limitsControllerPrintedCommand(t *testing.T, m *limitsWorkspace) string { + t.Helper() + m.viewport.GotoBottom() + view := ansi.Strip(m.View().Content) + for _, visible := range []string{"Ctrl+P: print command and exit", "> Back", "Apply change"} { + if !strings.Contains(view, visible) { + t.Fatalf("review clipped %q at %d columns: %q", visible, m.width, view) + } + } + for _, line := range strings.Split(view, "\n") { + if ansi.StringWidth(line) > m.width { + t.Fatalf("review exceeds %d columns: %q", m.width, line) + } + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit == nil || !m.quitting || m.interrupted || m.commandToPrint == "" { + t.Fatal("Ctrl+P did not freeze the reviewed command and quit normally") + } + var out bytes.Buffer + if err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard}); err != nil { + t.Fatal(err) + } + if out.String() != m.commandToPrint+"\n" { + t.Fatalf("finalization changed the command or added prose: %q", out.String()) + } + return out.String() +} + +func TestLimitsWorkspacePrintFreezesReviewedCommand(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + limitsControllerReview(t, m) + if !strings.Contains(m.details, "Command preview (inspection only)") { + t.Fatal("wrapped review did not identify the command as inspection-only") + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit == nil || m.commandToPrint == "" { + t.Fatal("review did not offer command printing") + } + frozen := m.commandToPrint + m.change.Amount, m.previewCommand = 99999, "changed after exit" + m.Update(tea.WindowSizeMsg{Width: 80, Height: 24}) + var out bytes.Buffer + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Stdout: &out, Stderr: io.Discard}) + if err != nil || out.String() != frozen+"\n" || backend.writes.Load() != 0 { + t.Fatalf("print changed reviewed bytes or sent a mutation: output=%q err=%v", out.String(), err) + } +} + +func TestLimitsWorkspacePrintOmissionsAndUnavailableStates(t *testing.T) { + for _, scenario := range []string{"private context", "unsupported shell", "literal prefix", "go run", "locked", "refresh required", "busy", "small terminal", "canceled", "output failure"} { + t.Run(scenario, func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.change, m.page = limitsChange{Kind: "limit_set", Amount: 12525}, "review" + omitted := true + switch scenario { + case "private context": + m.omitPreview = true + case "unsupported shell": + m.shell = "unknown-shell" + case "literal prefix": + prefix := "\\@literal" + m.change = limitsChange{Kind: "alert_create", SubjectPrefixSet: true, SubjectPrefix: &prefix} + case "go run": + m.invocation, m.omitPreview = "go run ./cmd/openai", true + default: + omitted = false + } + m.setReview() + switch scenario { + case "locked": + m.locked = true + case "refresh required": + m.refreshRequired = true + case "busy": + m.busy = true + case "small terminal": + m.Update(tea.WindowSizeMsg{Width: 39, Height: 12}) + case "canceled": + m.cancel() + case "output failure": + m.output = &limitsWorkspaceOutput{err: errors.New("synthetic sink failure")} + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit != nil || m.commandToPrint != "" || m.quitting || backend.writes.Load() != 0 { + t.Fatal("Ctrl+P exported an omitted command or bypassed an unavailable state") + } + if omitted && (m.previewCommand != "" || strings.Contains(m.View().Content, "Ctrl+P:")) { + t.Fatal("an omitted command retained its print action") + } + }) + } +} + +func TestLimitsWorkspacePrintPreservesFinalizationFailures(t *testing.T) { + for _, scenario := range []string{"caller cancellation", "interrupt", "program failure", "mutation failure", "terminal failure", "print failure", "short print"} { + t.Run(scenario, func(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + limitsControllerReview(t, m) + m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + ctx := t.Context() + want := errors.New("synthetic independent failure") + var out bytes.Buffer + opts := ShowJSONOpts{Context: ctx, Format: "text", Stdout: &out, Stderr: io.Discard} + var runErr error + switch scenario { + case "caller cancellation": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + want = context.Canceled + case "interrupt": + m.interrupted = true + want = context.Canceled + case "program failure": + runErr = want + case "mutation failure": + m.locked, m.mutationError = true, want + case "terminal failure": + m.output = &limitsWorkspaceOutput{err: want} + case "print failure": + opts.Stdout = limitsFailingWriter{err: want} + case "short print": + want = io.ErrShortWrite + opts.Stdout = limitsWorkspaceWriteFunc(func(data []byte) (int, error) { return 1, nil }) + } + err := finishLimitsWorkspace(ctx, m, runErr, opts) + if !errors.Is(err, want) || out.Len() != 0 { + t.Fatalf("print bypassed an independent failure: output=%q err=%v", out.String(), err) + } + if scenario == "interrupt" { + var interrupt *limitsInterrupt + if !errors.As(err, &interrupt) || interrupt.ExitCode() != 130 { + t.Fatal("print changed the interrupt exit status") + } + } + if scenario == "print failure" || scenario == "short print" { + if !strings.Contains(err.Error(), "Could not print the limits command") { + t.Fatalf("print failure lacks useful recovery: %v", err) + } + } + }) + } +} + +func TestLimitsWorkspacePrintPreservesSourceFailures(t *testing.T) { + for _, source := range []string{"costs", "first-cap-404", "costs-with-receipt"} { + for _, printFails := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/print-fails-%t", source, printFails), func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + var sourceErr error = errors.New("synthetic Costs failure") + resource := "costs" + if source == "first-cap-404" { + sourceErr = &openai.Error{StatusCode: http.StatusNotFound} + resource = "spend_limit" + m.snapshot.LimitAbsent, m.snapshot.LimitUnconfirmed = false, true + } + m.snapshot.Errors = []limitsSourceFailure{{Resource: resource, Err: sourceErr}} + limitsControllerReview(t, m) + if source == "first-cap-404" && (!m.change.BeforeMissing || !strings.Contains(m.details, "creates or replaces")) { + t.Fatal("first-cap review lost its explicit uncertainty warning") + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit == nil || m.commandToPrint == "" { + t.Fatal("a nonfatal source failure removed the print action") + } + if source == "costs-with-receipt" { + m.lastSaved = gjson.Parse("{\"object\":\"organization.spend_limit\",\"threshold_amount\":10001,\"currency\":\"USD\",\"interval\":\"month\"}") + m.lastSavedOperation = limitsChangeOperation("limit_set", false) + } + var out bytes.Buffer + opts := ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard} + if printFails { + opts.Stdout = limitsFailingWriter{err: syscall.EPIPE} + } + err := finishLimitsWorkspace(t.Context(), m, nil, opts) + if !errors.Is(err, sourceErr) || backend.writes.Load() != 0 { + t.Fatalf("printing hid the source failure or sent a mutation: %v", err) + } + if printFails { + if !errors.Is(err, syscall.EPIPE) || clioutput.IsBrokenPipe(err) || out.Len() != 0 { + t.Fatalf("print failure masked the independent source failure: %v", err) + } + return + } + if source == "costs-with-receipt" { + if !strings.HasSuffix(out.String(), m.commandToPrint+"\n") || !strings.Contains(strings.TrimSuffix(out.String(), m.commandToPrint+"\n"), "USD 100.01") { + t.Fatalf("source failure lost the confirmed receipt or complete command: %q", out.String()) + } + } else if out.String() != m.commandToPrint+"\n" { + t.Fatalf("source failure suppressed or changed the command: %q", out.String()) + } + }) + } + } +} + +func TestLimitsWorkspacePrintPreservesConfirmedReceipt(t *testing.T) { + for _, scenario := range []string{"confirmed result", "receipt failure", "confirmed removal"} { + t.Run(scenario, func(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + limitsControllerReview(t, m) + m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + m.lastSaved = gjson.Parse("{\"object\":\"organization.spend_limit\",\"threshold_amount\":10001,\"currency\":\"USD\",\"interval\":\"month\"}") + m.lastSavedOperation = limitsChangeOperation("limit_set", false) + if scenario == "confirmed removal" { + m.lastSaved = gjson.Parse("{\"object\":\"organization.spend_limit.deleted\",\"deleted\":true}") + m.lastSavedOperation = limitsChangeOperation("limit_delete", false) + m.snapshot.LimitUnconfirmed = true + m.snapshot.Errors = []limitsSourceFailure{{Resource: "spend_limit", Err: &openai.Error{StatusCode: http.StatusNotFound}}} + } + var out bytes.Buffer + opts := ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard} + if scenario == "receipt failure" { + opts.Stdout = limitsFailingWriter{err: syscall.EPIPE} + } + err := finishLimitsWorkspace(t.Context(), m, nil, opts) + if scenario == "receipt failure" { + if !errors.Is(err, syscall.EPIPE) || out.Len() != 0 { + t.Fatalf("print ignored a receipt failure: %v", err) + } + return + } + if err != nil || !strings.HasSuffix(out.String(), m.commandToPrint+"\n") { + t.Fatalf("confirmed receipt suppressed the command: %q err=%v", out.String(), err) + } + want := "USD 100.01" + if scenario == "confirmed removal" { + want = "Removal confirmed. Current hard-limit configuration was not returned (HTTP 404)." + } + if !strings.Contains(strings.TrimSuffix(out.String(), m.commandToPrint+"\n"), want) { + t.Fatalf("printing lost the preceding confirmed result: %q", out.String()) + } + }) + } +} + +// This checks emitted PowerShell bytes; native PowerShell execution is separate. +func TestLimitsWorkspacePrintedPowerShellCommandRetainsCallOperator(t *testing.T) { + for _, width := range []int{40, 80} { + t.Run(fmt.Sprintf("columns-%d", width), func(t *testing.T) { + m, backend := newLimitsController(t, width, 12) + m.change, m.page, m.shell = limitsChange{Kind: "limit_set", Amount: 12525}, "review", "pwsh" + m.previewExecutable = "C:\\Program Files\\" + strings.Repeat("long ", 20) + "Owner's Tools\\openai.exe" + m.requestArgs = []string{"--organization=org owner's"} + m.setReview() + command := limitsControllerPrintedCommand(t, m) + want := "& '" + strings.ReplaceAll(m.previewExecutable, "'", "''") + "' '--organization=org owner''s' admin organization spend-limit update --currency USD --interval month --threshold-amount 12525\n" + if command != want || backend.writes.Load() != 0 { + t.Fatalf("print changed PowerShell syntax or sent a mutation: %q", command) + } + }) + } +} + +func TestLimitsWorkspacePrintedCommandNativeShellPreservesArguments(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Bash/Zsh preview execution uses a POSIX recorder; native Windows requires separate coverage") } for _, shell := range []string{"bash", "zsh"} { t.Run(shell, func(t *testing.T) { - binary, directory := audioPreviewExecutable(t, "openai limits' recorder") + binary, directory := audioPreviewExecutable(t, strings.Repeat("long ", 20)+"limits'\nrecorder") marker := filepath.Join(t.TempDir(), "MUST_NOT_EXIST") subject := "Owner's \"alert\"; $(touch " + marker + "); `touch " + marker + "` $HOME \\path\nnext line" channel, err := json.Marshal(map[string]any{"type": "email", "recipients": []string{"finance@example.test"}, "subject_prefix": subject}) @@ -257,25 +511,28 @@ func TestLimitsDraftCommandPreviewNativeShellPreservesArguments(t *testing.T) { } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { - review := limitsReview(tc.change, tc.projectID, requestArgs, false, shell, binary) - _, tail, found := strings.Cut(review, "\nDirect API command\n") - if !found { - t.Fatalf("review omitted its copyable command: %q", review) - } - command, guidance, found := strings.Cut(tail, "\n\n") - if !found || command == "" || !strings.HasPrefix(guidance, "This command skips the workspace preflight and can retry automatically.") { - t.Fatalf("review did not separate the executable command from its guidance: %q", tail) - } - capture := executeAudioPreview(t, shell, command, directory) - want := append(append([]string{}, requestArgs...), tc.want...) - if !reflect.DeepEqual(capture.args, want) { - t.Fatalf("printed command changed arguments:\n got: %q\nwant: %q", capture.args, want) - } - if len(capture.stdin) != 0 || capture.protected != "unset" { - t.Fatal("printed command unexpectedly supplied stdin or changed its environment") - } - if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) { - t.Fatalf("printed command executed subject text: %v", err) + for _, width := range []int{40, 80} { + t.Run(fmt.Sprintf("columns-%d", width), func(t *testing.T) { + m, backend := newLimitsController(t, width, 12) + m.change, m.page, m.shell = tc.change, "review", shell + m.snapshot.ProjectID, m.requestArgs, m.previewExecutable = tc.projectID, requestArgs, binary + m.setReview() + command := limitsControllerPrintedCommand(t, m) + if backend.writes.Load() != 0 { + t.Fatal("printing a command wrote to the backend") + } + capture := executeAudioPreview(t, shell, command, directory) + want := append(append([]string{}, requestArgs...), tc.want...) + if !reflect.DeepEqual(capture.args, want) { + t.Fatalf("printed command changed arguments:\n got: %q\nwant: %q", capture.args, want) + } + if len(capture.stdin) != 0 || capture.protected != "unset" { + t.Fatal("printed command unexpectedly supplied stdin or changed its environment") + } + if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("printed command executed subject text: %v", err) + } + }) } }) } @@ -566,7 +823,7 @@ func TestLimitsDraftPreviewNamesDirectCommandDifferences(t *testing.T) { for _, shell := range []string{"bash", "zsh", "fish", "pwsh"} { t.Run(shell, func(t *testing.T) { preview := limitsReview(change, "", nil, false, shell, "/synthetic path/openai") - for _, want := range []string{"Direct API command", "skips the workspace preflight", "can retry automatically", "creates or replaces"} { + for _, want := range []string{"Command preview (inspection only)", "Ctrl+P prints the complete command", "skips the workspace preflight", "can retry automatically", "creates or replaces"} { if !strings.Contains(preview, want) { t.Errorf("preview omitted %q", want) } @@ -574,7 +831,7 @@ func TestLimitsDraftPreviewNamesDirectCommandDifferences(t *testing.T) { if strings.Contains(preview, "Equivalent command") || strings.Contains(preview, "Not configured") { t.Fatal("preview claimed unsupported equivalence or confirmed absence") } - command := strings.SplitN(strings.SplitN(preview, "Direct API command\n", 2)[1], "\n", 2)[0] + command, _ := limitsCommand(change, "", nil, false, shell, "/synthetic path/openai") if shell == "pwsh" && !strings.HasPrefix(command, "& '/synthetic path/openai' ") { t.Fatalf("PowerShell preview cannot invoke the quoted executable: %q", command) } diff --git a/pkg/custom/limits_workspace_view.go b/pkg/custom/limits_workspace_view.go index 92914da7..0d8900fb 100644 --- a/pkg/custom/limits_workspace_view.go +++ b/pkg/custom/limits_workspace_view.go @@ -112,6 +112,9 @@ func (m *limitsWorkspace) View() tea.View { } body = append(body, line(back+" "+apply)) footer = "↑/↓: scroll Tab: action Enter: select\nEsc: back Ctrl+C: quit" + if m.previewCommand != "" { + footer = "↑/↓: scroll Tab: action Enter: select\nCtrl+P: print command and exit\nEsc: back Ctrl+C: quit" + } } else { footer = "↑/↓: scroll Esc: back Ctrl+C: quit" } diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index 6b127582..58fe4947 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -37,6 +37,7 @@ The isolated child environment includes only fake credentials and the fixture en | `org-limit-delete` | Organization | Remove the hard limit and retain its confirmed receipt after a missing-record refresh | | `project-limit` | Project | Replace the hard limit with USD 125.25 per month | | `alert-create` | Organization | Create a USD 75.50 monthly alert with synthetic email settings | +| `alert-print` | Organization | Print the complete reviewed alert command after terminal restoration; perform no mutation | | `alert-edit` | Project | Change the alert to USD 150.25 and clear its prefix to null | | `alert-delete` | Organization | Delete the selected synthetic alert | | `rate-update` | Project | Change requests per minute to 250; preserve all other dimensions | @@ -48,7 +49,10 @@ The driver reads to the review's end before selecting Apply. It checks that the fixture receives no mutation before confirmation. The uncertain scene checks one mutation, retained failure guidance, and exit status 1. The cancellation scene waits for the fixture's request marker and requires exit status 130. -Successful scenes require exit status 0 and one mutation each. +Successful mutation scenes require exit status 0 and one mutation each. +The print scene uses Ctrl+P and requires exit status 0 with no mutation. +It compares the complete printed command bytes after terminal restoration, including the final newline. +Wrapped inspection rows do not establish command copyability. The fixture stores only synthetic request bodies, response hashes, and method/path/status records. It never logs request headers or credentials. @@ -66,6 +70,8 @@ Planned screenshot evidence comes from recorded checkpoints: - Form: entered values, retained optional-field semantics, and visible Review action. - Review top: selected resource and original state. - Review Apply: explicit final action after scrolling through the complete review. +- Review Print: the Ctrl+P action and the inspection-only command label. +- Printed command: the complete command after terminal restoration, with no mutation. - Saved state: refreshed resource and confirmed outcome. - Uncertain state: retained draft and blocked retry guidance. - Loading state: cancellation remains available while a read is pending. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh index 6ac48eaf..9c5c2395 100755 --- a/scripts/demos/limits-workspace/record.sh +++ b/scripts/demos/limits-workspace/record.sh @@ -50,7 +50,7 @@ shasum -a 256 "$record_source/record.sh" "$record_source/server.py" "$record_sou demo_window_size="${record_width}x$((record_height+4))" demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ --theme asciinema --fps-cap 15 --last-frame-duration 2) -record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel) +record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel alert-print) for record_scene in "${record_scenes[@]}"; do demo_capture_scene "$record_scene" 0 "$demo_runtime/after" "$demo_api_url/$record_scene/v1" \ "$record_scene: ${record_width}x${record_height}" \ diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index ff526cd3..a0c7a187 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -126,8 +126,7 @@ def mutations(scene): def checkpoint(terminal, evidence, output, name, required=()): screen = screen_text(terminal) - compact = "".join(line.rstrip() for line in screen.splitlines()) - if not all(text in compact for text in required): + if not all(text in screen for text in required): raise AssertionError(f"checkpoint {name} omitted its required preview: {required!r}") write_text(Path(str(output) + f".{name}.txt"), screen) evidence["checkpoints"].append({"name": name, "seconds": time.monotonic() - terminal.started}) @@ -152,16 +151,15 @@ def review(terminal, evidence, scene, output): preview_seen = False for _ in range(40): screen = screen_text(terminal) - compact = "".join(line.rstrip() for line in screen.splitlines()) - if "Command printing needs" in compact or "Command preview omitted" in compact: + if "Command printing needs" in screen or "Command preview omitted" in screen: raise AssertionError("review omitted the real command preview") - if not preview_seen and "Direct API command" in screen and evidence["staged_binary"]["path"] in compact: + if not preview_seen and "Command preview (inspection only)" in screen: checkpoint(terminal, evidence, output, "review-command", - ("Direct API command", evidence["staged_binary"]["path"])) + ("Command preview (inspection only)",)) preview_seen = True if "Apply change" in screen and "Read to the end" not in screen: break - if not preview_seen and "Direct API command" in screen: + if not preview_seen and "Command preview" in screen: key(terminal, evidence, "Show command preview", b"\x1b[B") else: key(terminal, evidence, "PageDown", b"\x1b[6~") @@ -171,6 +169,11 @@ def review(terminal, evidence, scene, output): raise AssertionError("review reached Apply without a command preview checkpoint") if mutations(scene): raise AssertionError("fixture received a mutation before explicit confirmation") + if scene == "alert-print": + wait_screen(terminal, ["Ctrl+P"]) + checkpoint(terminal, evidence, output, "review-print", ("Ctrl+P",)) + key(terminal, evidence, "Ctrl+P print and exit", b"\x10") + return key(terminal, evidence, "Tab to Apply", b"\t") wait_screen(terminal, ["> Apply change"]) checkpoint(terminal, evidence, output, "review-apply") @@ -198,20 +201,22 @@ def run_scene(terminal, scene, evidence, output): if scene.startswith("alert-"): key(terminal, evidence, "Alerts", b"\x1b[C") wait_screen(terminal, ["Create spend alert"]) - if scene != "alert-create": + creating = scene in {"alert-create", "alert-print"} + if not creating: key(terminal, evidence, "Select alert", b"\x1b[B") if scene == "alert-delete": key(terminal, evidence, "Delete selected alert", b"d") else: key(terminal, evidence, "Edit alert", b"\r") wait_screen(terminal, ["Monthly USD", "Review change"]) - replace(terminal, evidence, "Monthly USD", "75.50" if scene == "alert-create" else "150.25") + replace(terminal, evidence, "Monthly USD", "75.50" if creating else "150.25") key(terminal, evidence, "Recipients", b"\t") - if scene == "alert-create": + if creating: replace(terminal, evidence, "Synthetic recipient", "finance@example.test") key(terminal, evidence, "Subject prefix", b"\t") - if scene == "alert-create": - replace(terminal, evidence, "Subject prefix", "Demo") + if creating: + prefix = "Demo team's budget $HOME; keep 'quoted' text" if scene == "alert-print" else "Demo" + replace(terminal, evidence, "Subject prefix", prefix) key(terminal, evidence, "Prefix mode", b"\t") if scene == "alert-edit": key(terminal, evidence, "Replace prefix mode", b"\r") @@ -242,7 +247,34 @@ def run_scene(terminal, scene, evidence, output): checkpoint(terminal, evidence, output, "form") key(terminal, evidence, "Review", b"\r") review(terminal, evidence, scene, output) - if scene == "uncertain": + if scene == "alert-print": + terminal.finish(0) + scene_requests = requests(scene) + if mutations(scene) or not scene_requests or any(item["method"] != "GET" for item in scene_requests): + raise AssertionError("printing a reviewed command must perform no mutation") + channel = json.dumps({"recipients": ["finance@example.test"], + "subject_prefix": "Demo team's budget $HOME; keep 'quoted' text", + "type": "email"}, sort_keys=True, separators=(",", ":")) + command = (evidence["staged_binary"]["path"] + " '--base-url=" + evidence["request_base_url"] + + "' admin organization spend-alerts create" + " --currency USD --interval month --threshold-amount 7550 --notification-channel '" + + channel.replace("'", "'\\''") + "'") + output_flags = terminal.terminal_modes["initial"][1] + newline = "\r\n" if output_flags & termios.OPOST and output_flags & termios.ONLCR else "\n" + expected = (command + newline).encode() + restored = terminal.raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + start = len(terminal.raw) - len(expected) + if (start < restored or terminal.raw[start:] != expected + or terminal.raw[restored:].count(expected) != 1 + or terminal.raw.rfind(b"\x1b[?25h") > start): + raise AssertionError("printed command bytes differ or precede terminal restoration") + with Path(str(output) + ".printed-command.raw.txt").open("xb") as destination: + if destination.write(terminal.raw[start:]) != len(expected): + raise OSError("incomplete printed command evidence write") + evidence["printed_command"] = {"byte_offset": start, "byte_length": len(expected), + "sha256": hashlib.sha256(expected).hexdigest()} + checkpoint(terminal, evidence, output, "printed-command") + elif scene == "uncertain": wait_screen(terminal, ["Save not confirmed", "Review change"]) checkpoint(terminal, evidence, output, "uncertain") key(terminal, evidence, "Blocked repeat Enter", b"\r\r") @@ -277,7 +309,8 @@ def main(): args = ["projects", "limits", "--project-id", "proj_demo"] if project else ["admin", "limits"] evidence = {"scene": scene, "width": width, "height": height, "command": ["openai", *args], "keys": [], "checkpoints": [], "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), - "parent_shell": "/bin/bash", "recorder_runtime": str(runtime)} + "parent_shell": "/bin/bash", "recorder_runtime": str(runtime), + "request_base_url": os.environ["OPENAI_BASE_URL"]} print("$ openai " + " ".join(args), flush=True) original = termios.tcgetattr(1) relay = termios.tcgetattr(1) diff --git a/scripts/demos/limits-workspace/server.py b/scripts/demos/limits-workspace/server.py index f4dceaaf..33d80ac2 100755 --- a/scripts/demos/limits-workspace/server.py +++ b/scripts/demos/limits-workspace/server.py @@ -15,7 +15,7 @@ SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", - "alert-delete", "rate-update", "uncertain", "slow-cancel") + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print") def write_text(path, value): diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 61381ac7..d181938d 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -6,10 +6,11 @@ from pathlib import Path import sys import termios +from urllib.parse import urlsplit SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", - "alert-delete", "rate-update", "uncertain", "slow-cancel") + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print") def main(): @@ -53,10 +54,48 @@ def main(): assert relay[:-len(prompt)] == raw == child_bytes, (scene, "capture changed CLI bytes") assert hashlib.sha256(raw).hexdigest() == evidence["relayed_sha256"] mutations = [entry for entry in requests if entry["scene"] == scene and entry["method"] in {"POST", "DELETE"}] - assert len(mutations) == (0 if scene == "slow-cancel" else 1), (scene, "unexpected mutation count") + assert len(mutations) == (0 if scene in {"slow-cancel", "alert-print"} else 1), (scene, "unexpected mutation count") if scene == "slow-cancel": assert any(entry["scene"] == scene and entry["status"] == "held" for entry in requests) continue + checkpoints = {point["name"]: point for point in evidence["checkpoints"]} + assert "review-command" in checkpoints + preview = (root / (scene + "-evidence.review-command.txt")).read_text() + assert "Command preview (inspection only)" in preview, (scene, "missing inspection preview") + assert "Command printing needs" not in preview and "Command preview omitted" not in preview + final_screen = (root / (scene + "-evidence.exit.txt")).read_text() + assert "> Apply change" not in final_screen, (scene, "stale confirmation after exit") + if scene == "alert-print": + scene_requests = [entry for entry in requests if entry["scene"] == scene] + assert scene_requests and all(entry["method"] == "GET" for entry in scene_requests) + assert "review-print" in checkpoints and "printed-command" in checkpoints + assert "review-apply" not in checkpoints + assert checkpoints["review-command"]["seconds"] < checkpoints["review-print"]["seconds"] + keys = [event["key"] for event in evidence["keys"]] + assert keys.count("Ctrl+P print and exit") == 1 and "Apply" not in keys + endpoint = urlsplit(evidence["request_base_url"]) + assert endpoint.scheme == "http" and endpoint.hostname == "127.0.0.1" and endpoint.port + assert evidence["request_base_url"] == f"http://127.0.0.1:{endpoint.port}/alert-print/v1" + channel = json.dumps({"recipients": ["finance@example.test"], + "subject_prefix": "Demo team's budget $HOME; keep 'quoted' text", + "type": "email"}, sort_keys=True, separators=(",", ":")) + command = (staged["path"] + " '--base-url=" + evidence["request_base_url"] + + "' admin organization spend-alerts create" + " --currency USD --interval month --threshold-amount 7550 --notification-channel '" + + channel.replace("'", "'\\''") + "'") + newline = "\r\n" if initial[1] & termios.OPOST and initial[1] & termios.ONLCR else "\n" + expected = (command + newline).encode() + printed = (root / (scene + "-evidence.printed-command.raw.txt")).read_bytes() + start = len(raw) - len(expected) + restored_end = raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + assert raw.rfind(b"\x1b[?1049l") > raw.rfind(b"\x1b[?1049h") >= 0 + assert raw.rfind(b"\x1b[?25h") > raw.rfind(b"\x1b[?25l") >= 0 + assert restored_end <= start and raw.rfind(b"\x1b[?25h") < start + assert printed == expected == raw[start:], (scene, "printed command bytes changed") + assert raw[restored_end:].count(expected) == 1 + assert evidence["printed_command"] == {"byte_offset": start, "byte_length": len(expected), + "sha256": hashlib.sha256(expected).hexdigest()} + continue request = mutations[0] if scene in {"org-limit", "project-limit", "uncertain"}: assert request["body"] == {"currency": "USD", "interval": "month", "threshold_amount": 12525} @@ -76,20 +115,14 @@ def main(): assert request["status"] == "response_lost_after_write" else: assert request["status"] == 200 - checkpoints = {point["name"]: point for point in evidence["checkpoints"]} assert "review-command" in checkpoints and "review-apply" in checkpoints assert checkpoints["review-command"]["seconds"] < checkpoints["review-apply"]["seconds"] - preview = (root / (scene + "-evidence.review-command.txt")).read_text() - preview = "".join(line.rstrip() for line in preview.splitlines()) - assert "Direct API command" in preview and staged["path"] in preview, (scene, "missing real command preview") - assert "Command printing needs" not in preview and "Command preview omitted" not in preview assert [event["key"] for event in evidence["keys"]].count("Apply") == 1 - final_screen = (root / (scene + "-evidence.exit.txt")).read_text() - assert "> Apply change" not in final_screen, (scene, "stale confirmation after exit") - print("PASS: nine real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") + print("PASS: ten real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") print("PASS: one mutation per confirmed action; no retry after uncertain outcome; slow load canceled without mutation.") - print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; eight real command previews; staged files removed.") + print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; nine inspection previews; staged files removed.") print("PASS: all seven terminal mode fields restored before Bash exits; only Darwin PENDIN normalized.") + print("PASS: Ctrl+P exits without mutation; the complete command prints byte-for-byte after terminal restoration.") print("LIMIT: inspect all checkpoint images and GIF frames before claiming visual acceptance.") From 985eee116b4322727784dae737b0a6d9c5f5c93f Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 12:46:10 -0700 Subject: [PATCH 15/44] fix(demo): pass the explicit base URL for command printing --- scripts/demos/limits-workspace/scene.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index a0c7a187..86d71512 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -307,6 +307,8 @@ def main(): raise AssertionError("the recorder runtime must use an owned synthetic /private/tmp directory") project = scene in {"project-limit", "alert-edit", "rate-update"} args = ["projects", "limits", "--project-id", "proj_demo"] if project else ["admin", "limits"] + if scene == "alert-print": + args.insert(0, "--base-url=" + os.environ["OPENAI_BASE_URL"]) evidence = {"scene": scene, "width": width, "height": height, "command": ["openai", *args], "keys": [], "checkpoints": [], "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), "parent_shell": "/bin/bash", "recorder_runtime": str(runtime), From 2b3edf5bdd4959c804af431282d668acf8fde6b2 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 13:16:15 -0700 Subject: [PATCH 16/44] fix(limits): preserve checkpoint rendering and clarify prefix edits --- pkg/custom/limits_workspace_test.go | 9 +++++ pkg/custom/limits_workspace_view.go | 2 +- scripts/demos/limits-workspace/README.md | 3 ++ scripts/demos/limits-workspace/record.sh | 10 ++++-- scripts/demos/limits-workspace/validate.py | 39 ++++++++++++++++++++-- 5 files changed, 57 insertions(+), 6 deletions(-) diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index 5db113a2..a7b4b37f 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -697,6 +697,8 @@ func TestLimitsDraftExactAmountsAndPrefixPresence(t *testing.T) { {name: "replace empty", original: `,"subject_prefix":"Existing"`, mode: 1, set: true}, {name: "clear null", original: `,"subject_prefix":"Existing"`, mode: 2, set: true, null: true}, {name: "edited value", value: "New prefix", set: true}, + {name: "edited original", original: `,"subject_prefix":"Existing"`, value: "New prefix", set: true}, + {name: "edited original to empty", original: `,"subject_prefix":"Existing"`, set: true}, } { t.Run(tc.name, func(t *testing.T) { raw := strings.Replace(limitsControllerAlert, `["synthetic@example.invalid"]}`, `["synthetic@example.invalid"]`+tc.original+`}`, 1) @@ -782,12 +784,19 @@ func TestLimitsWorkspaceActionVisibleAtMinimumSize(t *testing.T) { m.draft.inputs[0].SetValue("100.01") if kind == "alert_create" { m.draft.inputs[1].SetValue("synthetic@example.invalid") + m.draft.inputs[2].SetValue("Edited prefix") + if view := ansi.Strip(m.View().Content); !strings.Contains(view, "Prefix: Keep unless edited") { + t.Fatalf("edited prefix has misleading mode guidance: %q", view) + } } } m.review() if m.page != "review" { t.Fatalf("fixture could not enter review: %s", m.formError) } + if kind == "alert_create" && (!m.change.SubjectPrefixSet || m.change.SubjectPrefix == nil || *m.change.SubjectPrefix != "Edited prefix") { + t.Fatalf("review did not retain the edited prefix: %+v", m.change) + } m.viewport.GotoBottom() view := ansi.Strip(m.View().Content) if !strings.Contains(view, "> Back") || !strings.Contains(view, "Apply change") || !strings.Contains(view, "Ctrl+C") { diff --git a/pkg/custom/limits_workspace_view.go b/pkg/custom/limits_workspace_view.go index 0d8900fb..82b19ca6 100644 --- a/pkg/custom/limits_workspace_view.go +++ b/pkg/custom/limits_workspace_view.go @@ -92,7 +92,7 @@ func (m *limitsWorkspace) View() tea.View { body = append(body, line(m.draft.labels[i]), marker+ansi.Truncate(m.draft.inputs[i].View(), width, "…")) } if count == 3 { - modes := []string{"Preserve original", "Replace (empty is allowed)", "Clear to null"} + modes := []string{"Keep unless edited", "Replace (empty is allowed)", "Clear to null"} body = append(body, mark("Prefix: "+modes[m.draft.prefixMode], m.focus == count)) } body = append(body, mark("Review change", m.focus >= count && !(count == 3 && m.focus == count))) diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index 58fe4947..c6878d8f 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -61,6 +61,9 @@ The scene driver restores terminal output processing and closes every owned chil The recorder preserves raw child bytes, child casts, outer casts, request logs, hashes, and actual process statuses. The validator checks byte identity across all three capture forms. +Checkpoint renderers receive no stdin. The recorder logs their exact names, timestamps, and output filenames. +A second validation pass compares that inventory with source checkpoints and all PNG and frame-GIF filenames. +It rejects missing, duplicate, altered, or unexpected checkpoint artifacts. It checks exact mutation bodies and guards against automatic retry after an uncertain write. Terminal captures merge stdout and stderr. They do not prove separate stream routing. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh index 9c5c2395..b9fc8169 100755 --- a/scripts/demos/limits-workspace/record.sh +++ b/scripts/demos/limits-workspace/record.sh @@ -66,14 +66,18 @@ shasum -a 256 -c "$demo_output/source-sha256.txt" > "$demo_output/source-validat "$demo_python" -I -B "$record_source/validate.py" "$demo_output" > "$demo_output/validation.txt" demo_assemble_capture 150 "${record_scenes[@]}" # Select checkpoint frames from the exact child captures. No UI text is reconstructed. +: > "$demo_output/checkpoint-frames.txt" for record_scene in "${record_scenes[@]}"; do while IFS=$'\t' read -r record_name record_seconds; do "$demo_agg" --quiet "${demo_render_options[@]}" --select "$record_seconds" \ - "$demo_output/$record_scene-evidence.child.cast" "$demo_output/$record_scene-$record_name-frame.gif" - "$demo_ffmpeg" -hide_banner -loglevel error -y -i "$demo_output/$record_scene-$record_name-frame.gif" \ - -frames:v 1 "$demo_output/$record_scene-$record_name.png" + "$demo_output/$record_scene-evidence.child.cast" "$demo_output/$record_scene-$record_name-frame.gif" < /dev/null + "$demo_ffmpeg" -nostdin -hide_banner -loglevel error -y -i "$demo_output/$record_scene-$record_name-frame.gif" \ + -frames:v 1 "$demo_output/$record_scene-$record_name.png" < /dev/null + printf '%s\t%s\t%s\t%s\t%s\n' "$record_scene" "$record_name" "$record_seconds" \ + "$record_scene-$record_name-frame.gif" "$record_scene-$record_name.png" >> "$demo_output/checkpoint-frames.txt" done < <("$demo_python" -I -B -c 'import json,sys; data=json.load(open(sys.argv[1])); [print(item["name"]+"\t"+str(item["seconds"])) for item in data["checkpoints"]]' "$demo_output/$record_scene-evidence.json") done +"$demo_python" -I -B "$record_source/validate.py" "$demo_output" --rendered-checkpoints >> "$demo_output/validation.txt" ( cd "$demo_output" shasum -a 256 -- *.cast *.gif *.png *.txt *.json *.jsonl *.sh > SHA256SUMS diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index d181938d..d4c1deed 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -3,6 +3,7 @@ import hashlib import json +import math from pathlib import Path import sys import termios @@ -13,10 +14,44 @@ "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print") +def validate_rendered_checkpoints(root): + expected_rows = [] + expected_pngs = {scene + ".png" for scene in SCENES} + expected_frames = {scene + "-frame.gif" for scene in SCENES} + for scene in SCENES: + evidence = json.loads((root / (scene + "-evidence.json")).read_text()) + assert evidence["scene"] == scene + names = set() + for point in evidence["checkpoints"]: + name, seconds = point["name"], point["seconds"] + assert name and all(character in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in name) + assert name not in names, (scene, "duplicate checkpoint name", name) + names.add(name) + assert type(seconds) in {int, float} and math.isfinite(seconds) and seconds >= 0 + frame, png = scene + "-" + name + "-frame.gif", scene + "-" + name + ".png" + expected_rows.append("\t".join((scene, name, str(seconds), frame, png))) + expected_frames.add(frame) + expected_pngs.add(png) + assert names, (scene, "missing checkpoints") + actual_rows = (root / "checkpoint-frames.txt").read_text().splitlines() + assert actual_rows == expected_rows, "rendered checkpoint names, times, or filenames differ from source evidence" + actual_pngs = {item.name for item in root.glob("*.png")} + actual_frames = {item.name for item in root.glob("*-frame.gif")} + assert actual_pngs == expected_pngs, ("PNG inventory mismatch", sorted(actual_pngs ^ expected_pngs)) + assert actual_frames == expected_frames, ("checkpoint GIF inventory mismatch", sorted(actual_frames ^ expected_frames)) + for name in expected_pngs | expected_frames: + artifact = root / name + assert artifact.is_file() and not artifact.is_symlink() and artifact.stat().st_size > 0, name + print("PASS: rendered checkpoint names, exact times, PNGs, and frame GIFs match source evidence without missing or extra files.") + + def main(): - if len(sys.argv) != 2: - raise SystemExit("usage: validate.py OUTPUT_DIRECTORY") + if len(sys.argv) not in {2, 3} or len(sys.argv) == 3 and sys.argv[2] != "--rendered-checkpoints": + raise SystemExit("usage: validate.py OUTPUT_DIRECTORY [--rendered-checkpoints]") root = Path(sys.argv[1]) + if len(sys.argv) == 3: + validate_rendered_checkpoints(root) + return requests = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] for scene in SCENES: evidence = json.loads((root / (scene + "-evidence.json")).read_text()) From 6304fb997b8dc28bcf3bc52b4c07cb3c4cd4e7d0 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 13:34:16 -0700 Subject: [PATCH 17/44] fix(limits): explain preflight HTTP failures and record recovery --- pkg/custom/limits_workspace.go | 4 + pkg/custom/limits_workspace_test.go | 87 +++++++++++++++++----- scripts/demos/limits-workspace/README.md | 5 ++ scripts/demos/limits-workspace/record.sh | 2 +- scripts/demos/limits-workspace/scene.py | 28 +++++++ scripts/demos/limits-workspace/server.py | 9 ++- scripts/demos/limits-workspace/validate.py | 34 +++++++-- 7 files changed, 145 insertions(+), 24 deletions(-) diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index 83800cd1..b3957374 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -755,6 +755,8 @@ func limitsPreflightHint(err error, invocation ...string) string { var apiErr *openai.Error if errors.As(err, &apiErr) { switch apiErr.StatusCode { + case 400, 422: + return fmt.Sprintf("The API returned HTTP %d. No write sent.\nRecheck the request fields and scope, then press r to refresh.", apiErr.StatusCode) case 401: return "Admin authentication failed. No write sent.\nKey setup: " + bin + " help setup admin" case 403: @@ -772,6 +774,8 @@ func limitsPreflightHint(err error, invocation ...string) string { default: return "The API returned HTTP 429. No write sent.\nWait before refreshing. Check API error details if this continues." } + default: + return fmt.Sprintf("The API returned HTTP %d. No write sent.\nPress r to refresh. If this continues, check API availability and access.", apiErr.StatusCode) } } if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index a7b4b37f..865bca46 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -901,11 +901,13 @@ func TestLimitsPreflightGuidancePreservesFailureScope(t *testing.T) { func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { for _, status := range []int{400, 422, 409, 503} { - for _, operation := range []string{"read", "mutation"} { + for _, operation := range []string{"read", "preflight", "mutation"} { t.Run(fmt.Sprintf("%s/%d", operation, status), func(t *testing.T) { - var writes atomic.Int32 + var writes, preflightReads atomic.Int32 + var rejectPreflight atomic.Bool backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { - if r.Method == http.MethodGet && operation == "mutation" { + preflight := r.Method == http.MethodGet && operation == "preflight" && rejectPreflight.Load() && strings.HasSuffix(r.URL.Path, "/spend_limit") + if r.Method == http.MethodGet && (operation == "mutation" || operation == "preflight" && !preflight) { if strings.HasSuffix(r.URL.Path, "/spend_limit") { limitsTestWrite(t, w, limitsTestLimit) } else { @@ -916,6 +918,9 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { if r.Method != http.MethodGet { writes.Add(1) } + if preflight { + preflightReads.Add(1) + } w.WriteHeader(status) limitsTestWrite(t, w, `{"error":{"message":"synthetic-private-message\u001b[2J","param":"synthetic-private-param\nline","code":"synthetic-private-code\u001b]0;title\u0007"}}`) }) @@ -925,18 +930,43 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { if operation == "read" { result = limitsControllerRun(t, m.load()) } else { - backend.client = openai.NewClient(append(backend.client.Options, option.WithMaxRetries(3))...) - m.snapshot.LimitAbsent, m.snapshot.Limit = false, gjson.Parse(limitsTestLimit) + if operation == "preflight" { + overview := limitsControllerRun(t, m.load()) + if overview.err != nil || len(overview.snapshot.Errors) != 0 || overview.snapshot.Limit.Raw != limitsTestLimit { + t.Fatal("preflight fixture did not load a successful overview") + } + m.Update(overview) + rejectPreflight.Store(true) + } else { + backend.client = openai.NewClient(append(backend.client.Options, option.WithMaxRetries(3))...) + m.snapshot.LimitAbsent, m.snapshot.Limit = false, gjson.Parse(limitsTestLimit) + } limitsControllerReview(t, m) m.viewport.GotoBottom() limitsControllerKey(m, tea.KeyTab) result = limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter)) var attempted *limitsMutationFailure - if !errors.As(result.err, &attempted) || !attempted.Attempted || writes.Load() != 1 { - t.Fatal("failed mutation lost its attempted state or sent more than once") + wantAttempted, wantWrites := operation == "mutation", int32(0) + if wantAttempted { + wantWrites = 1 + } + if !errors.As(result.err, &attempted) || attempted.Attempted != wantAttempted || writes.Load() != wantWrites { + t.Fatal("failed operation changed its attempted state or mutation count") } } m.Update(result) + if operation == "preflight" { + if m.locked || !m.refreshRequired || m.page != "form" || m.draft.inputs[0].Value() != "100.01" || m.change.Before.Raw != limitsTestLimit || m.change.Amount != 10001 { + t.Fatal("HTTP preflight failure discarded the draft or changed its recovery state") + } + if !strings.Contains(m.formError, fmt.Sprintf("HTTP %d", status)) || strings.Contains(m.formError, "synthetic-private") || strings.Contains(m.formError, "\x1b") { + t.Fatal("preflight form omitted the status or exposed response data") + } + m.setFocus(len(m.draft.inputs)) + if next := limitsControllerKey(m, tea.KeyEnter); next != nil || m.page != "form" || m.busy || !m.refreshRequired || preflightReads.Load() != 1 || writes.Load() != 0 { + t.Fatal("HTTP preflight failure allowed review or another request before refresh") + } + } limitsControllerKey(m, tea.KeyF1) if !strings.Contains(m.details, fmt.Sprintf("HTTP %d", status)) || strings.Contains(m.details, "synthetic-private") || strings.Contains(m.details, "\x1b") { t.Fatal("recovery omitted the status or exposed response data") @@ -956,22 +986,39 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { } return } - if !m.locked || !strings.Contains(m.details, "It may have succeeded") || strings.Contains(m.details, "No write sent.") { - t.Fatal("HTTP failure removed mutation uncertainty or the write lock") - } - limitsControllerKey(m, tea.KeyEscape) - limitsControllerKey(m, tea.KeyEnter) - m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) - limitsControllerKey(m, tea.KeyEnter) - if !m.locked || writes.Load() != 1 { - t.Fatal("Enter or refresh repeated a failed mutation") + if operation == "preflight" { + if !strings.Contains(m.details, "No write sent.") || !strings.Contains(m.details, "press r to refresh") && !strings.Contains(m.details, "Press r to refresh") || strings.Contains(m.details, "It may have succeeded") { + t.Fatal("preflight recovery lost safe refresh guidance or claimed a possible mutation") + } + if (status == 409 || status == 503) && !strings.Contains(m.details, "API availability and access") { + t.Fatal("preflight recovery omitted availability and access guidance") + } + } else { + if !m.locked || !strings.Contains(m.details, "It may have succeeded") || strings.Contains(m.details, "No write sent.") { + t.Fatal("HTTP failure removed mutation uncertainty or the write lock") + } + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, tea.KeyEnter) + m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) + limitsControllerKey(m, tea.KeyEnter) + if !m.locked || writes.Load() != 1 { + t.Fatal("Enter or refresh repeated a failed mutation") + } } m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) if !errors.Is(failure, result.err) || !errors.Is(failure, syscall.EPIPE) || clioutput.IsBrokenPipe(failure) { - t.Fatal("finalization discarded the mutation or output failure") + t.Fatal("finalization discarded the operation or output failure") } baseline := "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + if operation == "preflight" { + baseline = fmt.Sprintf("The API returned HTTP %d. No write sent.\n", status) + if status == 400 || status == 422 { + baseline += "Recheck the request fields and scope, then press r to refresh." + } else { + baseline += "Press r to refresh. If this continues, check API availability and access." + } + } for _, args := range [][]string{nil, {"--format", "json", "--format-error", "text"}, {"--format", "text", "--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { root := readableErrorTestCommand(t, args...) var out bytes.Buffer @@ -992,6 +1039,12 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { t.Fatal("machine error message changed") } default: + if operation == "preflight" { + if !strings.Contains(out.String(), baseline) || strings.Contains(out.String(), "may have succeeded") { + t.Fatal("final text lost preflight status, outcome, or recovery guidance") + } + break + } if !strings.Contains(out.String(), fmt.Sprintf("HTTP %d", status)) || !strings.Contains(out.String(), "may have succeeded") { t.Fatal("final text lost HTTP status or mutation uncertainty") } diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index c6878d8f..5609edda 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -43,6 +43,7 @@ The isolated child environment includes only fake credentials and the fixture en | `rate-update` | Project | Change requests per minute to 250; preserve all other dimensions | | `uncertain` | Organization | Apply a synthetic write, lose its response, and verify that repetition stays blocked | | `slow-cancel` | Organization | Cancel a held read with Ctrl+C; perform no mutation | +| `preflight-400` | Organization | Reject the preflight read, retain the draft, explain recovery, and exit with failure without mutation | Each mutation scene opens a form or deletion review. The driver reads to the review's end before selecting Apply. @@ -53,6 +54,9 @@ Successful mutation scenes require exit status 0 and one mutation each. The print scene uses Ctrl+P and requires exit status 0 with no mutation. It compares the complete printed command bytes after terminal restoration, including the final newline. Wrapped inspection rows do not establish command copyability. +The preflight scene loads the overview before returning HTTP 400 from the confirmation read. +It retains the draft, blocks review before refresh, and checks F1 recovery at both widths. +Normal exit preserves the failure and hides private response data. No mutation occurs. The fixture stores only synthetic request bodies, response hashes, and method/path/status records. It never logs request headers or credentials. @@ -77,6 +81,7 @@ Planned screenshot evidence comes from recorded checkpoints: - Printed command: the complete command after terminal restoration, with no mutation. - Saved state: refreshed resource and confirmed outcome. - Uncertain state: retained draft and blocked retry guidance. +- Preflight recovery: retained draft, required refresh, HTTP status, and field/scope guidance. - Loading state: cancellation remains available while a read is pending. - Final scene frame: restored shell, actual result, and no stale confirmation question. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh index b9fc8169..5cd383c3 100755 --- a/scripts/demos/limits-workspace/record.sh +++ b/scripts/demos/limits-workspace/record.sh @@ -50,7 +50,7 @@ shasum -a 256 "$record_source/record.sh" "$record_source/server.py" "$record_sou demo_window_size="${record_width}x$((record_height+4))" demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ --theme asciinema --fps-cap 15 --last-frame-duration 2) -record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel alert-print) +record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel alert-print preflight-400) for record_scene in "${record_scenes[@]}"; do demo_capture_scene "$record_scene" 0 "$demo_runtime/after" "$demo_api_url/$record_scene/v1" \ "$record_scene: ${record_width}x${record_height}" \ diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index 86d71512..1fc1ef1f 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -274,6 +274,34 @@ def run_scene(terminal, scene, evidence, output): evidence["printed_command"] = {"byte_offset": start, "byte_length": len(expected), "sha256": hashlib.sha256(expected).hexdigest()} checkpoint(terminal, evidence, output, "printed-command") + elif scene == "preflight-400": + wait_screen(terminal, ["HTTP 400", "125.25", "Review change"], ["Saving"]) + checkpoint(terminal, evidence, output, "preflight", ("HTTP 400", "125.25")) + key(terminal, evidence, "Blocked review before refresh", b"\r\r") + wait_screen(terminal, ["Refresh with r", "125.25", "Review change"]) + checkpoint(terminal, evidence, output, "refresh-required", ("Refresh with r", "125.25")) + key(terminal, evidence, "F1 recovery", b"\x1bOP") + help_screen = wait_screen(terminal, ["HTTP 400", "Recheck"]) + compact_help = "".join(help_screen.split()) + for message in ("No write sent.", "request fields and scope", "press r to refresh"): + if "".join(message.split()) not in compact_help: + raise AssertionError("preflight help omitted status or recovery guidance") + checkpoint(terminal, evidence, output, "preflight-help", ("HTTP 400",)) + key(terminal, evidence, "Back", b"\x1b") + wait_screen(terminal, ["Set hard spend limit"]) + key(terminal, evidence, "Quit", b"q") + terminal.finish(1) + if mutations(scene): + raise AssertionError("failed preflight sent a mutation") + if b"synthetic-private-preflight" in terminal.raw: + raise AssertionError("preflight recovery exposed private response data") + restored = terminal.raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + diagnostic = terminal.raw[restored:] + for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"press r to refresh"): + if message not in diagnostic: + raise AssertionError("normal exit lost the preflight diagnostic") + if b"may have succeeded" in diagnostic: + raise AssertionError("preflight exit incorrectly claimed an uncertain mutation") elif scene == "uncertain": wait_screen(terminal, ["Save not confirmed", "Review change"]) checkpoint(terminal, evidence, output, "uncertain") diff --git a/scripts/demos/limits-workspace/server.py b/scripts/demos/limits-workspace/server.py index 33d80ac2..0d08c552 100755 --- a/scripts/demos/limits-workspace/server.py +++ b/scripts/demos/limits-workspace/server.py @@ -15,7 +15,7 @@ SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", - "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print") + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") def write_text(path, value): @@ -51,6 +51,7 @@ def main(): gate = Path(sys.argv[4]) gate.mkdir() states = {name: initial_state(name in {"project-limit", "alert-edit", "rate-update"}) for name in SCENES} + limit_reads = {name: 0 for name in SCENES} write_text(sys.argv[3], json.dumps({"data": "synthetic only", "initial": states}, indent=2) + "\n") stop, lock, failures = threading.Event(), threading.Lock(), [] with open(sys.argv[2], "x", encoding="utf-8") as log: @@ -106,7 +107,13 @@ def handle_request(self): if time.monotonic() >= deadline: raise TimeoutError("slow scene did not confirm cancellation") return + limit_reads[scene] += 1 result = state["limit"] + if scene == "preflight-400" and limit_reads[scene] > 1: + status, result = 400, {"error": { + "message": "synthetic-private-preflight-message\u001b[2J", + "param": "synthetic-private-preflight-field\nline", + "code": "synthetic-private-preflight-code\u001b]0;title\u0007"}} if result is None: status, result = 404, {"error": {"message": "Synthetic limit is absent"}} elif self.command == "POST": diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index d4c1deed..fba6b089 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -11,7 +11,7 @@ SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", - "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print") + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") def validate_rendered_checkpoints(root): @@ -55,7 +55,7 @@ def main(): requests = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] for scene in SCENES: evidence = json.loads((root / (scene + "-evidence.json")).read_text()) - expected_status = 130 if scene == "slow-cancel" else 1 if scene == "uncertain" else 0 + expected_status = 130 if scene == "slow-cancel" else 1 if scene in {"uncertain", "preflight-400"} else 0 assert evidence["exit_status"] == expected_status, (scene, "incorrect child exit") assert evidence["cli_exit_status"] == evidence["exit_status"], (scene, "Bash changed the CLI exit status") assert evidence["parent_shell"] == "/bin/bash", (scene, "missing native Bash parent") @@ -89,7 +89,7 @@ def main(): assert relay[:-len(prompt)] == raw == child_bytes, (scene, "capture changed CLI bytes") assert hashlib.sha256(raw).hexdigest() == evidence["relayed_sha256"] mutations = [entry for entry in requests if entry["scene"] == scene and entry["method"] in {"POST", "DELETE"}] - assert len(mutations) == (0 if scene in {"slow-cancel", "alert-print"} else 1), (scene, "unexpected mutation count") + assert len(mutations) == (0 if scene in {"slow-cancel", "alert-print", "preflight-400"} else 1), (scene, "unexpected mutation count") if scene == "slow-cancel": assert any(entry["scene"] == scene and entry["status"] == "held" for entry in requests) continue @@ -131,6 +131,29 @@ def main(): assert evidence["printed_command"] == {"byte_offset": start, "byte_length": len(expected), "sha256": hashlib.sha256(expected).hexdigest()} continue + if scene == "preflight-400": + scene_requests = [entry for entry in requests if entry["scene"] == scene] + assert scene_requests and all(entry["method"] == "GET" for entry in scene_requests) + limit_reads = [entry for entry in scene_requests if entry["path"].endswith("/spend_limit")] + assert [entry["status"] for entry in limit_reads] == [200, 400], (scene, "preflight did not follow a successful overview") + assert all(entry["status"] == 200 for entry in scene_requests if not entry["path"].endswith("/spend_limit")) + required = ("review-command", "review-apply", "preflight", "refresh-required", "preflight-help") + assert all(name in checkpoints for name in required) + assert all(checkpoints[left]["seconds"] < checkpoints[right]["seconds"] for left, right in zip(required, required[1:])) + keys = [event["key"] for event in evidence["keys"]] + assert keys.count("Apply") == keys.count("Blocked review before refresh") == keys.count("F1 recovery") == 1 + for name, messages in (("preflight", ("HTTP 400", "125.25", "Review change")), + ("refresh-required", ("Refresh with r", "125.25", "Review change")), + ("preflight-help", ("HTTP 400", "No write sent.", "request fields and scope", "press r to refresh"))): + screen = (root / (scene + "-evidence." + name + ".txt")).read_text() + compact = "".join(screen.split()) + assert all("".join(message.split()) in compact for message in messages), (scene, name, "missing recovery evidence") + assert b"synthetic-private-preflight" not in raw, (scene, "private response leaked") + restored_end = raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + diagnostic = raw[restored_end:] + assert all(message in diagnostic for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"press r to refresh")) + assert b"may have succeeded" not in diagnostic + continue request = mutations[0] if scene in {"org-limit", "project-limit", "uncertain"}: assert request["body"] == {"currency": "USD", "interval": "month", "threshold_amount": 12525} @@ -153,11 +176,12 @@ def main(): assert "review-command" in checkpoints and "review-apply" in checkpoints assert checkpoints["review-command"]["seconds"] < checkpoints["review-apply"]["seconds"] assert [event["key"] for event in evidence["keys"]].count("Apply") == 1 - print("PASS: ten real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") + print("PASS: eleven real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") print("PASS: one mutation per confirmed action; no retry after uncertain outcome; slow load canceled without mutation.") - print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; nine inspection previews; staged files removed.") + print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; ten inspection previews; staged files removed.") print("PASS: all seven terminal mode fields restored before Bash exits; only Darwin PENDIN normalized.") print("PASS: Ctrl+P exits without mutation; the complete command prints byte-for-byte after terminal restoration.") + print("PASS: preflight HTTP 400 retains the draft; safe F1 recovery; zero mutations; normal exit preserves failure.") print("LIMIT: inspect all checkpoint images and GIF frames before claiming visual acceptance.") From 3aedfabab57075c82ff6630c5395265d841437ac Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 13:44:40 -0700 Subject: [PATCH 18/44] test(limits): return Costs page in preflight fixture --- pkg/custom/limits_workspace_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index 865bca46..361e9ada 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -910,6 +910,8 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { if r.Method == http.MethodGet && (operation == "mutation" || operation == "preflight" && !preflight) { if strings.HasSuffix(r.URL.Path, "/spend_limit") { limitsTestWrite(t, w, limitsTestLimit) + } else if strings.HasSuffix(r.URL.Path, "/costs") { + limitsTestWrite(t, w, `{"object":"page","data":[],"has_more":false}`) } else { limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) } From ab6b57c257aeb80e619a10d42d449f4ca503fa5e Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 14:05:20 -0700 Subject: [PATCH 19/44] fix limits recovery guidance after workspace exit --- pkg/custom/limits_workspace.go | 45 +++++-- pkg/custom/limits_workspace_test.go | 132 ++++++++++++++++++++- scripts/demos/limits-workspace/README.md | 2 + scripts/demos/limits-workspace/scene.py | 14 ++- scripts/demos/limits-workspace/validate.py | 12 +- 5 files changed, 190 insertions(+), 15 deletions(-) diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index b3957374..1228ba8d 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -185,10 +185,10 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error if pending || m.locked { message = "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() if mutationErr != nil { - humanDetail = limitsMutationHint(mutationErr, m.invocation) + humanDetail = limitsMutationRecovery(mutationErr, m.invocation, false) } } else if m.refreshRequired { - message = limitsPreflightHint(mutationErr, m.invocation) + message = limitsPreflightRecovery(mutationErr, false, m.invocation) + "\n" + m.readRecovery() if m.lastSaved.Exists() { message += "\nA preceding change was confirmed. The latest change was not sent." } @@ -748,21 +748,37 @@ func (m *limitsWorkspace) limitReadable() bool { func limitsSafeJSON(value gjson.Result) string { return limitsSafeLine(value.Raw) } func limitsPreflightHint(err error, invocation ...string) string { + return limitsPreflightRecovery(err, true, invocation...) +} + +func limitsPreflightRecovery(err error, inWorkspace bool, invocation ...string) string { bin := "openai" if len(invocation) != 0 && invocation[0] != "" { bin = invocation[0] } + refresh := "Press r to refresh." + validation := "Recheck the request fields and scope, then press r to refresh." + baseline := "Press r to refresh this draft's baseline." + conflict := "Press r to refresh and review the new baseline." + interrupted := "Press r to refresh before trying again." + wait := "Wait before refreshing. Check API error details if this continues." + if !inWorkspace { + refresh = "Reopen the limits workspace and review current settings before another change." + validation = "Recheck the request fields and scope.\n" + refresh + baseline, conflict, interrupted = refresh, refresh, refresh + wait = "Wait before reopening the limits workspace. Check API error details if this continues." + } var apiErr *openai.Error if errors.As(err, &apiErr) { switch apiErr.StatusCode { case 400, 422: - return fmt.Sprintf("The API returned HTTP %d. No write sent.\nRecheck the request fields and scope, then press r to refresh.", apiErr.StatusCode) + return fmt.Sprintf("The API returned HTTP %d. No write sent.\n%s", apiErr.StatusCode, validation) case 401: return "Admin authentication failed. No write sent.\nKey setup: " + bin + " help setup admin" case 403: return "The API denied access. No write sent.\nCheck Admin-key permissions and organization or project access.\nAccess setup: " + bin + " help setup admin" case 404: - return "The requested resource was not returned. No write sent.\nCheck the API address, scope, and resource ID. Press r to refresh." + return "The requested resource was not returned. No write sent.\nCheck the API address, scope, and resource ID. " + refresh case 429: switch apiErr.Code { case "organization_spend_limit_exceeded": @@ -772,26 +788,26 @@ func limitsPreflightHint(err error, invocation ...string) string { case "insufficient_quota", "billing_hard_limit_reached", "credit_balance_exhausted", "organization_usage_limit_exceeded": return "The API reports a billing or usage limit. No write sent.\nCheck billing and approved usage: https://platform.openai.com/settings/organization/limits" default: - return "The API returned HTTP 429. No write sent.\nWait before refreshing. Check API error details if this continues." + return "The API returned HTTP 429. No write sent.\n" + wait } default: - return fmt.Sprintf("The API returned HTTP %d. No write sent.\nPress r to refresh. If this continues, check API availability and access.", apiErr.StatusCode) + return fmt.Sprintf("The API returned HTTP %d. No write sent.\n%s If this continues, check API availability and access.", apiErr.StatusCode, refresh) } } if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return "The check was interrupted. No write sent. Press r to refresh before trying again." + return "The check was interrupted. No write sent. " + interrupted } if err == nil { - return "No change sent. Press r to refresh this draft's baseline." + return "No change sent. " + baseline } // Only exact locally defined messages select specific recovery guidance. switch err.Error() { case "Rate-limit editing requires a complete list. Reopen with a larger --max-items value or -1.": return "No write sent. Reopen with --max-items -1 to check every rate record." case "The configuration changed. Refresh before applying this change.": - return "Record changed. No write sent. Press r to refresh and review the new baseline." + return "Record changed. No write sent. " + conflict default: - return "No change sent. Press r to refresh this draft's baseline." + return "No change sent. " + baseline } } @@ -878,11 +894,18 @@ func limitsReadHint(err error, invocation string) string { } func limitsMutationHint(err error, invocation string) string { + return limitsMutationRecovery(err, invocation, true) +} + +func limitsMutationRecovery(err error, invocation string, inWorkspace bool) string { var apiErr *openai.Error if errors.As(err, &apiErr) { switch apiErr.StatusCode { case 401, 403, 404, 429: - return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "The change is not confirmed.") + if !inWorkspace && apiErr.StatusCode == 404 { + return "The requested resource was not returned. The change is not confirmed.\nCheck the API address, scope, and resource ID. Read the current record before another change." + } + return strings.ReplaceAll(limitsPreflightRecovery(err, inWorkspace, invocation), "No write sent.", "The change is not confirmed.") case 400, 422: return fmt.Sprintf("The API returned HTTP %d. The change is not confirmed.\nRecheck the reviewed fields. Read the current record before another change.", apiErr.StatusCode) default: diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index 361e9ada..fb5d50ec 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -1016,10 +1016,11 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { if operation == "preflight" { baseline = fmt.Sprintf("The API returned HTTP %d. No write sent.\n", status) if status == 400 || status == 422 { - baseline += "Recheck the request fields and scope, then press r to refresh." + baseline += "Recheck the request fields and scope.\nReopen the limits workspace and review current settings before another change." } else { - baseline += "Press r to refresh. If this continues, check API availability and access." + baseline += "Reopen the limits workspace and review current settings before another change. If this continues, check API availability and access." } + baseline += "\n" + m.readRecovery() } for _, args := range [][]string{nil, {"--format", "json", "--format-error", "text"}, {"--format", "text", "--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { root := readableErrorTestCommand(t, args...) @@ -1030,6 +1031,9 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { if strings.Contains(out.String(), "synthetic-private") || strings.Contains(out.String(), "\x1b") { t.Fatal("final diagnostic exposed response data") } + if strings.Contains(strings.ToLower(out.String()), "press r") { + t.Fatal("final diagnostic directed the user to a closed workspace") + } switch { case root.String("transform-error") != "": if out.String() != baseline+"\n" { @@ -1060,6 +1064,130 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { } } +func TestLimitsWorkspaceExitRecoveryUsesProcessActions(t *testing.T) { + for _, tc := range []struct { + name, want string + cause error + }{ + {"authentication", "help setup admin", &openai.Error{StatusCode: 401, Message: "synthetic-private-response"}}, + {"access", "organization or project access", &openai.Error{StatusCode: 403, Message: "synthetic-private-response"}}, + {"missing", "scope, and resource ID", &openai.Error{StatusCode: 404, Message: "synthetic-private-response"}}, + {"rate", "Wait before reopening", &openai.Error{StatusCode: 429, Code: "unknown_synthetic_code", Message: "synthetic-private-response"}}, + {"organization cap", "help admin organization spend-limit retrieve", &openai.Error{StatusCode: 429, Code: "organization_spend_limit_exceeded"}}, + {"project cap", "help admin organization projects spend-limit retrieve", &openai.Error{StatusCode: 429, Code: "project_spend_limit_exceeded"}}, + {"billing", "https://platform.openai.com/settings/organization/limits", &openai.Error{StatusCode: 429, Code: "credit_balance_exhausted"}}, + {"canceled", "check was interrupted", context.Canceled}, + {"deadline", "check was interrupted", context.DeadlineExceeded}, + {"changed", "Record changed", errors.New("The configuration changed. Refresh before applying this change.")}, + {"incomplete rates", "Reopen with --max-items -1", errors.New("Rate-limit editing requires a complete list. Reopen with a larger --max-items value or -1.")}, + {"unknown", "Reopen the limits workspace", errors.New("synthetic-private-response\x1b[2J")}, + } { + t.Run(tc.name, func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.invocation, m.snapshot.ProjectID = "custom-openai", "synthetic-private-project" + limitsControllerReview(t, m) + m.mutationError = &limitsMutationFailure{Err: tc.cause, Attempted: false} + m.Update(limitsWorkResult{mutation: true, err: m.mutationError}) + limitsControllerKey(m, tea.KeyF1) + if tc.name == "missing" || tc.name == "canceled" || tc.name == "deadline" || tc.name == "changed" || tc.name == "unknown" { + if !strings.Contains(m.details, "Press r to refresh") { + t.Fatal("in-workspace recovery lost its keyboard action") + } + } + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, 'q') + m.lastSaved = gjson.Parse(strings.Replace(limitsTestLimit, "organization.spend_limit", "project.spend_limit", 1)) + m.lastSavedOperation = limitsChangeOperation("limit_set", true) + var receipt bytes.Buffer + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "json", ExplicitFormat: true, Stdout: &receipt, Stderr: io.Discard}) + var safe *limitsWorkflowFailure + if !errors.As(failure, &safe) || !errors.Is(failure, tc.cause) || backend.writes.Load() != 0 { + t.Fatal("final recovery lost the cause, failure, or zero-write outcome") + } + if !gjson.Valid(receipt.String()) || gjson.Get(receipt.String(), "threshold_amount").Int() != 10000 || !strings.Contains(safe.Error(), "A preceding change was confirmed.") { + t.Fatal("final recovery discarded the preceding confirmed result") + } + for _, args := range [][]string{nil, {"--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { + root := readableErrorTestCommand(t, args...) + var out bytes.Buffer + if err := ShowCommandError(root, failure, &out); err != nil { + t.Fatal(err) + } + text := out.String() + if strings.Contains(text, "synthetic-private") || strings.Contains(text, "\x1b") || strings.Contains(strings.ToLower(text), "press r") || strings.Contains(text, "may have succeeded") { + t.Fatal("final recovery leaked private data, keyboard actions, or mutation uncertainty") + } + for _, want := range []string{tc.want, "custom-openai help admin organization projects spend-limit retrieve", "original connection, authentication, and scope options"} { + if !strings.Contains(text, want) { + t.Fatalf("final recovery omitted %q", want) + } + } + if root.String("transform-error") != "" { + if text != safe.Error()+"\n" { + t.Fatal("extracted message differs from the canonical final recovery") + } + } else if errorOutputFormat(root) == "json" { + var value map[string]any + if err := json.Unmarshal(out.Bytes(), &value); err != nil || len(value) != 1 || value["message"] != safe.Error() { + t.Fatal("machine error schema or message changed") + } + } + } + }) + } + for _, status := range []int{404, 429} { + t.Run(fmt.Sprintf("uncertain/%d", status), func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + cause := &openai.Error{StatusCode: status, Message: "synthetic-private-response"} + m.locked, m.mutationError = true, &limitsMutationFailure{Err: cause, Attempted: true} + if status == 404 && !strings.Contains(m.help(), "Press r to refresh") { + t.Fatal("in-workspace mutation recovery lost its keyboard guidance") + } + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Stdout: io.Discard, Stderr: io.Discard}) + if !errors.Is(failure, cause) { + t.Fatal("final mutation recovery lost its cause") + } + baseline := "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + var human, machine bytes.Buffer + if err := ShowCommandError(readableErrorTestCommand(t), failure, &human); err != nil { + t.Fatal(err) + } + if strings.Contains(strings.ToLower(human.String()), "press r") || strings.Contains(human.String(), "synthetic-private") || !strings.Contains(human.String(), baseline) { + t.Fatal("final mutation recovery lost uncertainty or exposed an invalid action") + } + if status == 404 && !strings.Contains(human.String(), "Read the current record before another change.") { + t.Fatal("uncertain 404 did not require a new read") + } + if err := ShowCommandError(readableErrorTestCommand(t, "--format-error", "json"), failure, &machine); err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(machine.Bytes(), &value); err != nil || len(value) != 1 || value["message"] != baseline { + t.Fatal("final mutation recovery changed the machine contract") + } + }) + } + t.Run("interrupted preflight and output failure", func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + cause := &openai.Error{StatusCode: 404, Message: "synthetic-private-response"} + m.refreshRequired, m.interrupted = true, true + m.mutationError = &limitsMutationFailure{Err: errors.Join(cause, context.Canceled), Attempted: false} + m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} + failure := finishLimitsWorkspace(t.Context(), m, tea.ErrInterrupted, ShowJSONOpts{Context: t.Context(), Stdout: io.Discard, Stderr: io.Discard}) + var interrupt interface{ ExitCode() int } + if !errors.Is(failure, cause) || !errors.Is(failure, context.Canceled) || !errors.Is(failure, syscall.EPIPE) || clioutput.IsBrokenPipe(failure) || !errors.As(failure, &interrupt) || interrupt.ExitCode() != 130 { + t.Fatal("final preflight recovery lost a joined cause or interruption status") + } + var out bytes.Buffer + if err := ShowCommandError(readableErrorTestCommand(t), failure, &out); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "synthetic-private") || strings.Contains(strings.ToLower(out.String()), "press r") || !strings.Contains(out.String(), "No write sent.") || !strings.Contains(out.String(), "Reopen the limits workspace") { + t.Fatal("interrupted final preflight recovery lost its process-safe action") + } + }) +} + func TestLimitsWorkspaceCanceledMutationRecovery(t *testing.T) { for _, cause := range []error{context.Canceled, context.DeadlineExceeded} { m, _ := newLimitsController(t, 80, 24) diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index 5609edda..5c3f34da 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -57,6 +57,8 @@ Wrapped inspection rows do not establish command copyability. The preflight scene loads the overview before returning HTTP 400 from the confirmation read. It retains the draft, blocks review before refresh, and checks F1 recovery at both widths. Normal exit preserves the failure and hides private response data. No mutation occurs. +Final recovery tells users to reopen the workspace and preserve the original request scope. +The recorder rejects workspace keyboard actions in failure diagnostics after terminal restoration. The fixture stores only synthetic request bodies, response hashes, and method/path/status records. It never logs request headers or credentials. diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index 1fc1ef1f..15b9eb76 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -297,7 +297,10 @@ def run_scene(terminal, scene, evidence, output): raise AssertionError("preflight recovery exposed private response data") restored = terminal.raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") diagnostic = terminal.raw[restored:] - for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"press r to refresh"): + for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", + b"Reopen the limits workspace", b"review current settings", + b"help admin organization spend-limit retrieve", + b"original connection, authentication, and scope options"): if message not in diagnostic: raise AssertionError("normal exit lost the preflight diagnostic") if b"may have succeeded" in diagnostic: @@ -384,6 +387,15 @@ def main(): raise AssertionError("supplied or staged CLI changed during capture") if str(binary).encode() in terminal.raw: raise AssertionError("capture exposed the private binary source path") + if terminal.child.returncode != 0: + restored = terminal.raw.rfind(b"\x1b[?1049l") + if restored < 0: + raise AssertionError("final diagnostic lacks terminal restoration") + diagnostic = b" ".join(terminal.raw[restored + len(b"\x1b[?1049l"):].lower().split()) + for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", + b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): + if action in diagnostic: + raise AssertionError("final diagnostic names an action from the closed workspace") evidence.update(exit_status=terminal.child.returncode, relayed_bytes=len(terminal.raw), relayed_sha256=hashlib.sha256(terminal.raw).hexdigest()) with Path(str(output) + ".tty").open("xb") as destination: diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index fba6b089..477f0f07 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -88,6 +88,13 @@ def main(): assert relay.endswith(prompt), (scene, "missing restored prompt") assert relay[:-len(prompt)] == raw == child_bytes, (scene, "capture changed CLI bytes") assert hashlib.sha256(raw).hexdigest() == evidence["relayed_sha256"] + if expected_status != 0: + restored = raw.rfind(b"\x1b[?1049l") + assert restored >= 0, (scene, "final diagnostic lacks terminal restoration") + diagnostic = b" ".join(raw[restored + len(b"\x1b[?1049l"):].lower().split()) + for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", + b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): + assert action not in diagnostic, (scene, "final diagnostic names an action from the closed workspace") mutations = [entry for entry in requests if entry["scene"] == scene and entry["method"] in {"POST", "DELETE"}] assert len(mutations) == (0 if scene in {"slow-cancel", "alert-print", "preflight-400"} else 1), (scene, "unexpected mutation count") if scene == "slow-cancel": @@ -151,7 +158,10 @@ def main(): assert b"synthetic-private-preflight" not in raw, (scene, "private response leaked") restored_end = raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") diagnostic = raw[restored_end:] - assert all(message in diagnostic for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"press r to refresh")) + assert all(message in diagnostic for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", + b"Reopen the limits workspace", b"review current settings", + b"help admin organization spend-limit retrieve", + b"original connection, authentication, and scope options")) assert b"may have succeeded" not in diagnostic continue request = mutations[0] From e3968446bed2fa3b67218b623b7dc82c1a2ea0c5 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 14:28:15 -0700 Subject: [PATCH 20/44] clarify limits overview count labels --- pkg/custom/limits_workspace.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index 1228ba8d..164fe9d2 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -730,9 +730,9 @@ func (m *limitsWorkspace) sourceSummary() string { } return "Hard spend limit unavailable. Refresh to retry." case 1: - return fmt.Sprintf("%d alerts loaded. Complete: %t", len(m.snapshot.Alerts), m.snapshot.AlertsComplete) + return fmt.Sprintf("Alerts loaded: %d Complete: %t", len(m.snapshot.Alerts), m.snapshot.AlertsComplete) default: - return fmt.Sprintf("%d rate records loaded. Complete: %t", len(m.snapshot.Rates), m.snapshot.RatesComplete) + return fmt.Sprintf("Rate records loaded: %d Complete: %t", len(m.snapshot.Rates), m.snapshot.RatesComplete) } } From 555fbc74574a65a547d6965443c5d19c825f1c68 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 14:57:31 -0700 Subject: [PATCH 21/44] test: audit Limits command group errors --- cmd/openai/error_coverage_test.go | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/cmd/openai/error_coverage_test.go b/cmd/openai/error_coverage_test.go index b8acc92c..55d38d8d 100644 --- a/cmd/openai/error_coverage_test.go +++ b/cmd/openai/error_coverage_test.go @@ -188,7 +188,9 @@ func buildErrorCoverageManifest(root *cli.Command) (errorCoverageManifest, error } if entry.HasChildren && (command.Action != nil || len(command.Arguments) != 0 || command.ArgValidator != nil) { switch name { - case "help", "help setup", "tokenizer": + case "help", "help setup", "tokenizer", + "admin limits", "admin organization limits", + "projects limits", "admin projects limits", "admin organization projects limits": entry.Class += "; reviewed positional-error group" entry.ActionContract = name default: @@ -511,6 +513,7 @@ func declaredErrorCoverageCases(manifest errorCoverageManifest) []errorCoverageC } message += " to see commands." code := 3 + suffix := []string{private} switch path { case "help": message = "Unknown help topic. Run openai help to see commands." @@ -520,8 +523,16 @@ func declaredErrorCoverageCases(manifest errorCoverageManifest) []errorCoverageC // This action-bearing group owns positional input validation. message = "Tokenizer takes no positional input. Open the editor without arguments, or use tokenizer count --text TEXT." code = 1 + case "admin limits", "admin organization limits", + "projects limits", "admin projects limits", "admin organization projects limits": + message = "Limits does not accept positional arguments. Use --help to inspect its options." + code = 1 + if strings.HasSuffix(path, "projects limits") { + // Reach positional validation without masking it behind a missing project identifier. + suffix = []string{"--project-id=proj_synthetic", private} + } } - add("unknown-child", "", []string{private}, code, message) + add("unknown-child", "", suffix, code, message) } var required, requiredAfterInput []string aliasOwners := map[string]string{} From 9343cf2151c6fd10dac4a21c0b2376422977cb76 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 15:26:25 -0700 Subject: [PATCH 22/44] fix: preserve Limits terminal and recovery contracts --- cmd/openai/limits_workflow_unix_test.go | 182 ++++++++++++++++++++++++ cmd/openai/spend_error_recovery_test.go | 139 ++++++++++++++++++ internal/terminal/shell_command.go | 3 +- pkg/custom/cmdutil.go | 2 +- pkg/custom/limits_workspace.go | 9 +- pkg/custom/limits_workspace_test.go | 84 ++++++----- pkg/custom/local_errors.go | 7 + pkg/custom/spend_error_guidance.go | 44 +++++- pkg/custom/spend_error_guidance_test.go | 78 ++++++++++ 9 files changed, 503 insertions(+), 45 deletions(-) diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go index a11cb623..53408348 100644 --- a/cmd/openai/limits_workflow_unix_test.go +++ b/cmd/openai/limits_workflow_unix_test.go @@ -6,14 +6,196 @@ import ( "bytes" "context" "encoding/json" + "errors" + "fmt" "io" "net/http" + "os" "os/exec" + "slices" "strings" + "sync/atomic" + "syscall" "testing" "time" + + "github.com/openai/openai-cli/generated/commands" + "github.com/openai/openai-cli/internal/app" + "github.com/openai/openai-cli/internal/terminal" ) +type limitsBorrowedOutputProbe struct { + fail bool + writes atomic.Int32 + failed atomic.Bool +} + +func (w *limitsBorrowedOutputProbe) Write(data []byte) (int, error) { + w.writes.Add(1) + if w.fail && bytes.Contains(data, []byte("q: quit")) && !w.failed.Swap(true) { + return 0, errors.New("synthetic-private-borrowed-output") + } + return os.Stdout.Write(data) +} + +// Exercise production startup with the supported embedding writer installed. +func TestMainLimitsBorrowedWriterProcess(t *testing.T) { + mode := os.Getenv("OPENAI_CLI_LIMITS_BORROWED_PROCESS") + if mode == "" { + return + } + separator := slices.Index(os.Args, "--") + if separator < 0 { + t.Fatal("missing subprocess argument separator") + } + sink := &limitsBorrowedOutputProbe{fail: mode == "write-failure"} + borrowed := terminal.BorrowedWriter{Writer: sink, File: os.Stdout} + commands.Command.Writer, commands.Command.Reader = borrowed, os.Stdin + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + code := app.Run(ctx, os.Args[separator+1:]) + if sink.writes.Load() == 0 || sink.fail && !sink.failed.Load() { + t.Fatal("workspace bypassed the borrowed output sink") + } + if _, err := os.Stdout.Stat(); err != nil { + t.Fatal("workspace closed the caller's terminal", err) + } + if _, err := fmt.Fprintln(os.Stderr, "borrowed-terminal-verified"); err != nil { + t.Fatal(err) + } + os.Exit(code) +} + +func TestMainLimitsWorkflowBorrowedTerminalWriter(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("Python 3 is required for native Unix borrowed-terminal checks") + } + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name, mode string + project bool + exit int + }{ + {"organization", "quit", false, 0}, + {"project", "quit", true, 0}, + {"interrupt", "interrupt", false, 130}, + {"output failure", "write-failure", false, 1}, + } { + t.Run(test.name, func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, test.project, r) + }) + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + args := []string{"-I", "-B", "-c", limitsBorrowedTerminalPTY, binary, + "-test.run=^TestMainLimitsBorrowedWriterProcess$", "--", "openai"} + args = append(args, limitsWorkflowArgs(test.project, false)...) + child := exec.CommandContext(ctx, python, args...) + child.Env = spendRecoverySetEnvironment(spendRecoveryEnvironment(t, server.URL), "CI", "") + child.Env = append(child.Env, "PATH=/usr/bin:/bin", "TERM=xterm-256color", "GOMAXPROCS=2", + "OPENAI_CLI_LIMITS_BORROWED_PROCESS="+test.mode) + child.Cancel = func() error { return child.Process.Signal(syscall.SIGTERM) } + child.WaitDelay = 2 * time.Second + output, err := child.CombinedOutput() + if err != nil || ctx.Err() != nil { + t.Fatalf("borrowed terminal harness failed: error=%v context=%v output=%s", err, ctx.Err(), output) + } + var got struct { + Code int + Stdout, Stderr string + } + if err := json.Unmarshal(output, &got); err != nil { + t.Fatalf("invalid borrowed terminal capture: %v; %s", err, output) + } + if got.Code != test.exit || !strings.Contains(got.Stderr, "borrowed-terminal-verified\n") { + t.Fatalf("borrowed terminal changed exit or ownership: %+v", got) + } + enter, leave := strings.Index(got.Stdout, "\x1b[?1049h"), strings.LastIndex(got.Stdout, "\x1b[?1049l") + if enter < 0 || leave <= enter || strings.Contains(got.Stdout+got.Stderr, "panic:") || strings.Contains(got.Stderr, "synthetic-private") { + t.Fatalf("workspace did not restore safe terminal output: %+v", got) + } + if test.exit == 0 && (!strings.Contains(got.Stdout[leave:], "Limits workspace closed.") || got.Stderr != "borrowed-terminal-verified\n") { + t.Fatalf("workspace did not close normally after restoration: %+v", got) + } + for _, path := range []string{spendThresholdPath(test.project, "spend-limit"), spendThresholdPath(test.project, "spend-alerts"), "/organization/costs"} { + if requests.count(path) != 1 { + t.Errorf("workspace did not load %s exactly once", path) + } + } + }) + } +} + +// The private PTY follows the existing process-test cleanup pattern. Only the +// ready workspace receives a key; output failure ends without injected input. +const limitsBorrowedTerminalPTY = ` +import errno, fcntl, json, os, pty, selectors, signal, struct, subprocess, sys, termios, time +def interrupted(signum, frame): + raise TimeoutError("borrowed terminal harness interrupted") +signal.signal(signal.SIGTERM, interrupted) +master, slave = pty.openpty() +fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 24, 80, 0, 0)) +def session(): + os.setsid() + fcntl.ioctl(slave, termios.TIOCSCTTY, 0) +selector = selectors.DefaultSelector() +process = None +completed = False +output = [bytearray(), bytearray()] +try: + process = subprocess.Popen(sys.argv[1:], stdin=slave, stdout=slave, stderr=subprocess.PIPE, preexec_fn=session) + os.close(slave) + slave = -1 + for descriptor, index in [(master, 0), (process.stderr.fileno(), 1)]: + os.set_blocking(descriptor, False) + selector.register(descriptor, selectors.EVENT_READ, index) + sent = False + mode = os.environ["OPENAI_CLI_LIMITS_BORROWED_PROCESS"] + deadline = time.monotonic() + 10 + while selector.get_map() or process.poll() is None: + if time.monotonic() >= deadline: + raise TimeoutError("borrowed terminal command exceeded ten seconds") + for key, _ in selector.select(.1): + try: + data = os.read(key.fd, 65536) + except OSError as error: + if error.errno == errno.EAGAIN: + continue + if error.errno != errno.EIO: + raise + data = b"" + if not data: + selector.unregister(key.fd) + continue + output[key.data].extend(data) + if len(output[key.data]) > 2 << 20: + raise RuntimeError("borrowed terminal capture exceeded its allowance") + if not sent and mode in ("quit", "interrupt") and b"q: quit" in output[0]: + key = b"\x03" if mode == "interrupt" else b"q" + os.write(master, key) + sent = True + code = process.wait(timeout=2) + completed = True +finally: + if process is not None and not completed: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2) + if process is not None: + process.stderr.close() + selector.close() + os.close(master) + if slave >= 0: + os.close(slave) +print(json.dumps({"Code": code, "Stdout": output[0].decode(), "Stderr": output[1].decode()})) +` + func TestMainLimitsWorkflowPartialFailureSurvivesClosedPipe(t *testing.T) { for _, test := range []struct { name string diff --git a/cmd/openai/spend_error_recovery_test.go b/cmd/openai/spend_error_recovery_test.go index 069135a7..24f5cf2f 100644 --- a/cmd/openai/spend_error_recovery_test.go +++ b/cmd/openai/spend_error_recovery_test.go @@ -96,6 +96,145 @@ func followSpendRecoveryHelp(t *testing.T, env []string, diagnostic, label strin return mainDispatchResult{} } +func TestMainSpendErrorRecoveryResponseConversion(t *testing.T) { + for _, project := range []bool{false, true} { + for _, operation := range []struct{ resource, action string }{ + {"spend-limit", "update"}, {"spend-limit", "delete"}, + {"spend-alerts", "create"}, {"spend-alerts", "update"}, {"spend-alerts", "delete"}, + {"rate-limits", "update-rate-limit"}, + } { + if !project && operation.resource == "rate-limits" { + continue + } + t.Run(fmt.Sprintf("project=%t/%s/%s", project, operation.resource, operation.action), func(t *testing.T) { + var requests atomic.Int32 + path := spendThresholdPath(project, operation.resource) + if operation.resource == "spend-alerts" && operation.action != "create" { + path += "/alert_synthetic" + } else if operation.resource == "rate-limits" { + path += "/rl_synthetic" + } + method := http.MethodPost + if operation.action == "delete" { + method = http.MethodDelete + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + if r.Method != method || r.URL.Path != path { + t.Errorf("unexpected mutation request: %s %s", r.Method, r.URL.Path) + } + body, err := io.ReadAll(r.Body) + if err != nil || method == http.MethodDelete && len(body) != 0 { + t.Errorf("unexpected mutation body: %s; %v", body, err) + } + if method != http.MethodDelete { + var fields map[string]any + if err := json.Unmarshal(body, &fields); err != nil { + t.Errorf("invalid mutation body: %s; %v", body, err) + } else if operation.resource == "rate-limits" { + if len(fields) != 1 || fields["max_requests_per_1_minute"] != float64(300) { + t.Errorf("rate-limit fields changed: %s", body) + } + } else if fields["threshold_amount"] != float64(10000) || fields["currency"] != "USD" || fields["interval"] != "month" { + t.Errorf("mutation fields changed: %s", body) + } + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"synthetic-private-response": broken}`) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + argv := append([]string{"openai"}, spendThresholdArgs(project, operation.resource, operation.action)...) + if operation.resource == "rate-limits" { + argv = append(argv, "--rate-limit-id", "rl_synthetic", "--max-requests-per-1-minute", "300") + } else if operation.action != "delete" { + argv = append(argv, "--currency", "USD", "--interval", "month", "--threshold-amount", "10000") + if operation.resource == "spend-alerts" { + argv = append(argv, "--notification-channel", `{"type":"email","recipients":["synthetic@example.invalid"]}`) + } + } + argv = append(argv, "--format", "yaml") + got := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "text")...) + checkSpendRecoveryFailure(t, got, "The API may have received the change.", "Check its current state before repeating", "Keep the original API address") + if operation.resource == "rate-limits" { + checkSpendRecoveryFailure(t, got, "rate-limit change", "help admin organization projects rate-limits list-rate-limits") + } + followSpendRecoveryHelp(t, env, got.stderr, "Read options") + if requests.Load() != 1 { + t.Fatal("response conversion or read help repeated a mutation", requests.Load()) + } + if operation.resource != "rate-limits" && (project || operation.resource != "spend-alerts" || operation.action != "create") { + return + } + const baseline = "Could not decode JSON: invalid JSON syntax." + for _, format := range []string{"json", "jsonl", "text"} { + args := append(slices.Clone(argv), "--format-error", format) + if format == "text" { + args = append(args, "--transform-error", "message") + } + before := requests.Load() + machine := runMainDispatchWithEnv(t, "bash", env, args...) + checkSpendRecoveryFailure(t, machine, baseline) + if requests.Load() != before+1 { + t.Fatal("machine error formatting retried the mutation") + } + if format == "text" { + if machine.stderr != baseline+"\n" { + t.Fatal("error extraction changed", machine.stderr) + } + } else { + var value map[string]any + if json.Unmarshal([]byte(machine.stderr), &value) != nil || len(value) != 1 || value["message"] != baseline { + t.Fatal("machine error contract changed", machine.stderr) + } + } + } + }) + } + } +} + +func TestMainSpendErrorRecoveryInputDecodeDoesNotClaimMutation(t *testing.T) { + for _, test := range []struct{ name, input string }{ + {"syntax", `{"synthetic-private-input": broken}`}, + {"type", `[{"synthetic-private-input":"value"}]`}, + } { + t.Run(test.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + argv := append([]string{"openai"}, spendThresholdArgs(false, "spend-alerts", "create")...) + argv = append(argv, "--currency", "USD", "--interval", "month", "--threshold-amount", "10000", + "--notification-channel", test.input, "--format", "yaml") + human := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "text")...) + checkSpendRecoveryFailure(t, human) + if human.stderr == "" || strings.Contains(human.stderr, "may have received") || strings.Contains(human.stderr, "before repeating") { + t.Fatal("input decoding incorrectly claimed an attempted mutation", human.stderr) + } + machine := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "json")...) + checkSpendRecoveryFailure(t, machine) + var value map[string]any + if json.Unmarshal([]byte(machine.stderr), &value) != nil || len(value) != 1 { + t.Fatal("invalid machine input error", machine.stderr) + } + message, ok := value["message"].(string) + if !ok || message == "" { + t.Fatal("machine input error omitted its message", machine.stderr) + } + extracted := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "text", "--transform-error", "message")...) + checkSpendRecoveryFailure(t, extracted) + if extracted.stderr != message+"\n" || requests.Load() != 0 { + t.Fatal("input error changed extraction or sent an HTTP request", extracted.stderr, requests.Load()) + } + }) + } +} + func TestMainSpendErrorRecoveryAdminSetup(t *testing.T) { for _, args := range [][]string{ {"admin", "organization", "spend-limit", "retrieve"}, diff --git a/internal/terminal/shell_command.go b/internal/terminal/shell_command.go index ee4a3119..b8c19ff5 100644 --- a/internal/terminal/shell_command.go +++ b/internal/terminal/shell_command.go @@ -26,8 +26,7 @@ func FormatCommand(args []string, shell string) string { if quote == nil { return "" } - words := make([]string, 0, len(args)+1) - words = append(words, "openai") + words := []string{"openai"} for _, arg := range args { words = append(words, quote(arg)) } diff --git a/pkg/custom/cmdutil.go b/pkg/custom/cmdutil.go index 952af4c0..c6aa01e9 100644 --- a/pkg/custom/cmdutil.go +++ b/pkg/custom/cmdutil.go @@ -388,7 +388,7 @@ func showJSON(res gjson.Result, opts ShowJSONOpts, selectTransformer transformer opts.Transform = "" formatted, err := formatJSON(res, opts) if err != nil { - return err + return spendMutationResponseFailure(err, opts) } _, err = (clioutput.Writer{Context: opts.Context, Sink: opts.Stdout}).Write(formatted) return err diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index 164fe9d2..0fc35b13 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -75,7 +75,8 @@ func (e *limitsInterrupt) Unwrap() error { return e.error } // limitsWorkspaceOutput records failures that the terminal renderer may ignore. // Its output context stays active so shutdown can still restore terminal modes. type limitsWorkspaceOutput struct { - clioutput.TerminalWriter + clioutput.Writer + term.File stop context.CancelFunc mu sync.Mutex err error @@ -107,7 +108,7 @@ func (w *limitsWorkspaceOutput) Err() error { } func runLimitsWorkspace(ctx context.Context, cmd *cli.Command, backend limitsBackend, opts ShowJSONOpts) error { - file := opts.Stdout.(*os.File) + file := opts.Stdout.(term.File) width, height, err := term.GetSize(file.Fd()) if err != nil { return err @@ -125,8 +126,8 @@ func runLimitsWorkspace(ctx context.Context, cmd *cli.Command, backend limitsBac if err != nil || strings.HasPrefix(m.invocation, "go run ") { m.omitPreview = true } - m.output = &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ - Writer: clioutput.Writer{Context: context.Background(), Sink: opts.Stdout}, File: file}, stop: stop} + m.output = &limitsWorkspaceOutput{ + Writer: clioutput.Writer{Context: context.Background(), Sink: opts.Stdout}, File: file, stop: stop} options := []tea.ProgramOption{tea.WithInput(cmd.Root().Reader), tea.WithOutput(m.output), tea.WithContext(programContext), tea.WithWindowSize(width, height)} if !m.color { options = append(options, tea.WithColorProfile(colorprofile.NoTTY)) diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index fb5d50ec..d91f6e45 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -23,6 +23,7 @@ import ( tea "charm.land/bubbletea/v2" "github.com/charmbracelet/x/ansi" clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-cli/internal/terminal" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/option" "github.com/tidwall/gjson" @@ -1256,9 +1257,9 @@ func TestLimitsWorkspaceOutputPreservesFailuresAndCleanup(t *testing.T) { return 0, later } }) - out := &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ + out := &limitsWorkspaceOutput{ Writer: clioutput.Writer{Context: context.Background(), Sink: sink}, - }, stop: stop} + stop: stop} write := func(value string) (int, error) { return out.Write([]byte(value)) } if method == "WriteString" { write = func(value string) (int, error) { return io.WriteString(out, value) } @@ -1291,13 +1292,13 @@ func TestLimitsWorkspaceOutputPreservesCancellation(t *testing.T) { workspaceContext, stop := context.WithCancel(t.Context()) defer stop() calls := 0 - out := &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ + out := &limitsWorkspaceOutput{ Writer: clioutput.Writer{Context: outputContext, Sink: limitsWorkspaceWriteFunc(func(data []byte) (int, error) { calls++ cancelOutput() return len(data), nil })}, - }, stop: stop} + stop: stop} wantBytes, wantCalls := len("result"), 1 if when == "before write" { cancelOutput() @@ -1316,39 +1317,48 @@ func TestLimitsWorkspaceOutputPreservesCancellation(t *testing.T) { } func TestLimitsWorkspaceOutputPreservesTerminalFile(t *testing.T) { - file, err := os.CreateTemp(t.TempDir(), "terminal-output") - if err != nil { - t.Fatal(err) - } - t.Cleanup(func() { file.Close() }) - if _, err := file.WriteString("input"); err != nil { - t.Fatal(err) - } - if _, err := file.Seek(0, io.SeekStart); err != nil { - t.Fatal(err) - } - var sink bytes.Buffer - out := &limitsWorkspaceOutput{TerminalWriter: clioutput.TerminalWriter{ - Writer: clioutput.Writer{Context: t.Context(), Sink: &sink}, File: file, - }, stop: func() { t.Fatal("successful output canceled the workspace") }} - if out.Fd() != file.Fd() { - t.Fatal("output wrapper changed the descriptor used for terminal detection") - } - if _, err := io.WriteString(out, "output"); err != nil || sink.String() != "output" { - t.Fatal("output wrapper bypassed its guarded sink", err) - } - input := make([]byte, len("input")) - if _, err := io.ReadFull(out, input); err != nil || string(input) != "input" { - t.Fatal("output wrapper changed terminal reads", err) - } - if out.Err() != nil { - t.Fatal("successful writes recorded an error", out.Err()) - } - if err := out.Close(); err != nil { - t.Fatal(err) - } - if _, err := file.Stat(); !errors.Is(err, os.ErrClosed) { - t.Fatal("output wrapper changed terminal file ownership", err) + for _, borrowed := range []bool{false, true} { + t.Run(fmt.Sprintf("borrowed=%t", borrowed), func(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "terminal-output") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { file.Close() }) + if _, err := file.WriteString("input"); err != nil { + t.Fatal(err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + var sink bytes.Buffer + out := &limitsWorkspaceOutput{ + Writer: clioutput.Writer{Context: t.Context(), Sink: &sink}, File: file, + stop: func() { t.Fatal("successful output canceled the workspace") }} + if borrowed { + writer := terminal.BorrowedWriter{Writer: &sink, File: file} + out.Writer.Sink, out.File = writer, writer + } + if out.Fd() != file.Fd() { + t.Fatal("output wrapper changed the descriptor used for terminal detection") + } + if _, err := io.WriteString(out, "output"); err != nil || sink.String() != "output" { + t.Fatal("output wrapper bypassed its guarded sink", err) + } + input := make([]byte, len("input")) + if _, err := io.ReadFull(out, input); err != nil || string(input) != "input" { + t.Fatal("output wrapper changed terminal reads", err) + } + if out.Err() != nil { + t.Fatal("successful writes recorded an error", out.Err()) + } + if err := out.Close(); err != nil { + t.Fatal(err) + } + _, err = file.Stat() + if borrowed && err != nil || !borrowed && !errors.Is(err, os.ErrClosed) { + t.Fatal("output wrapper changed terminal file ownership", err) + } + }) } } diff --git a/pkg/custom/local_errors.go b/pkg/custom/local_errors.go index 86642642..e591abe1 100644 --- a/pkg/custom/local_errors.go +++ b/pkg/custom/local_errors.go @@ -63,6 +63,11 @@ func localErrorMessageForDisplay(root *cli.Command, failure error, human bool) s var batchFailure *batchcommands.WorkflowError var limitsFailure *limitsWorkflowFailure var finiteListFailure *clioutput.FiniteJSONListError + var spendResponseFailure *spendMutationResponseError + responseRecovery := "" + if human && errors.As(failure, &spendResponseFailure) { + responseRecovery = readableSpendMutationFailure(root, command) + } switch { case errors.As(failure, &limitsFailure): return limitsFailure.Error() @@ -83,6 +88,8 @@ func localErrorMessageForDisplay(root *cli.Command, failure error, human bool) s return utilityFailure.Error() case errors.Is(failure, context.DeadlineExceeded), errors.As(failure, &timeout) && timeout.Timeout(): return "The request timed out. The API may have received it; check its status before repeating it." + case responseRecovery != "": + return responseRecovery case errors.As(failure, &modelsListFailure): return modelsListFailure.Error() case errors.As(failure, &finiteListFailure): diff --git a/pkg/custom/spend_error_guidance.go b/pkg/custom/spend_error_guidance.go index 3838719b..7a5d371b 100644 --- a/pkg/custom/spend_error_guidance.go +++ b/pkg/custom/spend_error_guidance.go @@ -1,7 +1,9 @@ package custom import ( + "encoding/json" "errors" + "io" "github.com/openai/openai-cli/internal/clihelp" localcommands "github.com/openai/openai-cli/internal/commands" @@ -18,6 +20,40 @@ func commandErrorContext(root *cli.Command, failure error) *cli.Command { const spendRecoveryContext = "Keep the original API address, authentication, organization, and project options for any check." +// Response conversion happens after the mutation returned successfully. Keep +// that provenance separate from JSON errors raised while preparing a request. +type spendMutationResponseError struct{ cause error } + +func (e *spendMutationResponseError) Error() string { return e.cause.Error() } +func (e *spendMutationResponseError) Unwrap() error { return e.cause } + +func spendMutationResponseFailure(err error, opts ShowJSONOpts) error { + if opts.OutputKind != OutputResponse { + return err + } + switch opts.Operation { + case "(resource) admin.organization.spend_limit > (method) update", + "(resource) admin.organization.spend_limit > (method) delete", + "(resource) admin.organization.projects.spend_limit > (method) update", + "(resource) admin.organization.projects.spend_limit > (method) delete", + "(resource) admin.organization.spend_alerts > (method) create", + "(resource) admin.organization.spend_alerts > (method) update", + "(resource) admin.organization.spend_alerts > (method) delete", + "(resource) admin.organization.projects.spend_alerts > (method) create", + "(resource) admin.organization.projects.spend_alerts > (method) update", + "(resource) admin.organization.projects.spend_alerts > (method) delete", + "(resource) admin.organization.projects.rate_limits > (method) update_rate_limit": + default: + return err + } + var syntaxError *json.SyntaxError + var typeError *json.UnmarshalTypeError + if errors.As(err, &syntaxError) || errors.As(err, &typeError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return &spendMutationResponseError{cause: err} + } + return err +} + // A status alone cannot establish configuration at an arbitrary API address. // Recovery uses declared command metadata, never request URLs or response prose. func readableSpendNotFoundMessage(root *cli.Command, failure error) string { @@ -68,6 +104,7 @@ func readableOrganizationSpendLimitMessage(root *cli.Command) string { func readableSpendMutationFailure(root, command *cli.Command) string { path := "" + change := "spend change" switch localcommands.ResourceName(command) { case "admin:organization:spend-limit": if command.Name == "update" || command.Name == "delete" { @@ -88,11 +125,16 @@ func readableSpendMutationFailure(root, command *cli.Command) string { case "update", "delete": path = prefix + "retrieve" } + case "admin:organization:projects:rate-limits": + if command.Name == "update-rate-limit" { + path = "admin organization projects rate-limits list-rate-limits" + change = "rate-limit change" + } } if path == "" { return "" } - return "The API response did not confirm the spend change.\n" + + return "The API response did not confirm the " + change + ".\n" + "The API may have received the change.\n" + "Check its current state before repeating the command.\n" + "Read options: " + clihelp.DisplayInvocation(root) + " help " + path + "\n" + diff --git a/pkg/custom/spend_error_guidance_test.go b/pkg/custom/spend_error_guidance_test.go index 13b0a74c..15f5fc98 100644 --- a/pkg/custom/spend_error_guidance_test.go +++ b/pkg/custom/spend_error_guidance_test.go @@ -10,11 +10,14 @@ import ( "net/http" "net/url" "strings" + "syscall" "testing" "github.com/openai/openai-cli/internal/commands/admin" + clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-go/v3" "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" "github.com/urfave/cli/v3" ) @@ -24,6 +27,61 @@ func spendGuidanceTestCommand(resource, operation string) *cli.Command { }} } +func TestSpendErrorGuidanceResponseConversionProvenance(t *testing.T) { + command := spendGuidanceTestCommand("admin:organization:spend-alerts", "create") + options := ShowJSONOpts{Context: t.Context(), Format: "yaml", ExplicitFormat: true, + Operation: "(resource) admin.organization.spend_alerts > (method) create", + OutputKind: OutputResponse, Stdout: io.Discard, Stderr: io.Discard} + failure := ShowJSON(gjson.Parse(`{"synthetic-private-response": broken}`), options) + var marker *spendMutationResponseError + var syntaxFailure *json.SyntaxError + require.ErrorAs(t, failure, &marker) + require.ErrorAs(t, failure, &syntaxFailure) + require.ErrorIs(t, failure, syntaxFailure) + require.Equal(t, syntaxFailure.Error(), failure.Error()) + root := readableErrorTestCommand(t) + contextual := withCommandError(command, failure) + require.Contains(t, readableLocalErrorMessage(root, contextual), "The API may have received the change.") + require.Equal(t, "Could not decode JSON: invalid JSON syntax.", localErrorMessage(root, contextual)) + for _, cause := range []error{context.Canceled, context.DeadlineExceeded} { + joined := withCommandError(command, errors.Join(failure, cause)) + require.ErrorIs(t, joined, syntaxFailure) + require.ErrorIs(t, joined, cause) + require.Equal(t, localErrorMessage(root, joined), readableLocalErrorMessage(root, joined)) + } + writeFailure := &clioutput.WriteError{Err: syscall.EPIPE} + joined := errors.Join(failure, writeFailure) + require.ErrorIs(t, joined, syntaxFailure) + require.ErrorIs(t, joined, syscall.EPIPE) + require.False(t, clioutput.IsBrokenPipe(joined), "a closed pipe must not discard a response failure") + inputFailure := withCommandError(command, syntaxFailure) + require.Equal(t, localErrorMessage(root, inputFailure), readableLocalErrorMessage(root, inputFailure)) + for _, kind := range []string{"read response", "error presentation"} { + t.Run(kind, func(t *testing.T) { + opts := options + if kind == "read response" { + opts.Operation = "(resource) admin.organization.spend_alerts > (method) retrieve" + } else { + opts.OutputKind = OutputUnspecified + } + err := ShowJSON(gjson.Parse(`{"synthetic-private-response": broken}`), opts) + require.ErrorAs(t, err, &syntaxFailure) + require.False(t, errors.As(err, &marker)) + }) + } + options.Stdout = failOutputWriter{err: syscall.EPIPE} + failure = ShowJSON(gjson.Parse(`{"id":"alert_synthetic"}`), options) + require.ErrorIs(t, failure, syscall.EPIPE) + require.True(t, clioutput.IsBrokenPipe(failure)) + require.False(t, errors.As(failure, &marker), "sink failures must retain their output classification") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + options.Context = ctx + failure = ShowJSON(gjson.Parse(`{"synthetic-private-response": broken}`), options) + require.ErrorIs(t, failure, context.Canceled) + require.False(t, errors.As(failure, &marker), "pre-conversion cancellation must retain its classification") +} + func TestSpendErrorGuidanceAdminMetadata(t *testing.T) { root := readableErrorTestCommand(t) root.Metadata["help-invocation"] = "'/tmp/synthetic app/openai'" @@ -125,9 +183,29 @@ func TestSpendErrorGuidanceMutationReadRoutes(t *testing.T) { {"admin:organization:spend-alerts", "create", "admin organization spend-alerts list"}, {"admin:organization:projects:spend-alerts", "create", "admin organization projects spend-alerts list"}, {"admin:organization:projects:spend-alerts", "update", "admin organization projects spend-alerts retrieve"}, + {"admin:organization:projects:rate-limits", "update-rate-limit", "admin organization projects rate-limits list-rate-limits"}, } { require.Contains(t, readableSpendMutationFailure(root, spendGuidanceTestCommand(test.resource, test.operation)), "help "+test.read) } + network := &url.Error{Op: "Post", URL: "https://synthetic-private.invalid", Err: io.EOF} + for _, operation := range []string{"update-rate-limit", "list-rate-limits"} { + failure := withCommandError(spendGuidanceTestCommand("admin:organization:projects:rate-limits", operation), network) + var human, machine bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &human)) + baseline := localErrorMessage(root, failure) + if operation == "update-rate-limit" { + require.Contains(t, human.String(), "The API response did not confirm the rate-limit change.") + require.Contains(t, human.String(), "help admin organization projects rate-limits list-rate-limits") + require.Contains(t, human.String(), spendRecoveryContext) + } else { + require.Equal(t, baseline+"\n", human.String()) + } + require.NotContains(t, human.String(), "synthetic-private") + require.NoError(t, ShowCommandError(readableErrorTestCommand(t, "--format-error", "json"), failure, &machine)) + var value map[string]any + require.NoError(t, json.Unmarshal(machine.Bytes(), &value)) + require.Equal(t, map[string]any{"message": baseline}, value) + } } func TestSpendErrorGuidanceLimitCodesStayScoped(t *testing.T) { From 2d8b411d08763da9211234d11668eacaacc6d2b8 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 15:30:34 -0700 Subject: [PATCH 23/44] test: reject Limits watchdog fallback --- cmd/openai/limits_workflow_unix_test.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go index 53408348..63a12239 100644 --- a/cmd/openai/limits_workflow_unix_test.go +++ b/cmd/openai/limits_workflow_unix_test.go @@ -54,6 +54,9 @@ func TestMainLimitsBorrowedWriterProcess(t *testing.T) { ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) defer cancel() code := app.Run(ctx, os.Args[separator+1:]) + if err := ctx.Err(); err != nil { + t.Fatal("workspace required fallback cancellation", err) + } if sink.writes.Load() == 0 || sink.fail && !sink.failed.Load() { t.Fatal("workspace bypassed the borrowed output sink") } From 7cd3066f7c738faa12807e8c256fa3db96666f41 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 15:56:55 -0700 Subject: [PATCH 24/44] test: correct Limits failure probes --- cmd/openai/limits_workflow_unix_test.go | 14 +++++++++++--- cmd/openai/spend_error_recovery_test.go | 12 ++++++------ 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go index 63a12239..2d6e41f1 100644 --- a/cmd/openai/limits_workflow_unix_test.go +++ b/cmd/openai/limits_workflow_unix_test.go @@ -27,15 +27,20 @@ import ( type limitsBorrowedOutputProbe struct { fail bool writes atomic.Int32 + ready atomic.Bool failed atomic.Bool } func (w *limitsBorrowedOutputProbe) Write(data []byte) (int, error) { w.writes.Add(1) - if w.fail && bytes.Contains(data, []byte("q: quit")) && !w.failed.Swap(true) { + if w.fail && w.ready.Load() && !w.failed.Swap(true) { return 0, errors.New("synthetic-private-borrowed-output") } - return os.Stdout.Write(data) + n, err := os.Stdout.Write(data) + if err == nil && n == len(data) && bytes.Contains(data, []byte("q: quit")) { + w.ready.Store(true) + } + return n, err } // Exercise production startup with the supported embedding writer installed. @@ -134,7 +139,7 @@ func TestMainLimitsWorkflowBorrowedTerminalWriter(t *testing.T) { } // The private PTY follows the existing process-test cleanup pattern. Only the -// ready workspace receives a key; output failure ends without injected input. +// ready workspace receives a key; output failure ends after a redraw signal. const limitsBorrowedTerminalPTY = ` import errno, fcntl, json, os, pty, selectors, signal, struct, subprocess, sys, termios, time def interrupted(signum, frame): @@ -181,6 +186,9 @@ try: key = b"\x03" if mode == "interrupt" else b"q" os.write(master, key) sent = True + elif not sent and mode == "write-failure" and b"q: quit" in output[0]: + os.kill(process.pid, signal.SIGWINCH) + sent = True code = process.wait(timeout=2) completed = True finally: diff --git a/cmd/openai/spend_error_recovery_test.go b/cmd/openai/spend_error_recovery_test.go index 24f5cf2f..f2f0ec1a 100644 --- a/cmd/openai/spend_error_recovery_test.go +++ b/cmd/openai/spend_error_recovery_test.go @@ -197,7 +197,7 @@ func TestMainSpendErrorRecoveryResponseConversion(t *testing.T) { func TestMainSpendErrorRecoveryInputDecodeDoesNotClaimMutation(t *testing.T) { for _, test := range []struct{ name, input string }{ - {"syntax", `{"synthetic-private-input": broken}`}, + {"syntax", `[synthetic-private-input not closed`}, {"type", `[{"synthetic-private-input":"value"}]`}, } { t.Run(test.name, func(t *testing.T) { @@ -208,15 +208,15 @@ func TestMainSpendErrorRecoveryInputDecodeDoesNotClaimMutation(t *testing.T) { })) defer server.Close() env := spendRecoveryEnvironment(t, server.URL) - argv := append([]string{"openai"}, spendThresholdArgs(false, "spend-alerts", "create")...) + argv := spendThresholdArgs(false, "spend-alerts", "create") argv = append(argv, "--currency", "USD", "--interval", "month", "--threshold-amount", "10000", - "--notification-channel", test.input, "--format", "yaml") - human := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "text")...) + "--notification-channel", test.input) + human := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format", "yaml", "--format-error", "text"}, argv...)...) checkSpendRecoveryFailure(t, human) if human.stderr == "" || strings.Contains(human.stderr, "may have received") || strings.Contains(human.stderr, "before repeating") { t.Fatal("input decoding incorrectly claimed an attempted mutation", human.stderr) } - machine := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "json")...) + machine := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format", "yaml", "--format-error", "json"}, argv...)...) checkSpendRecoveryFailure(t, machine) var value map[string]any if json.Unmarshal([]byte(machine.stderr), &value) != nil || len(value) != 1 { @@ -226,7 +226,7 @@ func TestMainSpendErrorRecoveryInputDecodeDoesNotClaimMutation(t *testing.T) { if !ok || message == "" { t.Fatal("machine input error omitted its message", machine.stderr) } - extracted := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "text", "--transform-error", "message")...) + extracted := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format", "yaml", "--format-error", "text", "--transform-error", "message"}, argv...)...) checkSpendRecoveryFailure(t, extracted) if extracted.stderr != message+"\n" || requests.Load() != 0 { t.Fatal("input error changed extraction or sent an HTTP request", extracted.stderr, requests.Load()) From ed7fbd9e6d90288b5d3fb7ca1f7df92fed8ea42e Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 16:03:44 -0700 Subject: [PATCH 25/44] fix: explain terminal output failures safely --- cmd/openai/limits_workflow_unix_test.go | 9 ++-- pkg/custom/local_errors.go | 4 ++ pkg/custom/readable_errors_test.go | 62 +++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 3 deletions(-) diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go index 2d6e41f1..4e03e358 100644 --- a/cmd/openai/limits_workflow_unix_test.go +++ b/cmd/openai/limits_workflow_unix_test.go @@ -37,7 +37,7 @@ func (w *limitsBorrowedOutputProbe) Write(data []byte) (int, error) { return 0, errors.New("synthetic-private-borrowed-output") } n, err := os.Stdout.Write(data) - if err == nil && n == len(data) && bytes.Contains(data, []byte("q: quit")) { + if err == nil && n == len(data) && bytes.Contains(data, []byte("Website controls and source errors: ?")) { w.ready.Store(true) } return n, err @@ -129,6 +129,9 @@ func TestMainLimitsWorkflowBorrowedTerminalWriter(t *testing.T) { if test.exit == 0 && (!strings.Contains(got.Stdout[leave:], "Limits workspace closed.") || got.Stderr != "borrowed-terminal-verified\n") { t.Fatalf("workspace did not close normally after restoration: %+v", got) } + if test.mode == "write-failure" && got.Stderr != "Could not write command output. Check your terminal, output file, or receiving command.\nborrowed-terminal-verified\n" { + t.Fatalf("workspace output failure omitted safe recovery: %q", got.Stderr) + } for _, path := range []string{spendThresholdPath(test.project, "spend-limit"), spendThresholdPath(test.project, "spend-alerts"), "/organization/costs"} { if requests.count(path) != 1 { t.Errorf("workspace did not load %s exactly once", path) @@ -182,11 +185,11 @@ try: output[key.data].extend(data) if len(output[key.data]) > 2 << 20: raise RuntimeError("borrowed terminal capture exceeded its allowance") - if not sent and mode in ("quit", "interrupt") and b"q: quit" in output[0]: + if not sent and mode in ("quit", "interrupt") and b"Website controls and source errors: ?" in output[0]: key = b"\x03" if mode == "interrupt" else b"q" os.write(master, key) sent = True - elif not sent and mode == "write-failure" and b"q: quit" in output[0]: + elif not sent and mode == "write-failure" and b"Website controls and source errors: ?" in output[0]: os.kill(process.pid, signal.SIGWINCH) sent = True code = process.wait(timeout=2) diff --git a/pkg/custom/local_errors.go b/pkg/custom/local_errors.go index e591abe1..ea070819 100644 --- a/pkg/custom/local_errors.go +++ b/pkg/custom/local_errors.go @@ -161,6 +161,10 @@ func localErrorMessageForDisplay(root *cli.Command, failure error, human bool) s if errors.As(failure, &networkError) || errors.As(failure, &connectionError) { return "Could not connect to the API. Check your connection, proxy, and --base-url setting." } + var writeFailure *clioutput.WriteError + if human && errors.As(failure, &writeFailure) { + return "Could not write command output. Check your terminal, output file, or receiving command." + } // The usage buffer is deliberately not a fallback: it repeats raw values. return "The command could not be completed. Check your arguments with --help." } diff --git a/pkg/custom/readable_errors_test.go b/pkg/custom/readable_errors_test.go index 5ce6d788..65f8f2e9 100644 --- a/pkg/custom/readable_errors_test.go +++ b/pkg/custom/readable_errors_test.go @@ -10,9 +10,11 @@ import ( "net" "net/http" "net/url" + "os" "strings" "testing" + clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-cli/internal/requestflag" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/packages/ssestream" @@ -190,6 +192,65 @@ func TestCommandErrorStructuredLocalFallbackDoesNotEchoDiagnostic(t *testing.T) } } +func TestCommandErrorOutputFailureHumanGuidance(t *testing.T) { + const humanMessage = "Could not write command output. Check your terminal, output file, or receiving command." + const machineMessage = "The command could not be completed. Check your arguments with --help." + cause := errors.New("synthetic-private-prompt https://synthetic.invalid/?token=synthetic-secret \x1b]52;c;synthetic-secret\a") + writer := clioutput.Writer{Context: t.Context(), Sink: errorSink{cause}} + _, failure := writer.Write([]byte("synthetic output")) + var writeFailure *clioutput.WriteError + require.ErrorAs(t, failure, &writeFailure) + for _, test := range []struct { + name string + flags []string + want string + }{ + {"human", nil, humanMessage}, + {"quiet", []string{"--quiet"}, humanMessage}, + {"JSON", []string{"--format-error", "json"}, machineMessage}, + {"extraction", []string{"--format-error", "text", "--transform-error", "message"}, machineMessage}, + } { + t.Run(test.name, func(t *testing.T) { + var out bytes.Buffer + require.NoError(t, ShowCommandError(readableErrorTestCommand(t, test.flags...), failure, &out)) + if test.name == "JSON" { + var value map[string]string + require.NoError(t, json.Unmarshal(out.Bytes(), &value)) + require.Equal(t, map[string]string{"message": test.want}, value) + } else { + require.Equal(t, test.want+"\n", out.String()) + } + assertReadableErrorContainsNoPrivateDetails(t, out.String()) + require.ErrorIs(t, failure, cause) + }) + } + var value any + syntaxFailure := json.Unmarshal([]byte(`{"synthetic-private-prompt":`), &value) + require.Error(t, syntaxFailure) + for _, test := range []struct { + name string + cause error + }{ + {"cancellation", context.Canceled}, + {"deadline", context.DeadlineExceeded}, + {"syntax", syntaxFailure}, + {"file", &os.PathError{Op: "write", Path: "synthetic-private-output", Err: os.ErrPermission}}, + {"API", &openai.Error{StatusCode: http.StatusForbidden}}, + } { + t.Run("precedence/"+test.name, func(t *testing.T) { + root := readableErrorTestCommand(t) + var baseline, out bytes.Buffer + require.NoError(t, ShowCommandError(root, test.cause, &baseline)) + joined := errors.Join(failure, test.cause) + require.NoError(t, ShowCommandError(root, joined, &out)) + require.Equal(t, baseline.String(), out.String()) + assertReadableErrorContainsNoPrivateDetails(t, out.String()) + require.ErrorIs(t, joined, cause) + require.ErrorIs(t, joined, test.cause) + }) + } +} + func TestCommandErrorInvalidFormatStillPresentsSafely(t *testing.T) { root := readableErrorTestCommand(t, "--format-error", "synthetic-secret") var out bytes.Buffer @@ -209,6 +270,7 @@ func readableErrorTestCommand(t *testing.T, args ...string) *cli.Command { &cli.StringFlag{Name: "transform-error"}, &cli.StringFlag{Name: "transform"}, &cli.BoolFlag{Name: "raw-output"}, + &cli.BoolFlag{Name: "quiet"}, }, Action: func(context.Context, *cli.Command) error { return nil }, } From 35d74ad279b2f91b637234cb0099d5f6f7d41442 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 16:16:26 -0700 Subject: [PATCH 26/44] test: keep Limits shell recorder helpers local --- pkg/custom/limits_workspace_test.go | 67 ++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 2 deletions(-) diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index d91f6e45..c5b36008 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -10,6 +10,7 @@ import ( "math" "net/http" "os" + "os/exec" "path/filepath" "reflect" "runtime" @@ -26,6 +27,7 @@ import ( "github.com/openai/openai-cli/internal/terminal" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/option" + "github.com/stretchr/testify/require" "github.com/tidwall/gjson" ) @@ -473,13 +475,74 @@ func TestLimitsWorkspacePrintedPowerShellCommandRetainsCallOperator(t *testing.T } } +func limitsPreviewExecutable(t *testing.T, name string) (string, string) { + t.Helper() + directory := filepath.Join(t.TempDir(), "CLI preview's directory") + require.NoError(t, os.Mkdir(directory, 0700)) + path := filepath.Join(directory, name) + // This executable records shell arguments and stdin. It cannot call the API. + const script = `#!/bin/sh +set -eu +printf '%s\000' "$@" > "$LIMITS_PREVIEW_ARGS" +printf '%s' "${OPENAI_UNTRUSTED_STDIN-unset}" > "$LIMITS_PREVIEW_PROTECTED" +/bin/cat > "$LIMITS_PREVIEW_STDIN" +` + require.NoError(t, os.WriteFile(path, []byte(script), 0700)) + return path, directory +} + +type limitsPreviewCapture struct { + args []string + stdin []byte + protected string +} + +func executeLimitsPreview(t *testing.T, shell, command, pathDirectory string) limitsPreviewCapture { + t.Helper() + path := "/bin/" + shell + if _, err := os.Stat(path); err != nil { + if strings.Contains(","+os.Getenv("OPENAI_CLI_REQUIRE_NATIVE_SHELLS")+",", ","+shell+",") { + t.Fatalf("required native shell %s is unavailable", shell) + } + t.Skipf("native %s is unavailable", shell) + } + directory := t.TempDir() + argsPath, stdinPath, protectedPath := filepath.Join(directory, "argv"), filepath.Join(directory, "stdin"), filepath.Join(directory, "protected") + args := []string{"--noprofile", "--norc", "-c", command} + if shell == "zsh" { + args = []string{"-f", "-c", command} + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + process := exec.CommandContext(ctx, path, args...) + process.WaitDelay = 250 * time.Millisecond + process.Dir = t.TempDir() + process.Env = []string{"PATH=" + pathDirectory + ":/usr/bin:/bin", "LANG=en_US.UTF-8", + "LIMITS_PREVIEW_ARGS=" + argsPath, "LIMITS_PREVIEW_STDIN=" + stdinPath, + "LIMITS_PREVIEW_PROTECTED=" + protectedPath} + var stdout, stderr bytes.Buffer + process.Stdout, process.Stderr = &stdout, &stderr + require.NoError(t, process.Run(), "shell=%s stderr=%q", shell, stderr.String()) + require.NoError(t, ctx.Err()) + require.Empty(t, stdout.String()) + require.Empty(t, stderr.String()) + rawArgs, err := os.ReadFile(argsPath) + require.NoError(t, err) + require.True(t, bytes.HasSuffix(rawArgs, []byte{0})) + input, err := os.ReadFile(stdinPath) + require.NoError(t, err) + protected, err := os.ReadFile(protectedPath) + require.NoError(t, err) + return limitsPreviewCapture{strings.Split(string(rawArgs[:len(rawArgs)-1]), "\x00"), input, string(protected)} +} + func TestLimitsWorkspacePrintedCommandNativeShellPreservesArguments(t *testing.T) { if runtime.GOOS == "windows" { t.Skip("Bash/Zsh preview execution uses a POSIX recorder; native Windows requires separate coverage") } for _, shell := range []string{"bash", "zsh"} { t.Run(shell, func(t *testing.T) { - binary, directory := audioPreviewExecutable(t, strings.Repeat("long ", 20)+"limits'\nrecorder") + binary, directory := limitsPreviewExecutable(t, strings.Repeat("long ", 20)+"limits'\nrecorder") marker := filepath.Join(t.TempDir(), "MUST_NOT_EXIST") subject := "Owner's \"alert\"; $(touch " + marker + "); `touch " + marker + "` $HOME \\path\nnext line" channel, err := json.Marshal(map[string]any{"type": "email", "recipients": []string{"finance@example.test"}, "subject_prefix": subject}) @@ -522,7 +585,7 @@ func TestLimitsWorkspacePrintedCommandNativeShellPreservesArguments(t *testing.T if backend.writes.Load() != 0 { t.Fatal("printing a command wrote to the backend") } - capture := executeAudioPreview(t, shell, command, directory) + capture := executeLimitsPreview(t, shell, command, directory) want := append(append([]string{}, requestArgs...), tc.want...) if !reflect.DeepEqual(capture.args, want) { t.Fatalf("printed command changed arguments:\n got: %q\nwant: %q", capture.args, want) From cc902da508ba18d18478b8e08bd0aa67368e74de Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Mon, 19 Oct 2026 17:03:49 -0700 Subject: [PATCH 27/44] Polish Limits workflows and expose fine-tuning model limits --- cmd/openai/fine_tuning_model_limits_test.go | 381 ++++++++++++++++++ .../fine_tuning_model_limits_unix_test.go | 111 +++++ docs/limits-workspace.md | 27 +- pkg/custom/command.go | 1 + pkg/custom/fine_tuning_model_limits.go | 99 +++++ pkg/custom/limits_draft.go | 16 + pkg/custom/limits_workspace.go | 43 +- pkg/custom/limits_workspace_test.go | 279 +++++++++++++ pkg/custom/limits_workspace_view.go | 36 +- scripts/demos/limits-workspace/README.md | 7 +- scripts/demos/limits-workspace/scene.py | 49 ++- scripts/demos/limits-workspace/validate.py | 17 +- 12 files changed, 1043 insertions(+), 23 deletions(-) create mode 100644 cmd/openai/fine_tuning_model_limits_test.go create mode 100644 cmd/openai/fine_tuning_model_limits_unix_test.go create mode 100644 pkg/custom/fine_tuning_model_limits.go diff --git a/cmd/openai/fine_tuning_model_limits_test.go b/cmd/openai/fine_tuning_model_limits_test.go new file mode 100644 index 00000000..aef52997 --- /dev/null +++ b/cmd/openai/fine_tuning_model_limits_test.go @@ -0,0 +1,381 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/goccy/go-yaml" + "github.com/tidwall/gjson" +) + +const fineTuningModelLimitsResponse = `{"object":"synthetic.future_limits","data":[{"model":"synthetic-model","future":{"exact":9007199254740993,"decimal":0.100000000000000001,"unset":null,"text":"quoted \"value\""}}],"has_more":true,"next_page":"synthetic-unfollowed-cursor"}` + +func fineTuningModelLimitsEnvironment(t *testing.T, endpoint string) []string { + t.Helper() + home := t.TempDir() + return []string{ + "HOME=" + home, "USERPROFILE=" + home, "XDG_CONFIG_HOME=" + home, + "OPENAI_BASE_URL=" + endpoint, "OPENAI_API_KEY=sk-fake-fine-tuning", + "OPENAI_ADMIN_KEY=sk-fake-admin-unused", "OPENAI_ORG_ID=org_synthetic_fine_tuning", + "OPENAI_PROJECT_ID=proj_synthetic_fine_tuning", "OPENAI_CUSTOM_HEADERS=X-Synthetic-Context: retained", + "NO_COLOR=1", "FORCE_COLOR=0", "CI=1", + } +} + +func TestMainFineTuningModelLimitsRequestAndCredentials(t *testing.T) { + for _, tc := range []struct { + name, key, admin, headers, authorization string + flags []string + missing, overrideContext bool + }{ + {name: "ordinary only", key: "sk-fake-fine-tuning", authorization: "Bearer sk-fake-fine-tuning"}, + {name: "both keys", key: "sk-fake-fine-tuning", admin: "sk-fake-admin-unused", authorization: "Bearer sk-fake-fine-tuning"}, + {name: "root overrides", key: "sk-fake-environment-unused", admin: "sk-fake-admin-unused", authorization: "Bearer sk-fake-flag", + flags: []string{"--api-key=sk-fake-flag", "--admin-api-key=sk-fake-admin-flag-unused", "--organization=org_synthetic_override", "--project=proj_synthetic_override"}, overrideContext: true}, + {name: "explicit authorization", key: "sk-fake-fine-tuning-unused", admin: "sk-fake-admin-unused", authorization: "Bearer synthetic-header", + flags: []string{"--header=Authorization: Bearer synthetic-header"}}, + {name: "explicit authorization without API key", admin: "sk-fake-admin-unused", authorization: "Bearer synthetic-header", + flags: []string{"--header=Authorization: Bearer synthetic-header"}}, + {name: "environment authorization", admin: "sk-fake-admin-unused", authorization: "Bearer synthetic-last", + headers: "Authorization: Bearer synthetic-first\nAUTHORIZATION: Bearer synthetic-last"}, + {name: "API key before environment authorization", key: "sk-fake-fine-tuning", admin: "sk-fake-admin-unused", + authorization: "Bearer sk-fake-fine-tuning", headers: "Authorization: Bearer synthetic-header-unused"}, + {name: "empty API key with environment authorization", key: "sk-fake-environment-unused", admin: "sk-fake-admin-unused", + authorization: "Bearer synthetic-header", headers: "Authorization: Bearer synthetic-header", flags: []string{"--api-key="}}, + {name: "admin only rejected", admin: "sk-fake-admin-unused", missing: true}, + {name: "missing credentials", missing: true}, + {name: "empty API key override", key: "sk-fake-environment-unused", admin: "sk-fake-admin-unused", + flags: []string{"--api-key="}, missing: true}, + {name: "empty authorization override", key: "sk-fake-fine-tuning", admin: "sk-fake-admin-unused", + flags: []string{"--header=Authorization:"}, missing: true}, + {name: "empty last environment authorization", admin: "sk-fake-admin-unused", + headers: "Authorization: Bearer synthetic-first\nAuthorization:", missing: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodGet || r.URL.RequestURI() != "/tenant/v1/fine_tuning/model_limits" { + t.Errorf("unexpected read request: %s %s", r.Method, r.URL.RequestURI()) + } + wantOrg, wantProject := "org_synthetic_fine_tuning", "proj_synthetic_fine_tuning" + if tc.overrideContext { + wantOrg, wantProject = "org_synthetic_override", "proj_synthetic_override" + } + if r.Header.Get("Authorization") != tc.authorization || + r.Header.Get("OpenAI-Organization") != wantOrg || r.Header.Get("OpenAI-Project") != wantProject || + r.Header.Get("X-Synthetic-Context") != "retained" { + t.Error("ordinary credential selection or request context changed") + } + body, err := io.ReadAll(r.Body) + if err != nil || len(body) != 0 { + t.Errorf("read request has a body or read error: bytes=%d error=%v", len(body), err) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, fineTuningModelLimitsResponse) + })) + defer server.Close() + env := fineTuningModelLimitsEnvironment(t, server.URL+"/tenant/v1/") + env = spendRecoverySetEnvironment(env, "OPENAI_API_KEY", tc.key) + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", tc.admin) + env = spendRecoverySetEnvironment(env, "OPENAI_CUSTOM_HEADERS", "X-Synthetic-Context: retained\n"+tc.headers) + flags := slices.Clone(tc.flags) + if tc.overrideContext { + env = spendRecoverySetEnvironment(env, "OPENAI_BASE_URL", "http://127.0.0.1:1/unused/") + flags = append(flags, "--base-url="+server.URL+"/tenant/v1/") + } + args := append([]string{"openai", "fine-tuning", "model-limits", "--format=json"}, flags...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if tc.missing { + if got.code != 1 || got.stdout != "" || requests.Load() != 0 { + t.Fatalf("missing credentials reached HTTP or changed exit: code=%d requests=%d", got.code, requests.Load()) + } + payload := decodeMainStructuredError(t, "json", got.stderr) + if !strings.Contains(fmt.Sprint(payload["message"]), "OPENAI_API_KEY") || + !strings.Contains(fmt.Sprint(payload["message"]), "admin key cannot replace") { + t.Fatal("missing ordinary credentials lost actionable guidance") + } + if strings.Contains(got.stderr, "sk-fake-") || strings.Contains(got.stderr, "synthetic-first") { + t.Fatal("credential failure exposed a synthetic credential") + } + return + } + if got.code != 0 || got.stderr != "" || requests.Load() != 1 || !json.Valid([]byte(got.stdout)) { + t.Fatalf("read failed or followed an undocumented cursor: code=%d requests=%d stderr=%q", got.code, requests.Load(), got.stderr) + } + if gjson.Get(got.stdout, "data.0.future.exact").Raw != "9007199254740993" || + gjson.Get(got.stdout, "next_page").String() != "synthetic-unfollowed-cursor" { + t.Fatal("read lost unfamiliar response fields") + } + }) + } +} + +func TestMainFineTuningModelLimitsOutputModes(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, fineTuningModelLimitsResponse) + })) + defer server.Close() + env := fineTuningModelLimitsEnvironment(t, server.URL) + for _, format := range []string{"json", "jsonl", "pretty", "raw", "yaml"} { + t.Run(format, func(t *testing.T) { + before := requests.Load() + got := runMainDispatchWithEnv(t, "bash", env, "openai", "fine-tuning", "model-limits", "--format="+format) + if got.code != 0 || got.stderr != "" || requests.Load() != before+1 { + t.Fatalf("format failed: code=%d stderr=%q", got.code, got.stderr) + } + if format == "pretty" { + for _, value := range []string{"fine-tuning model-limits", "synthetic-model", "9007199254740993", "0.100000000000000001", "synthetic-unfollowed-cursor"} { + if !strings.Contains(got.stdout, value) { + t.Fatalf("pretty output omitted %q", value) + } + } + return + } + output := []byte(got.stdout) + if format == "yaml" { + if !strings.Contains(got.stdout, "decimal: 0.100000000000000001") { + t.Fatal("YAML output rounded an unfamiliar decimal") + } + var err error + output, err = yaml.YAMLToJSON(output) + if err != nil { + t.Fatal(err) + } + } + if !json.Valid(output) || gjson.GetBytes(output, "data.0.future.exact").Raw != "9007199254740993" || + gjson.GetBytes(output, "data.0.future.unset").Raw != "null" || + gjson.GetBytes(output, "data.0.future.text").String() != `quoted "value"` || + !gjson.GetBytes(output, "has_more").Bool() { + t.Fatal("output dropped or coerced unfamiliar response fields") + } + // YAML uses the shared converter. JSON modes must retain original decimal tokens. + if format != "yaml" && gjson.GetBytes(output, "data.0.future.decimal").Raw != "0.100000000000000001" { + t.Fatal("JSON output rounded an unfamiliar decimal") + } + if format == "raw" && strings.TrimSpace(got.stdout) != fineTuningModelLimitsResponse { + t.Fatal("raw output changed the response bytes") + } + }) + } + for _, tc := range []struct { + name string + args []string + want string + }{ + {"extraction", []string{"--transform=data.0.future.exact"}, "9007199254740993"}, + {"raw string", []string{"--transform=data.0.future.text", "--raw-output"}, `quoted "value"`}, + {"quiet", []string{"--quiet", "--format=json", "--transform=data.0.future.decimal"}, "0.100000000000000001"}, + } { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"openai", "fine-tuning", "model-limits"}, tc.args...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if got.code != 0 || got.stderr != "" || strings.TrimSpace(got.stdout) != tc.want { + t.Fatalf("extraction contract changed: %+v", got) + } + }) + } +} + +func TestMainFineTuningModelLimitsLargeResponse(t *testing.T) { + const size = 32<<20 + 1 + body := `{"future":{"payload":"` + strings.Repeat("x", size) + `","exact":9007199254740993},"data":[]}` + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + got := runMainDispatchWithEnv(t, "bash", fineTuningModelLimitsEnvironment(t, server.URL), + "openai", "fine-tuning", "model-limits", "--format=raw") + if got.code != 0 || got.stderr != "" || requests.Load() != 1 { + t.Fatalf("large read failed: code=%d requests=%d stderr=%q", got.code, requests.Load(), got.stderr) + } + if strings.TrimSpace(got.stdout) != body { + t.Fatal("large unfamiliar response was truncated or rewritten") + } +} + +func TestMainFineTuningModelLimitsResponseValidation(t *testing.T) { + for _, tc := range []struct { + name, body string + invalid bool + }{ + {"empty", "", true}, + {"malformed", "synthetic-private-response\x1b]52;c;private\a", true}, + {"array", `[{"unfamiliar":9007199254740993}]`, false}, + {"string", `"unfamiliar response"`, false}, + {"number", "1e10000", false}, + {"true", "true", false}, + {"false", "false", false}, + {"null", "null", false}, + } { + t.Run(tc.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, tc.body) + })) + defer server.Close() + got := runMainDispatchWithEnv(t, "bash", fineTuningModelLimitsEnvironment(t, server.URL), + "openai", "fine-tuning", "model-limits", "--format=json") + if requests.Load() != 1 { + t.Fatalf("read issued %d requests", requests.Load()) + } + if tc.invalid { + if got.code != 1 || got.stdout != "" { + t.Fatalf("invalid response reported success: code=%d", got.code) + } + payload := decodeMainStructuredError(t, "json", got.stderr) + if payload["message"] != "Fine-tuning model limits returned an invalid JSON response. Check --base-url, then retry this read." { + t.Fatal("invalid response lost safe recovery guidance") + } + if strings.Contains(got.stderr, "synthetic-private") || strings.ContainsAny(got.stderr, "\x1b\a") { + t.Fatal("invalid response exposed private contents") + } + return + } + var compact bytes.Buffer + if got.code != 0 || got.stderr != "" || json.Compact(&compact, []byte(got.stdout)) != nil || compact.String() != tc.body { + t.Fatalf("valid JSON shape changed: code=%d stderr=%q stdout=%q", got.code, got.stderr, got.stdout) + } + }) + } +} + +func TestMainFineTuningModelLimitsErrors(t *testing.T) { + const details = `{"message":"synthetic-private-response\u001b]52;c;private\u0007","type":"invalid_request_error","code":"synthetic_code","param":null,"future":{"exact":9007199254740993}}` + for _, tc := range []struct { + name, contentType, body string + status int + flags []string + }{ + {"ordinary authentication", "application/json", `{"error":` + details + `}`, 401, []string{"--format-error=text"}}, + {"unavailable route", "application/json", `{"error":` + details + `}`, 404, []string{"--format-error=text"}}, + {"JSON", "application/json", `{"error":` + details + `}`, 400, []string{"--format-error=json"}}, + {"YAML", "application/json", `{"error":` + details + `}`, 400, []string{"--format-error=yaml"}}, + {"extraction", "application/json", `{"error":` + details + `}`, 400, []string{"--format-error=json", "--transform-error=future.exact"}}, + {"empty server error", "text/html", "synthetic-private-response", 502, []string{"--format-error=json"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := runMainCommandAPIErrorResponse(t, tc.status, tc.contentType, tc.body, + []string{"fine-tuning", "model-limits"}, tc.flags...) + control := runMainAPIErrorResponse(t, tc.status, tc.contentType, tc.body, tc.flags...) + if got != control { + t.Fatalf("ordinary API error contract changed: got=%+v control=%+v", got, control) + } + if strings.HasSuffix(tc.name, "authentication") || tc.name == "unavailable route" { + if strings.Contains(got.stderr, "synthetic-private") || strings.Contains(got.stderr, "\x1b") || + strings.Contains(got.stderr, "setup admin") { + t.Fatal("human error exposed private data or requested admin authentication") + } + } + }) + } +} + +func TestMainFineTuningModelLimitsHelpAndValidation(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, fineTuningModelLimitsResponse) + })) + defer server.Close() + env := fineTuningModelLimitsEnvironment(t, server.URL) + for _, argv := range [][]string{ + {"openai", "fine-tuning", "--help"}, + {"openai", "fine-tuning", "model-limits", "--help"}, + {"openai", "help", "fine-tuning", "model-limits"}, + } { + t.Run(strings.Join(argv[1:], " "), func(t *testing.T) { + got := runMainDispatchWithEnv(t, "bash", env, argv...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "model-limits") || requests.Load() != 0 { + t.Fatalf("help hid the command or made a request: %+v", got) + } + if len(argv) == 4 && argv[2] == "model-limits" || argv[1] == "help" { + if !strings.Contains(got.stdout, "openai fine-tuning model-limits") { + t.Fatal("command help omitted its invocation") + } + } + }) + } + for _, args := range [][]string{{"synthetic-private-operand"}, {"--unknown-synthetic=private"}, {"--max-items=1"}} { + t.Run(strings.Join(args, " "), func(t *testing.T) { + argv := append([]string{"openai", "--format-error=json", "fine-tuning", "model-limits"}, args...) + got := runMainDispatchWithEnv(t, "bash", env, argv...) + if got.code != 1 || got.stdout != "" || requests.Load() != 0 { + t.Fatalf("invalid arguments reached the endpoint: %+v", got) + } + decodeMainStructuredError(t, "json", got.stderr) + if strings.Contains(got.stderr, "synthetic-private-operand") || strings.Contains(got.stderr, "=private") { + t.Fatal("argument validation exposed a rejected value") + } + }) + } +} + +func TestFineTuningModelLimitsErrorCoverage(t *testing.T) { + env := errorCoverageEnvironment(t.TempDir(), "http://127.0.0.1:1") + manifest := readErrorCoverageManifest(t, env) + var found bool + for _, command := range manifest.Commands { + if strings.Join(command.Path, " ") == "fine-tuning model-limits" { + if command.HasChildren || command.Compatibility || command.Class != "public" { + t.Fatal("fine-tuning model limits is not an ordinary public leaf") + } + found = true + } + } + if !found { + t.Fatal("error inventory omitted fine-tuning model limits") + } + for _, probe := range declaredErrorCoverageCases(manifest) { + if probe.Name != "fine-tuning model-limits/unknown-option/" { + continue + } + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format-error=json"}, probe.Args...)...) + payload := decodeMainStructuredError(t, "json", got.stderr) + if got.code != probe.Code || got.stdout != "" || payload["message"] != probe.Message { + t.Fatal("the inventoried parser probe did not preserve its public diagnostic") + } + return + } + t.Fatal("error inventory omitted the new leaf's unknown-option probe") +} + +func TestMainFineTuningModelLimitsBaseURLSecurity(t *testing.T) { + var requests atomic.Int32 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusTeapot) + })) + defer proxy.Close() + env := adminCredentialsProxyEnv(proxy.URL, fineTuningModelLimitsEnvironment(t, + "http://synthetic-user:synthetic-password@remote.example/v1?token=synthetic-private")) + got := runMainDispatchWithEnv(t, "bash", env, "openai", "fine-tuning", "model-limits", "--format-error=json") + if got.code != 1 || got.stdout != "" || requests.Load() != 0 { + t.Fatal("insecure remote base URL reached HTTP") + } + payload := decodeMainStructuredError(t, "json", got.stderr) + if !strings.Contains(fmt.Sprint(payload["message"]), "HTTPS") { + t.Fatal("base URL rejection omitted recovery") + } + if bytes.ContainsAny([]byte(got.stderr), "\x1b\a") || strings.Contains(got.stderr, "synthetic-password") || + strings.Contains(got.stderr, "remote.example") { + t.Fatal("base URL rejection exposed private URL contents") + } +} diff --git a/cmd/openai/fine_tuning_model_limits_unix_test.go b/cmd/openai/fine_tuning_model_limits_unix_test.go new file mode 100644 index 00000000..912af995 --- /dev/null +++ b/cmd/openai/fine_tuning_model_limits_unix_test.go @@ -0,0 +1,111 @@ +//go:build !windows + +package main + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "testing" +) + +func TestMainFineTuningModelLimitsNativeHelpExamples(t *testing.T) { + work, binary := buildNativeShellCLI(t) + executable := filepath.Join(work, binary) + invocation := "'" + strings.ReplaceAll(executable, "'", "'\\''") + "'" + var requests, adminRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if strings.HasPrefix(r.URL.Path, "/organization/") || strings.HasPrefix(r.URL.Path, "/admin/") { + adminRequests.Add(1) + } + if r.Method != http.MethodGet || r.URL.RequestURI() != "/fine_tuning/model_limits" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.RequestURI()) + } + if r.Header.Get("Authorization") != "Bearer fake-native-shell-key" { + t.Error("request omitted the ordinary synthetic credential") + } + body, err := io.ReadAll(r.Body) + if err != nil || len(body) != 0 { + t.Errorf("read request has a body or read error: bytes=%d error=%v", len(body), err) + } + w.Header().Set("Content-Type", "application/json") + if _, err := fmt.Fprint(w, fineTuningModelLimitsResponse); err != nil { + t.Errorf("write synthetic limits response: %v", err) + } + })) + defer server.Close() + required := strings.Split(os.Getenv("OPENAI_CLI_REQUIRE_NATIVE_SHELLS"), ",") + for _, shell := range []nativeShell{ + {"bash", "bash", "", "export OPENAI_API_KEY='fake-native-shell-key'; ", []string{"--noprofile", "--norc", "-c"}}, + {"zsh", "zsh", "", "export OPENAI_API_KEY='fake-native-shell-key'; ", []string{"-f", "-c"}}, + } { + t.Run(shell.name, func(t *testing.T) { + path, err := exec.LookPath(shell.executable) + if err != nil { + if slices.Contains(required, shell.name) { + t.Fatalf("required shell is unavailable: %v", err) + } + t.Skipf("%s is not installed", shell.name) + } + shell.executable = path + for _, lookup := range []string{"direct_path", "installed"} { + t.Run(lookup, func(t *testing.T) { + directory, home, searchPath := t.TempDir(), t.TempDir(), t.TempDir() + printedInvocation := invocation + if lookup == "direct_path" { + if err := os.WriteFile(filepath.Join(searchPath, "openai"), []byte("#!/bin/sh\nprintf 'wrong executable\\n'\nexit 99\n"), 0o700); err != nil { + t.Fatal(err) + } + } else { + searchPath, printedInvocation = work, "openai" + } + t.Setenv("PATH", searchPath) + for _, helpArgs := range []struct{ name, args string }{ + {"concise", " fine-tuning model-limits --help"}, + {"full", " help fine-tuning model-limits"}, + } { + t.Run(helpArgs.name, func(t *testing.T) { + requests.Store(0) + adminRequests.Store(0) + help := runNativeShell(t, shell, directory, home, server.URL, invocation+helpArgs.args) + if help.code != 0 || help.stderr != "" || requests.Load() != 0 { + t.Fatalf("help failed or made a request: %+v", help) + } + _, examples, found := strings.Cut(help.stdout, "EXAMPLES:\n") + if !found { + t.Fatal("help omitted executable examples") + } + examples, _, _ = strings.Cut(examples, "\nOPTIONS:") + var commands []string + for line := range strings.SplitSeq(examples, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), printedInvocation+" fine-tuning model-limits") { + commands = append(commands, strings.TrimSpace(line)) + } + } + if len(commands) != 1 || commands[0] != printedInvocation+" fine-tuning model-limits" { + t.Fatalf("help changed the example or executable identity %q:\n%s", printedInvocation, help.stdout) + } + got := runNativeShell(t, shell, directory, home, server.URL, shell.setKey+commands[0]) + if got.code != 0 || got.stderr != "" || requests.Load() != 1 || adminRequests.Load() != 0 { + t.Fatalf("copied example failed: result=%+v requests=%d admin=%d", got, requests.Load(), adminRequests.Load()) + } + for _, value := range []string{"synthetic-model", "9007199254740993", "0.100000000000000001", "synthetic-unfollowed-cursor"} { + if strings.Count(got.stdout, value) != 1 { + t.Fatalf("example output lost or duplicated %q: %q", value, got.stdout) + } + } + }) + } + }) + } + }) + } +} diff --git a/docs/limits-workspace.md b/docs/limits-workspace.md index a5a6b141..eddd89bb 100644 --- a/docs/limits-workspace.md +++ b/docs/limits-workspace.md @@ -21,7 +21,7 @@ openai admin limits show --format json --transform spend_alerts `--max-items` defaults to 100 records per list. Use `-1` for all records or `0` to validate the list request without retaining records. -The overview reports whether each list is complete. +The interactive workspace puts each list's Complete or Incomplete status before its loaded count. Failed reads preserve available data and return a nonzero exit status. Cancellation can stop output before the CLI prints those records. The `source_errors` field identifies failed sources without exposing response error bodies. @@ -51,6 +51,7 @@ Values beyond that precision remain available in `costs.buckets`; the summary re | r | Refresh the overview or a rejected draft | | i | Inspect the selected original API record | | ? | Show workspace help and API boundaries | +| F1 | Show help, including complete form diagnostics | | d | Delete the selected alert | | Escape | Return without saving | | q | Close the overview | @@ -60,10 +61,17 @@ Values beyond that precision remain available in `costs.buckets`; the summary re The workspace needs at least 40 columns and 12 rows. Smaller terminals retain cancellation and block submission. The workspace restores the terminal screen when it exits. +Escape closes help and restores the active form or review. +Forms retain their fields, focus, and subject-prefix mode. +Repeated F1 presses preserve that return point. +Escape from a review returns to the form, like Back. Every change opens a before-and-after review. +Spend and alert reviews show exact monthly USD amounts alongside the original API fields. Scroll to the end before selecting Apply. Back is the initial selection, including after resizing. +Returning from help also selects Back. +Resizing during help requires rereading the review before selecting Apply. The review shows a shell command when the CLI can reproduce the request context safely. Wrapped command rows are for inspection only. Press Ctrl+P to print the complete command after the terminal screen restores. @@ -125,6 +133,20 @@ Inspect the resource before reopening the workspace or repeating the action. A refresh alone cannot prove that an uncertain alert creation failed. Cancellation does not prove that the server canceled a dispatched write. +## Fine-tuning model limits + +The new `openai fine-tuning model-limits` command reads the documented fine-tuning model limits. +Use an ordinary `OPENAI_API_KEY`; this command does not use admin-key fallback. +Root organization, project, connection, and output options remain available. +The command reads one endpoint and preserves all response fields, including unknown fields. +It adds no pagination or mutation flags. + +```sh +openai fine-tuning model-limits --format json +``` + +See the [fine-tuning rate-limits documentation](https://developers.openai.com/api/docs/guides/rate-limits#fine-tuning-rate-limits). + ## Platform controls without matching contracts The [Platform limits page](https://platform.openai.com/settings/organization/limits) includes controls beyond these public operations. @@ -134,8 +156,5 @@ The workspace does not emulate these controls through undocumented requests. Project rate records do not describe every organization-wide model limit or shared model-family limit. Historical costs are not a live allowance or enforcement counter. -The documented fine-tuning model-limits endpoint is absent from the pinned generated schema and SDK. -That endpoint needs authoritative schema and generator support before CLI integration. - See the [spend-limits guide](https://developers.openai.com/api/docs/guides/spend-limits) and [rate-limits guide](https://developers.openai.com/api/docs/guides/rate-limits) for API behavior. diff --git a/pkg/custom/command.go b/pkg/custom/command.go index f47b9cb9..b3f39bb3 100644 --- a/pkg/custom/command.go +++ b/pkg/custom/command.go @@ -46,6 +46,7 @@ func ConfigureCommand(root *cli.Command) { registerAdminSetup(root) admin.ConfigureCredentials(root) localcommands.ConfigureSubgroups(root) + registerFineTuningModelLimits(root) registerLimitsCommands(root) localcommands.ConfigureTasks(root) registerAudioCommands(root) diff --git a/pkg/custom/fine_tuning_model_limits.go b/pkg/custom/fine_tuning_model_limits.go new file mode 100644 index 00000000..bfe763e0 --- /dev/null +++ b/pkg/custom/fine_tuning_model_limits.go @@ -0,0 +1,99 @@ +package custom + +import ( + "context" + "os" + "strings" + + "github.com/openai/openai-cli/internal/apiquery" + "github.com/openai/openai-cli/internal/clihelp" + localcommands "github.com/openai/openai-cli/internal/commands" + requestconfig "github.com/openai/openai-cli/internal/request" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +func registerFineTuningModelLimits(root *cli.Command) { + group := root.Command("fine-tuning") + if group == nil || group.Command("model-limits") != nil { + return + } + group.Commands = append(group.Commands, &cli.Command{ + Name: "model-limits", Usage: "Read organization fine-tuning rate limits.", + Description: "Uses an ordinary API key. Supports full JSON output and standard output options.", + Suggest: true, HideHelpCommand: true, + Metadata: map[string]any{"help-content": clihelp.Content{ + Examples: []clihelp.Example{{Description: "Read fine-tuning rate limits:", Command: "fine-tuning model-limits"}}, + }}, + Action: handleFineTuningModelLimits, + }) +} + +func handleFineTuningModelLimits(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Present() { + return &localcommands.LocalError{Message: "Fine-tuning model limits takes no positional arguments. Use --help to inspect its options."} + } + options, err := FlagOptions(cmd, apiquery.NestedQueryFormatBrackets, apiquery.ArrayQueryFormatBrackets, EmptyBody, false) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := checkFineTuningModelLimitsCredentials(cmd); err != nil { + return err + } + client := openai.NewClient(GetDefaultRequestOptions(cmd)...) + var response []byte + // Client.Get permits admin fallback. Clear it before explicit request-header overrides. + options = append([]option.RequestOption{option.WithAdminAPIKey("")}, options...) + options = append(options, option.WithResponseBodyInto(&response)) + if err := client.Get(ctx, "fine_tuning/model_limits", nil, nil, options...); err != nil { + return err + } + if !gjson.ValidBytes(response) { + return &localcommands.LocalError{Message: "Fine-tuning model limits returned an invalid JSON response. Check --base-url, then retry this read."} + } + root := cmd.Root() + return ShowJSON(gjson.ParseBytes(response), ShowJSONOpts{ + Context: ctx, Operation: "(resource) fine-tuning > (method) model-limits", OutputKind: OutputResponse, + ExplicitFormat: root.IsSet("format"), Format: root.String("format"), + RawOutput: root.Bool("raw-output"), Transform: root.String("transform"), + Title: "fine-tuning model-limits", Stdout: root.Writer, Stderr: root.ErrWriter, + }) +} + +// Match shared request-header precedence without imposing an admin credential requirement. +func checkFineTuningModelLimitsCredentials(cmd *cli.Command) error { + missing := &localcommands.LocalError{Message: "API key required for fine-tuning model limits. Set OPENAI_API_KEY or an explicit Authorization header. An admin key cannot replace an API key."} + headers, err := requestconfig.Headers(cmd) + if err != nil { + return err + } + if values, ok := headers["Authorization"]; ok { + if values[0] != "" { + return nil + } + return missing + } + key := os.Getenv("OPENAI_API_KEY") + if cmd.IsSet("api-key") { + key = cmd.String("api-key") + } + if key != "" { + return nil + } + var authorization string + for line := range strings.SplitSeq(os.Getenv("OPENAI_CUSTOM_HEADERS"), "\n") { + name, value, ok := strings.Cut(line, ":") + if ok && strings.EqualFold(strings.TrimSpace(name), "Authorization") { + authorization = strings.TrimSpace(value) + } + } + if authorization != "" { + return nil + } + return missing +} diff --git a/pkg/custom/limits_draft.go b/pkg/custom/limits_draft.go index ebafdcc8..ba9855df 100644 --- a/pkg/custom/limits_draft.go +++ b/pkg/custom/limits_draft.go @@ -194,6 +194,22 @@ func limitsReview(c limitsChange, projectID string, requestArgs []string, omit b if c.ID != "" { result += "ID: " + c.ID + "\n" } + if c.Kind == "limit_set" || c.Kind == "limit_delete" || strings.HasPrefix(c.Kind, "alert_") { + label, previous := "Hard spend limit", "Not confirmed" + if strings.HasPrefix(c.Kind, "alert_") { + label = "Alert threshold" + } + if c.Before.Exists() { + previous = limitsThresholdLabel(c.Before) + } else if c.Kind == "alert_create" { + previous = "New alert" + } + next := "USD " + formatLimitsDollars(c.Amount) + " / month" + if strings.HasSuffix(c.Kind, "_delete") { + next = "Remove this resource" + } + result += "\n" + label + "\nBefore: " + previous + "\nAfter: " + next + "\n" + } result += "\nBefore\n" + before + "\n\nAfter\n" if strings.HasSuffix(c.Kind, "_delete") { result += "Remove this resource.\n" diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index 0fc35b13..e22e64ab 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -16,6 +16,7 @@ import ( "github.com/charmbracelet/x/term" "github.com/openai/openai-cli/internal/clihelp" clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-cli/internal/readable" clirequest "github.com/openai/openai-cli/internal/request" "github.com/openai/openai-cli/internal/terminal" "github.com/openai/openai-go/v3" @@ -37,6 +38,8 @@ type limitsMenuItem struct { type limitsWorkspace struct { detailsHeading string + detailsReturnPage string + detailsReturnOffset int previewExecutable string previewCommand, commandToPrint string invocation string @@ -513,6 +516,7 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { m.setReview() } if m.page == "details" { + m.detailsReturnOffset = 0 m.setDetails(m.details) } m.viewport.SetWidth(max(1, msg.Width-2)) @@ -563,13 +567,35 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { return m, nil } if key == "esc" { - m.page, m.apply = "home", false + m.apply = false + switch m.page { + case "details": + m.page, m.detailsReturnPage = m.detailsReturnPage, "" + switch m.page { + case "review": + m.setReview() + m.viewport.SetYOffset(m.detailsReturnOffset) + case "form": + m.setFocus(m.focus) + default: + m.page = "home" + } + case "review": + m.page = "form" + m.setFocus(m.focus) + default: + m.page = "home" + } return m, nil } if m.small() { return m, nil } if key == "f1" || key == "?" && m.page != "form" || key == "?" && m.locked { + if m.page != "details" { + m.detailsReturnPage, m.detailsReturnOffset = m.page, m.viewport.YOffset() + } + m.apply = false m.page, m.detailsHeading = "details", "Limits workspace help" m.setDetails(m.help()) return m, nil @@ -602,6 +628,7 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { row := rows[m.selection] if key == "i" { if row.value.Exists() { + m.detailsReturnPage = "home" m.page, m.detailsHeading = "details", "Original API record" m.setDetails(limitsSafeJSON(row.value)) } @@ -655,7 +682,7 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { if key == "enter" { if !m.apply { m.page = "form" - m.setFocus(0) + m.setFocus(m.focus) return m, nil } if m.viewport.AtBottom() && !m.locked && m.ctx.Err() == nil && (m.output == nil || m.output.Err() == nil) { @@ -731,9 +758,15 @@ func (m *limitsWorkspace) sourceSummary() string { } return "Hard spend limit unavailable. Refresh to retry." case 1: - return fmt.Sprintf("Alerts loaded: %d Complete: %t", len(m.snapshot.Alerts), m.snapshot.AlertsComplete) + if !m.snapshot.AlertsComplete { + return fmt.Sprintf("Incomplete. Alerts loaded: %d", len(m.snapshot.Alerts)) + } + return fmt.Sprintf("Complete. Alerts loaded: %d", len(m.snapshot.Alerts)) default: - return fmt.Sprintf("Rate records loaded: %d Complete: %t", len(m.snapshot.Rates), m.snapshot.RatesComplete) + if !m.snapshot.RatesComplete { + return fmt.Sprintf("Incomplete. Rate records loaded: %d", len(m.snapshot.Rates)) + } + return fmt.Sprintf("Complete. Rate records loaded: %d", len(m.snapshot.Rates)) } } @@ -848,6 +881,8 @@ func (m *limitsWorkspace) help() string { failure := m.mutationError m.mu.Unlock() text = limitsPreflightHint(failure, invocation) + "\n\n" + text + } else if m.formError != "" && (m.page == "form" || m.detailsReturnPage == "form") { + text = "Draft details\n" + readable.Text(m.formError) + "\n\n" + text } for _, failure := range m.snapshot.Errors { // Source names come from our loader, never response text. diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index c5b36008..a19979f0 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -22,6 +22,7 @@ import ( "charm.land/bubbles/v2/viewport" tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/colorprofile" "github.com/charmbracelet/x/ansi" clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-cli/internal/terminal" @@ -870,6 +871,276 @@ func TestLimitsWorkspaceActionVisibleAtMinimumSize(t *testing.T) { } } +func TestLimitsWorkspaceHelpRestoresDraftAndReview(t *testing.T) { + t.Run("form", func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.begin("alert_update", gjson.Parse(limitsControllerAlert)) + values := []string{"123.45", "finance@example.test", "Edited $HOME prefix"} + for i, value := range values { + m.draft.inputs[i].SetValue(value) + m.draft.inputs[i].SetCursor(i + 1) + } + for mode := 0; mode < 3; mode++ { + m.draft.prefixMode = mode + for focus := 0; focus < 5; focus++ { + m.setFocus(focus) + before, err := m.draft.validate() + require.NoError(t, err) + for _, key := range []rune{tea.KeyF1, tea.KeyF1, '?', tea.KeyEscape} { + require.Nil(t, limitsControllerKey(m, key)) + } + require.Equal(t, "form", m.page) + require.Equal(t, focus, m.focus) + require.Equal(t, mode, m.draft.prefixMode) + for i, value := range values { + require.Equal(t, value, m.draft.inputs[i].Value()) + require.Equal(t, i == focus, m.draft.inputs[i].Focused()) + require.Equal(t, i+1, m.draft.inputs[i].Position()) + } + after, err := m.draft.validate() + require.NoError(t, err) + require.Equal(t, limitsChangeBody(before), limitsChangeBody(after)) + } + } + require.Zero(t, backend.writes.Load()) + require.Zero(t, backend.reads.Load()) + }) + for _, resized := range []bool{false, true} { + t.Run(fmt.Sprintf("review_resize_%t", resized), func(t *testing.T) { + m, backend := newLimitsController(t, 80, 24) + limitsControllerReview(t, m) + command := m.previewCommand + m.viewport.GotoBottom() + review := m.viewport.View() + limitsControllerKey(m, tea.KeyTab) + require.True(t, m.apply) + limitsControllerKey(m, tea.KeyF1) + limitsControllerKey(m, tea.KeyF1) + if resized { + m.Update(tea.WindowSizeMsg{Width: 40, Height: 13}) + } + limitsControllerKey(m, tea.KeyEscape) + require.Equal(t, "review", m.page) + require.False(t, m.apply) + require.Equal(t, command, m.previewCommand) + require.Equal(t, "100.01", m.draft.inputs[0].Value()) + if resized { + require.Zero(t, m.viewport.YOffset()) + require.False(t, m.viewport.AtBottom()) + limitsControllerKey(m, tea.KeyTab) + require.False(t, m.apply, "resize must require rereading before Apply") + } else { + require.Equal(t, review, m.viewport.View()) + require.True(t, m.viewport.AtBottom()) + } + require.Zero(t, backend.writes.Load()) + }) + } + t.Run("home_and_inspection", func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.snapshot.Alerts = []gjson.Result{gjson.Parse(limitsControllerAlert)} + m.section, m.selection = 1, 1 + for _, key := range []rune{tea.KeyF1, tea.KeyF1, tea.KeyEscape, 'i', tea.KeyF1, tea.KeyEscape} { + require.Nil(t, limitsControllerKey(m, key)) + } + require.Equal(t, "home", m.page) + require.Equal(t, 1, m.section) + require.Equal(t, 1, m.selection) + require.False(t, m.quitting) + require.Zero(t, backend.writes.Load()) + }) +} + +func TestLimitsWorkspaceReviewEscapeMatchesBack(t *testing.T) { + for _, key := range []rune{tea.KeyEscape, tea.KeyEnter} { + t.Run(fmt.Sprintf("key_%d", key), func(t *testing.T) { + m, backend := newLimitsController(t, 40, 13) + m.begin("alert_update", gjson.Parse(limitsControllerAlert)) + m.draft.inputs[0].SetValue("125.25") + m.draft.inputs[2].SetValue("Changed prefix") + m.draft.prefixMode = 1 + m.setFocus(4) + limitsControllerKey(m, tea.KeyEnter) + require.Equal(t, "review", m.page) + require.False(t, m.apply) + if key == tea.KeyEscape { + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + require.True(t, m.apply) + } + limitsControllerKey(m, key) + require.Equal(t, "form", m.page) + require.Equal(t, 4, m.focus) + require.Equal(t, 1, m.draft.prefixMode) + require.Equal(t, "125.25", m.draft.inputs[0].Value()) + require.Equal(t, "Changed prefix", m.draft.inputs[2].Value()) + require.False(t, m.apply) + limitsControllerKey(m, tea.KeyEnter) + require.Equal(t, "review", m.page) + require.False(t, m.apply) + require.Equal(t, int64(12525), m.change.Amount) + require.Equal(t, "Changed prefix", *m.change.SubjectPrefix) + require.Zero(t, backend.writes.Load()) + }) + } +} + +func TestLimitsWorkspaceFormDiagnosticsFitAndRemainAccessible(t *testing.T) { + for _, width := range []int{40, 80} { + for _, height := range []int{12, 13, 14, 24} { + for _, state := range []string{"validation", "preflight", "uncertain"} { + t.Run(fmt.Sprintf("%dx%d_%s", width, height, state), func(t *testing.T) { + m, backend := newLimitsController(t, width, height) + m.begin("alert_create", gjson.Result{}) + m.draft.inputs[0].SetValue("125.25") + m.draft.inputs[1].SetValue("Name ") + m.review() + require.Equal(t, "form", m.page) + require.Contains(t, m.formError, "without display names") + if state != "validation" { + m.draft.inputs[1].SetValue("finance@example.test") + m.review() + m.mutationError = &limitsMutationFailure{ + Attempted: state == "uncertain", + Err: &openai.Error{StatusCode: 400, Message: "synthetic-private-response"}, + } + m.Update(limitsWorkResult{mutation: true, err: m.mutationError}) + } + outcome := strings.SplitN(m.formError, "\n", 2)[0] + for focus := 0; focus < 5; focus++ { + m.setFocus(focus) + for _, dark := range []bool{false, true} { + m.color, m.dark = false, dark + raw := m.View().Content + var monochrome bytes.Buffer + writer := colorprofile.Writer{Forward: &monochrome, Profile: colorprofile.NoTTY} + _, err := io.WriteString(&writer, raw) + require.NoError(t, err) + view := monochrome.String() + require.Contains(t, strings.Join(strings.Fields(view), " "), outcome) + for _, control := range []string{"Review change", "F1: help", "Esc: back", "Ctrl+C"} { + require.Contains(t, view, control) + } + if state == "preflight" { + require.Contains(t, view, "r: refresh") + } + require.NotContains(t, view, "\x1b") + require.NotContains(t, view, "synthetic-private-response") + require.LessOrEqual(t, len(strings.Split(view, "\n")), height) + for _, line := range strings.Split(view, "\n") { + require.LessOrEqual(t, ansi.StringWidth(line), width) + } + } + } + limitsControllerKey(m, tea.KeyF1) + require.Equal(t, "details", m.page) + require.NotContains(t, m.details, "synthetic-private-response") + if state == "validation" { + require.Contains(t, m.details, outcome) + } else if state == "preflight" { + require.Contains(t, m.details, outcome) + require.Contains(t, m.details, "request fields and scope") + } else { + require.Contains(t, m.details, "It may have succeeded") + } + m.viewport.GotoBottom() + require.True(t, m.viewport.AtBottom()) + limitsControllerKey(m, tea.KeyEscape) + require.Equal(t, "form", m.page) + require.Equal(t, 4, m.focus) + require.Equal(t, "125.25", m.draft.inputs[0].Value()) + require.Zero(t, backend.writes.Load()) + }) + } + } + } +} + +func TestLimitsWorkspaceListStatusStaysVisible(t *testing.T) { + for _, complete := range []bool{false, true} { + t.Run(fmt.Sprintf("complete_%t", complete), func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.snapshot.ProjectID = "proj_synthetic" + m.snapshot.Alerts = []gjson.Result{gjson.Parse(limitsControllerAlert)} + m.snapshot.Rates = []gjson.Result{gjson.Parse(limitsControllerRate)} + m.snapshot.AlertsComplete, m.snapshot.RatesComplete = complete, complete + m.snapshot.CostsTotal, m.snapshot.CostsComplete = "1.23", false + for _, section := range []int{1, 2} { + m.section = section + status, label, tab := "Incomplete.", "Alerts loaded: 1", "[Alerts]" + if complete { + status = "Complete." + } + if section == 2 { + label, tab = "Rate records loaded: 1", "[Rates]" + } + for _, colored := range []bool{false, true} { + for _, dark := range []bool{false, true} { + m.color, m.dark = colored, dark + view := ansi.Strip(m.View().Content) + require.Contains(t, view, status+" "+label) + require.Contains(t, view, tab) + require.Contains(t, view, "Reported Costs (incomplete): USD 1.23") + require.Contains(t, view, "q: quit") + require.NotContains(t, view, "true") + require.NotContains(t, view, "false") + require.LessOrEqual(t, len(strings.Split(view, "\n")), m.height) + for _, line := range strings.Split(view, "\n") { + require.LessOrEqual(t, ansi.StringWidth(line), m.width) + } + } + } + } + require.Zero(t, backend.writes.Load()) + }) + } +} + +func TestLimitsWorkspaceReviewShowsExactDollarChanges(t *testing.T) { + for _, tc := range []struct { + name, kind, before, amount, previous, next string + }{ + {"limit", "limit_set", limitsTestLimit, "125.25", "USD 100.00 / month", "USD 125.25 / month"}, + {"alert", "alert_update", limitsControllerAlert, "0.00", "USD 100.00 / month", "USD 0.00 / month"}, + {"new_alert", "alert_create", "", "0.01", "New alert", "USD 0.01 / month"}, + {"unknown_limit", "limit_set", "", "125.25", "Not confirmed", "USD 125.25 / month"}, + {"large_limit", "limit_set", strings.Replace(limitsTestLimit, "10000", "9223372036854775806", 1), "92233720368547758.07", "USD 92233720368547758.06 / month", "USD 92233720368547758.07 / month"}, + {"remove_limit", "limit_delete", limitsTestLimit, "", "USD 100.00 / month", "Remove this resource"}, + {"remove_alert", "alert_delete", limitsControllerAlert, "", "USD 100.00 / month", "Remove this resource"}, + } { + t.Run(tc.name, func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.snapshot.LimitAbsent = tc.before == "" + m.begin(tc.kind, gjson.Parse(tc.before)) + if len(m.draft.inputs) != 0 { + m.draft.inputs[0].SetValue(tc.amount) + if tc.kind == "alert_create" { + m.draft.inputs[1].SetValue("finance@example.test") + } + action := len(m.draft.inputs) + if action == 3 { + action++ + } + m.setFocus(action) + limitsControllerKey(m, tea.KeyEnter) + } + require.Equal(t, "review", m.page, m.formError) + require.False(t, m.apply) + require.Contains(t, m.details, "Before: "+tc.previous) + require.Contains(t, m.details, "After: "+tc.next) + if tc.before != "" { + require.Contains(t, m.details, tc.before) + } + if tc.amount != "" { + require.Contains(t, m.previewCommand, fmt.Sprintf("--threshold-amount %d", m.change.Amount)) + require.Contains(t, m.details, fmt.Sprintf(`"threshold_amount": %d`, m.change.Amount)) + } + require.Contains(t, m.details, "\n"+m.previewCommand+"\n") + require.Zero(t, backend.writes.Load()) + }) + } +} + func TestLimitsDraftCommandPreviewPreservesPublicRoutes(t *testing.T) { for _, tc := range []struct { change limitsChange @@ -1064,6 +1335,10 @@ func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { t.Fatal("HTTP failure removed mutation uncertainty or the write lock") } limitsControllerKey(m, tea.KeyEscape) + if m.page != "form" { + t.Fatal("recovery help did not restore the locked draft") + } + limitsControllerKey(m, tea.KeyEscape) limitsControllerKey(m, tea.KeyEnter) m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) limitsControllerKey(m, tea.KeyEnter) @@ -1159,6 +1434,10 @@ func TestLimitsWorkspaceExitRecoveryUsesProcessActions(t *testing.T) { } } limitsControllerKey(m, tea.KeyEscape) + if m.page != "form" { + t.Fatal("recovery help did not restore the rejected draft") + } + limitsControllerKey(m, tea.KeyEscape) limitsControllerKey(m, 'q') m.lastSaved = gjson.Parse(strings.Replace(limitsTestLimit, "organization.spend_limit", "project.spend_limit", 1)) m.lastSavedOperation = limitsChangeOperation("limit_set", true) diff --git a/pkg/custom/limits_workspace_view.go b/pkg/custom/limits_workspace_view.go index 82b19ca6..af652861 100644 --- a/pkg/custom/limits_workspace_view.go +++ b/pkg/custom/limits_workspace_view.go @@ -47,7 +47,7 @@ func (m *limitsWorkspace) View() tea.View { } for i := range sections { if i == m.section { - sections[i] = "[" + sections[i] + "]" + sections[i] = accent.Render("[" + sections[i] + "]") } } body = append(body, strings.Join(sections, " "), line(m.sourceSummary()), line(m.costsSummary())) @@ -81,25 +81,41 @@ func (m *limitsWorkspace) View() tea.View { case "form": body = append(body, "Tab: next field · Shift+Tab: previous") count := len(m.draft.inputs) - start := max(0, min(m.focus, count-1)-max(0, (m.height-9)/2)) - end := min(count, start+max(1, (m.height-7)/2)) + var notice []string + if m.formError != "" { + first := limitsSafeLine(strings.SplitN(m.formError, "\n", 2)[0]) + notice = strings.Split(ansi.Wrap(first, width, ""), "\n") + if len(notice) > 2 { + notice = []string{notice[0], ansi.Truncate(strings.Join(notice[1:], " "), width, "…")} + } + } + // Reserve navigation, Review, diagnostics, and the optional prefix action before fields. + fieldRows := m.height - 4 - 2 - len(notice) + if count == 3 { + fieldRows-- + } + visible := max(1, fieldRows/2) + start := max(0, min(m.focus, count-1)-visible+1) + end := min(count, start+visible) for i := start; i < end; i++ { m.draft.inputs[i].SetWidth(max(1, width-4)) - marker := " " + marker, label := " ", line(m.draft.labels[i]) if i == m.focus { marker = "> " + label = accent.Render(label) } - body = append(body, line(m.draft.labels[i]), marker+ansi.Truncate(m.draft.inputs[i].View(), width, "…")) + body = append(body, label, marker+ansi.Truncate(m.draft.inputs[i].View(), width, "…")) } if count == 3 { modes := []string{"Keep unless edited", "Replace (empty is allowed)", "Clear to null"} body = append(body, mark("Prefix: "+modes[m.draft.prefixMode], m.focus == count)) } body = append(body, mark("Review change", m.focus >= count && !(count == 3 && m.focus == count))) - if m.formError != "" { - body = append(body, line(strings.SplitN(m.formError, "\n", 2)[0])) - } + body = append(body, notice...) footer = "Enter: next / select\nF1: help Esc: back Ctrl+C: quit" + if m.refreshRequired { + footer = "r: refresh Enter: next field\nF1: help Esc: back Ctrl+C: quit" + } case "review", "details": body = append(body, m.viewport.View()) if m.page == "review" { @@ -111,9 +127,9 @@ func (m *limitsWorkspace) View() tea.View { apply = " Read to the end before applying" } body = append(body, line(back+" "+apply)) - footer = "↑/↓: scroll Tab: action Enter: select\nEsc: back Ctrl+C: quit" + footer = "↑/↓: scroll Tab: action Enter: select\nF1: help Esc: back Ctrl+C: quit" if m.previewCommand != "" { - footer = "↑/↓: scroll Tab: action Enter: select\nCtrl+P: print command and exit\nEsc: back Ctrl+C: quit" + footer = "↑/↓: scroll Tab: action Enter: select\nCtrl+P: print command and exit\nF1: help Esc: back Ctrl+C: quit" } } else { footer = "↑/↓: scroll Esc: back Ctrl+C: quit" diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index 5c3f34da..97ce1d29 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -36,7 +36,7 @@ The isolated child environment includes only fake credentials and the fixture en | `org-limit` | Organization | Replace the hard limit with USD 125.25 per month | | `org-limit-delete` | Organization | Remove the hard limit and retain its confirmed receipt after a missing-record refresh | | `project-limit` | Project | Replace the hard limit with USD 125.25 per month | -| `alert-create` | Organization | Create a USD 75.50 monthly alert with synthetic email settings | +| `alert-create` | Organization | Correct an invalid amount, retain the draft through help and Back, then create a USD 75.50 alert | | `alert-print` | Organization | Print the complete reviewed alert command after terminal restoration; perform no mutation | | `alert-edit` | Project | Change the alert to USD 150.25 and clear its prefix to null | | `alert-delete` | Organization | Delete the selected synthetic alert | @@ -48,6 +48,10 @@ The isolated child environment includes only fake credentials and the fixture en Each mutation scene opens a form or deletion review. The driver reads to the review's end before selecting Apply. It checks that the fixture receives no mutation before confirmation. +The create-alert scene checks visible validation errors and complete help text. +Repeated help preserves the draft, prefix mode, and selected Review action. +Escape from review returns to the draft. Help from review clears Apply selection. +The final request verifies the retained amount, recipient, and prefix. The uncertain scene checks one mutation, retained failure guidance, and exit status 1. The cancellation scene waits for the fixture's request marker and requires exit status 130. Successful mutation scenes require exit status 0 and one mutation each. @@ -56,6 +60,7 @@ It compares the complete printed command bytes after terminal restoration, inclu Wrapped inspection rows do not establish command copyability. The preflight scene loads the overview before returning HTTP 400 from the confirmation read. It retains the draft, blocks review before refresh, and checks F1 recovery at both widths. +Escape from help returns to the retained draft before another Escape opens the overview. Normal exit preserves the failure and hides private response data. No mutation occurs. Final recovery tells users to reopen the workspace and preserve the original request scope. The recorder rejects workspace keyboard actions in failure diagnostics after terminal restoration. diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index 15b9eb76..0c6af77e 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -176,6 +176,18 @@ def review(terminal, evidence, scene, output): return key(terminal, evidence, "Tab to Apply", b"\t") wait_screen(terminal, ["> Apply change"]) + if scene == "alert-create": + checkpoint(terminal, evidence, output, "review-before-help", ("> Apply change",)) + key(terminal, evidence, "Review help", b"\x1bOP") + wait_screen(terminal, ["Spend manages hard monthly"]) + checkpoint(terminal, evidence, output, "review-help") + key(terminal, evidence, "Return to review", b"\x1b") + wait_screen(terminal, ["> Back", "Apply change"], ["> Apply change"]) + checkpoint(terminal, evidence, output, "review-return", ("> Back",)) + if mutations(scene): + raise AssertionError("returning from help applied the draft") + key(terminal, evidence, "Select Apply again", b"\t") + wait_screen(terminal, ["> Apply change"]) checkpoint(terminal, evidence, output, "review-apply") key(terminal, evidence, "Apply", b"\r") @@ -209,7 +221,8 @@ def run_scene(terminal, scene, evidence, output): else: key(terminal, evidence, "Edit alert", b"\r") wait_screen(terminal, ["Monthly USD", "Review change"]) - replace(terminal, evidence, "Monthly USD", "75.50" if creating else "150.25") + amount = "75.505" if scene == "alert-create" else "75.50" if creating else "150.25" + replace(terminal, evidence, "Monthly USD", amount) key(terminal, evidence, "Recipients", b"\t") if creating: replace(terminal, evidence, "Synthetic recipient", "finance@example.test") @@ -224,6 +237,35 @@ def run_scene(terminal, scene, evidence, output): key(terminal, evidence, "Review action", b"\t") checkpoint(terminal, evidence, output, "form") key(terminal, evidence, "Review", b"\r") + if scene == "alert-create": + wait_screen(terminal, ["Enter a USD amount", "Review change"]) + checkpoint(terminal, evidence, output, "validation", ("Enter a USD amount",)) + if mutations(scene): + raise AssertionError("an invalid amount sent a mutation") + key(terminal, evidence, "Validation help", b"\x1bOP") + wait_screen(terminal, ["Draft details", "two decimal places."]) + checkpoint(terminal, evidence, output, "validation-help", ("two decimal places.",)) + key(terminal, evidence, "Repeated help", b"\x1bOP") + key(terminal, evidence, "Return to draft", b"\x1b") + wait_screen(terminal, ["> Review change", "Demo", "Enter a USD amount"]) + checkpoint(terminal, evidence, output, "draft-return", ("> Review change", "Demo")) + for _ in range(4): + key(terminal, evidence, "Previous alert field", b"\x1b[Z") + wait_screen(terminal, ["Monthly USD", "75.505"]) + replace(terminal, evidence, "Correct Monthly USD", "75.50") + for _ in range(4): + key(terminal, evidence, "Next alert field", b"\t") + key(terminal, evidence, "Review corrected draft", b"\r") + wait_screen(terminal, ["> Back", "Enter: select"]) + key(terminal, evidence, "Escape from review", b"\x1b") + wait_screen(terminal, ["> Review change", "Demo"]) + for _ in range(4): + key(terminal, evidence, "Inspect retained alert amount", b"\x1b[Z") + wait_screen(terminal, ["Monthly USD", "75.50", "Review change"]) + checkpoint(terminal, evidence, output, "review-back", ("Monthly USD", "75.50")) + for _ in range(4): + key(terminal, evidence, "Return to review action", b"\t") + key(terminal, evidence, "Review retained draft", b"\r") elif scene == "org-limit-delete": key(terminal, evidence, "Select removal", b"\x1b[B") wait_screen(terminal, ["Remove hard spend limit"]) @@ -287,7 +329,10 @@ def run_scene(terminal, scene, evidence, output): if "".join(message.split()) not in compact_help: raise AssertionError("preflight help omitted status or recovery guidance") checkpoint(terminal, evidence, output, "preflight-help", ("HTTP 400",)) - key(terminal, evidence, "Back", b"\x1b") + key(terminal, evidence, "Return to retained draft", b"\x1b") + wait_screen(terminal, ["125.25", "Review change"]) + checkpoint(terminal, evidence, output, "preflight-draft", ("125.25", "Review change")) + key(terminal, evidence, "Back to overview", b"\x1b") wait_screen(terminal, ["Set hard spend limit"]) key(terminal, evidence, "Quit", b"q") terminal.finish(1) diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 477f0f07..8dfe0c13 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -144,14 +144,15 @@ def main(): limit_reads = [entry for entry in scene_requests if entry["path"].endswith("/spend_limit")] assert [entry["status"] for entry in limit_reads] == [200, 400], (scene, "preflight did not follow a successful overview") assert all(entry["status"] == 200 for entry in scene_requests if not entry["path"].endswith("/spend_limit")) - required = ("review-command", "review-apply", "preflight", "refresh-required", "preflight-help") + required = ("review-command", "review-apply", "preflight", "refresh-required", "preflight-help", "preflight-draft") assert all(name in checkpoints for name in required) assert all(checkpoints[left]["seconds"] < checkpoints[right]["seconds"] for left, right in zip(required, required[1:])) keys = [event["key"] for event in evidence["keys"]] assert keys.count("Apply") == keys.count("Blocked review before refresh") == keys.count("F1 recovery") == 1 for name, messages in (("preflight", ("HTTP 400", "125.25", "Review change")), ("refresh-required", ("Refresh with r", "125.25", "Review change")), - ("preflight-help", ("HTTP 400", "No write sent.", "request fields and scope", "press r to refresh"))): + ("preflight-help", ("HTTP 400", "No write sent.", "request fields and scope", "press r to refresh")), + ("preflight-draft", ("125.25", "Review change"))): screen = (root / (scene + "-evidence." + name + ".txt")).read_text() compact = "".join(screen.split()) assert all("".join(message.split()) in compact for message in messages), (scene, name, "missing recovery evidence") @@ -164,6 +165,18 @@ def main(): b"original connection, authentication, and scope options")) assert b"may have succeeded" not in diagnostic continue + if scene == "alert-create": + required = ("validation", "validation-help", "draft-return", "review-back", + "review-top", "review-before-help", "review-help", "review-return", "review-apply", "saved") + assert all(name in checkpoints for name in required) + assert all(checkpoints[left]["seconds"] < checkpoints[right]["seconds"] for left, right in zip(required, required[1:])) + for name, messages in (("validation", ("Enter a USD amount", "Review change")), + ("validation-help", ("Draft details", "two decimal places.")), + ("draft-return", ("> Review change", "Demo")), + ("review-back", ("Monthly USD", "75.50")), + ("review-return", ("> Back", "Apply change"))): + screen = (root / (scene + "-evidence." + name + ".txt")).read_text() + assert all(message in screen for message in messages), (scene, name, "missing draft recovery evidence") request = mutations[0] if scene in {"org-limit", "project-limit", "uncertain"}: assert request["body"] == {"currency": "USD", "interval": "month", "threshold_amount": 12525} From 29056f28e7aa6164c996985fe594075079f8f01b Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 00:48:36 -0700 Subject: [PATCH 28/44] Make Limits compact and complete project rate controls --- cmd/openai/limits_workflow_test.go | 50 ++++++++ cmd/openai/limits_workflow_unix_test.go | 8 +- docs/limits-workspace.md | 34 ++++- pkg/custom/limits_commands.go | 17 +++ pkg/custom/limits_draft.go | 25 +++- pkg/custom/limits_service.go | 11 +- pkg/custom/limits_service_test.go | 80 +++++++++++- pkg/custom/limits_values.go | 24 +++- pkg/custom/limits_values_test.go | 2 +- pkg/custom/limits_workspace.go | 83 ++++++++++-- pkg/custom/limits_workspace_test.go | 141 ++++++++++++++++++++- pkg/custom/limits_workspace_view.go | 77 ++++++++--- scripts/demos/limits-workspace/README.md | 9 +- scripts/demos/limits-workspace/record.sh | 3 +- scripts/demos/limits-workspace/scene.py | 25 ++-- scripts/demos/limits-workspace/server.py | 6 +- scripts/demos/limits-workspace/validate.py | 23 +++- 17 files changed, 548 insertions(+), 70 deletions(-) diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go index 49d743d2..2efab4ed 100644 --- a/cmd/openai/limits_workflow_test.go +++ b/cmd/openai/limits_workflow_test.go @@ -610,3 +610,53 @@ func TestMainLimitsWorkflowCostsPrecisionAndPartialFailure(t *testing.T) { }) } } + +func TestMainLimitsTokensPerDayFlagAndStdin(t *testing.T) { + for _, test := range []struct{ name, input, value, want string }{ + {"flag zero", "", "0", "0"}, + {"flag signed", "", "-1", "-1"}, + {"flag exact", "", "9223372036854775807", "9223372036854775807"}, + {"JSON", `{"max_tokens_per_1_day":9007199254740993,"future":{"kept":true}}`, "", "9007199254740993"}, + {"YAML", "max_tokens_per_1_day: 9007199254740993\nfuture:\n kept: true\n", "", "9007199254740993"}, + {"explicit override", `{"max_tokens_per_1_day":9007199254740993,"future":{"kept":true}}`, "0", "0"}, + } { + t.Run(test.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodPost || r.URL.Path != "/organization/projects/proj_synthetic/rate_limits/rl_synthetic" { + t.Errorf("unexpected rate route: %s %s", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer sk-fake-spend-admin" { + t.Error("rate update did not use the admin credential") + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + value := gjson.ParseBytes(body) + if value.Get("max_tokens_per_1_day").Raw != test.want { + t.Errorf("daily tokens changed: %s", body) + } + if test.input != "" && value.Get("future.kept").Type != gjson.True { + t.Error("stdin lost an unfamiliar field") + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"id":"rl_synthetic","object":"project.rate_limit","model":"synthetic-model","max_tokens_per_1_day":%s}`, test.want) + })) + defer server.Close() + args := []string{"openai", "--format", "json", "projects", "rate-limits", "update-rate-limit", "--project-id", "proj_synthetic", "--rate-limit-id", "rl_synthetic"} + if test.value != "" { + args = append(args, "--max-tokens-per-1-day="+test.value) + } + got := runMainDispatchWithStdin(t, "bash", []string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "NO_COLOR=1"}, shellFileInput(t, []byte(test.input)), args...) + if got.code != 0 || got.stderr != "" || requests.Load() != 1 || gjson.Get(got.stdout, "max_tokens_per_1_day").Raw != test.want { + t.Fatalf("daily token command failed: code=%d stderr=%q requests=%d stdout=%s", got.code, got.stderr, requests.Load(), got.stdout) + } + }) + } + got := runMainDispatch(t, "bash", "openai", "projects", "rate-limits", "update-rate-limit", "--help") + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "--max-tokens-per-1-day") { + t.Fatalf("daily token flag missing from help: %+v", got) + } +} diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go index 4e03e358..71e9fe2f 100644 --- a/cmd/openai/limits_workflow_unix_test.go +++ b/cmd/openai/limits_workflow_unix_test.go @@ -122,11 +122,13 @@ func TestMainLimitsWorkflowBorrowedTerminalWriter(t *testing.T) { if got.Code != test.exit || !strings.Contains(got.Stderr, "borrowed-terminal-verified\n") { t.Fatalf("borrowed terminal changed exit or ownership: %+v", got) } - enter, leave := strings.Index(got.Stdout, "\x1b[?1049h"), strings.LastIndex(got.Stdout, "\x1b[?1049l") - if enter < 0 || leave <= enter || strings.Contains(got.Stdout+got.Stderr, "panic:") || strings.Contains(got.Stderr, "synthetic-private") { + ready := strings.Index(got.Stdout, "Website controls and source errors: ?") + restored := strings.LastIndex(got.Stdout, "\x1b[?25h") + if ready < 0 || restored <= ready || strings.Contains(got.Stdout, "\x1b[?1049h") || strings.Contains(got.Stdout, "\x1b[?1049l") || + strings.Contains(got.Stdout+got.Stderr, "panic:") || strings.Contains(got.Stderr, "synthetic-private") { t.Fatalf("workspace did not restore safe terminal output: %+v", got) } - if test.exit == 0 && (!strings.Contains(got.Stdout[leave:], "Limits workspace closed.") || got.Stderr != "borrowed-terminal-verified\n") { + if test.exit == 0 && (!strings.Contains(got.Stdout[restored:], "Limits workspace closed.") || got.Stderr != "borrowed-terminal-verified\n") { t.Fatalf("workspace did not close normally after restoration: %+v", got) } if test.mode == "write-failure" && got.Stderr != "Could not write command output. Check your terminal, output file, or receiving command.\nborrowed-terminal-verified\n" { diff --git a/docs/limits-workspace.md b/docs/limits-workspace.md index eddd89bb..015f2aa1 100644 --- a/docs/limits-workspace.md +++ b/docs/limits-workspace.md @@ -107,13 +107,40 @@ An empty replacement remains an empty string. The workspace does not require a hard limit before creating an alert. The API still validates account-specific constraints. -Project Rates supports all six public rate fields. +Project Rates supports seven named rate fields, including tokens per day. The form sends only changed fields. Blank fields remain unchanged. Zero has no assumed unlimited meaning. -The API decides which integer values the project permits. +The workspace accepts nonnegative integers for fields returned as integers. +The workspace skips missing and null fields. +Sparse records can expose only daily limits. +Fine-tuned `ft:` models and `-shared` records remain available for inspection. +The API decides account-specific caps. +Scriptable commands retain their existing input behavior. Rate updates require a complete preflight list; use `--max-items -1` when necessary. +The named tokens-per-day flag extends an existing API field: + +```sh +openai projects rate-limits update-rate-limit \ + --project-id proj_example --rate-limit-id rl_example \ + --max-tokens-per-1-day 100000 +``` + +JSON and YAML stdin can also send `max_tokens_per_1_day`. +The workspace verifies the returned model and every requested value before confirming success. +A mismatched confirmation locks further writes and directs you to inspect current settings. + +## Compact terminal interface + +The workspace uses an inline panel and preserves shell scrollback. +The panel uses up to 76 columns and 18 rows. +The minimum terminal size remains 40 columns and 12 rows. +Selection uses a fixed-width marker and keeps the same font weight. +Forms and details open through explicit actions. +Short details use only their content height. +Machine output and redirected output remain read-only. + ## Concurrent changes and recovery The CLI checks the selected record again before writing. @@ -122,6 +149,9 @@ A changed baseline stops the write. A 404 response leaves the current limit unconfirmed. Setting a limit requires reviewing that uncertainty and explicitly approving creation or replacement. Refresh the retained draft before reviewing it again. +If a refreshed rate field becomes unavailable, its edited draft remains in F1 details. +Select Discard unavailable edits to remove those drafts explicitly. +Other edits remain intact. Ambiguous or unfamiliar replacement configuration remains available for inspection. The form refuses edits that could discard that configuration. diff --git a/pkg/custom/limits_commands.go b/pkg/custom/limits_commands.go index 7ee09617..69e77961 100644 --- a/pkg/custom/limits_commands.go +++ b/pkg/custom/limits_commands.go @@ -12,6 +12,7 @@ import ( "github.com/openai/openai-cli/internal/commands/admin" clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-cli/internal/output/transform" + "github.com/openai/openai-cli/internal/requestflag" "github.com/urfave/cli/v3" ) @@ -39,6 +40,22 @@ func registerLimitsCommands(root *cli.Command) { organization.Commands = append(organization.Commands, newLimitsCommand(false)) if projects := organization.Command("projects"); projects != nil { projects.Commands = append(projects.Commands, newLimitsCommand(true)) + if rates := projects.Command("rate-limits"); rates != nil { + if update := rates.Command("update-rate-limit"); update != nil { + found := false + for _, flag := range update.Flags { + for _, name := range flag.Names() { + found = found || name == "max-tokens-per-1-day" + } + } + if !found { + update.Flags = append(update.Flags, &requestflag.Flag[int64]{ + Name: "max-tokens-per-1-day", BodyPath: "max_tokens_per_1_day", + Usage: "The maximum number of tokens per day for this model.", + }) + } + } + } } } diff --git a/pkg/custom/limits_draft.go b/pkg/custom/limits_draft.go index ba9855df..193425da 100644 --- a/pkg/custom/limits_draft.go +++ b/pkg/custom/limits_draft.go @@ -20,6 +20,7 @@ var limitsRateInputs = []struct{ key, label string }{ {"max_images_per_1_minute", "Images / minute"}, {"max_audio_megabytes_per_1_minute", "Audio megabytes / minute"}, {"batch_1_day_max_input_tokens", "Batch input tokens / day"}, + {"max_tokens_per_1_day", "Tokens / day"}, } type limitsDraft struct { @@ -67,8 +68,10 @@ func newLimitsDraft(change limitsChange) (limitsDraft, error) { text := "" if value.Type == gjson.Number { text = value.Raw + add(field.label+" (blank: unchanged)", text) + } else { + add(field.label+" (unavailable for this model)", "") } - add(field.label+" (blank: unchanged)", text) } } for i, input := range d.inputs { @@ -119,6 +122,9 @@ func (d limitsDraft) validate() (limitsChange, error) { if value == "" || value == d.initial[i] { continue } + if c.Before.Get(field.key).Type != gjson.Number { + return c, fmt.Errorf("%s is unavailable for this model. Inspect the original API record.", field.label) + } digits := strings.TrimPrefix(value, "-") if digits == "" { return c, fmt.Errorf("%s needs an integer.", field.label) @@ -132,6 +138,9 @@ func (d limitsDraft) validate() (limitsChange, error) { if err != nil { return c, fmt.Errorf("%s exceeds the supported integer range.", field.label) } + if number < 0 { + return c, fmt.Errorf("%s cannot be negative. Enter zero or a positive integer.", field.label) + } c.RateValues[field.key] = number } if len(c.RateValues) == 0 { @@ -141,6 +150,20 @@ func (d limitsDraft) validate() (limitsChange, error) { return c, nil } +func (d limitsDraft) inputAvailable(index int) bool { + return d.change.Kind != "rate_update" || index >= len(d.inputs) || + d.change.Before.Get(limitsRateInputs[index].key).Type == gjson.Number +} + +func (d limitsDraft) hasUnavailableEdits() bool { + for i := range d.inputs { + if !d.inputAvailable(i) && d.inputs[i].Value() != "" && d.inputs[i].Value() != d.initial[i] { + return true + } + } + return false +} + func limitsChangeTitle(kind string) string { switch kind { case "limit_set": diff --git a/pkg/custom/limits_service.go b/pkg/custom/limits_service.go index 472f73a7..077cbb45 100644 --- a/pkg/custom/limits_service.go +++ b/pkg/custom/limits_service.go @@ -519,6 +519,8 @@ func (backend *limitsAPIBackend) Mutate(ctx context.Context, change limitsChange params.MaxAudioMegabytesPer1Minute = openai.Int(value) case "batch_1_day_max_input_tokens": params.Batch1DayMaxInputTokens = openai.Int(value) + case "max_tokens_per_1_day": + options = append(options, option.WithJSONSet(field, value)) } } attempted = true @@ -575,8 +577,10 @@ func limitsMutationConfirmedContext(ctx context.Context, value gjson.Result, cha return ctx.Err() } if resource == "rate_limit" { - if value.Get("model").Type != gjson.String || value.Get("model").Str == "" || - value.Get("max_requests_per_1_minute").Type != gjson.Number || value.Get("max_tokens_per_1_minute").Type != gjson.Number { + if err := limitsEditableBaselineContext(ctx, value, "rate_update"); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } return invalid } } else { @@ -593,6 +597,9 @@ func limitsMutationConfirmedContext(ctx context.Context, value gjson.Result, cha } mismatch := errors.New("The API returned different values from the reviewed change. Inspect the returned record before another change.") if resource == "rate_limit" { + if value.Get("model").Str != change.Before.Get("model").Str { + return mismatch + } for field, requested := range change.RateValues { if err := ctx.Err(); err != nil { return err diff --git a/pkg/custom/limits_service_test.go b/pkg/custom/limits_service_test.go index ff52eb8c..20e193c5 100644 --- a/pkg/custom/limits_service_test.go +++ b/pkg/custom/limits_service_test.go @@ -181,13 +181,14 @@ func TestLimitsServiceAlertPrefixPresenceAndNoMutationRetries(t *testing.T) { } } -func TestLimitsServiceRateUpdatePreservesZeroNegativeAndUnknownResponseFields(t *testing.T) { +func TestLimitsServiceRateUpdatePreservesZeroPrecisionAndUnknownResponseFields(t *testing.T) { var body []byte + baseline := strings.Replace(limitsTestRate, `"future":`, `"max_requests_per_1_day":1,"max_images_per_1_minute":2,"max_audio_megabytes_per_1_minute":3,"batch_1_day_max_input_tokens":4,"max_tokens_per_1_day":5,"future":`, 1) backend := limitsTestBackend(t, "proj_demo", 100, func(w http.ResponseWriter, r *http.Request) { if strings.HasSuffix(r.URL.Path, "/proj_demo") { limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) } else if r.Method == http.MethodGet { - limitsTestWrite(t, w, `{"object":"list","data":[`+limitsTestRate+`],"has_more":false}`) + limitsTestWrite(t, w, `{"object":"list","data":[`+baseline+`],"has_more":false}`) } else { var err error body, err = io.ReadAll(r.Body) @@ -196,7 +197,7 @@ func TestLimitsServiceRateUpdatePreservesZeroNegativeAndUnknownResponseFields(t } var updated map[string]json.RawMessage var changed map[string]json.RawMessage - if err := json.Unmarshal([]byte(limitsTestRate), &updated); err != nil { + if err := json.Unmarshal([]byte(baseline), &updated); err != nil { t.Error(err) } if err := json.Unmarshal(body, &changed); err != nil { @@ -212,13 +213,15 @@ func TestLimitsServiceRateUpdatePreservesZeroNegativeAndUnknownResponseFields(t limitsTestWrite(t, w, string(encoded)) } }) - value, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(limitsTestRate), - RateValues: map[string]int64{"max_requests_per_1_minute": 0, "max_tokens_per_1_minute": -1, + value, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(baseline), + RateValues: map[string]int64{"max_requests_per_1_minute": 0, "max_tokens_per_1_minute": 201, "max_requests_per_1_day": 123, "max_images_per_1_minute": 4, - "max_audio_megabytes_per_1_minute": 5, "batch_1_day_max_input_tokens": 9007199254740993}}) + "max_audio_megabytes_per_1_minute": 5, "batch_1_day_max_input_tokens": 9007199254740993, + "max_tokens_per_1_day": 9223372036854775807}}) require.NoError(t, err) require.Equal(t, "9007199254740993", gjson.GetBytes(body, "batch_1_day_max_input_tokens").Raw) - require.JSONEq(t, `{"max_requests_per_1_minute":0,"max_tokens_per_1_minute":-1,"max_requests_per_1_day":123,"max_images_per_1_minute":4,"max_audio_megabytes_per_1_minute":5,"batch_1_day_max_input_tokens":9007199254740993}`, string(body)) + require.JSONEq(t, `{"max_requests_per_1_minute":0,"max_tokens_per_1_minute":201,"max_requests_per_1_day":123,"max_images_per_1_minute":4,"max_audio_megabytes_per_1_minute":5,"batch_1_day_max_input_tokens":9007199254740993,"max_tokens_per_1_day":9223372036854775807}`, string(body)) + require.Equal(t, "9223372036854775807", gjson.GetBytes(body, "max_tokens_per_1_day").Raw) require.Equal(t, "0.1234567890123456789", value.Get("future.precision").Raw) } @@ -810,3 +813,66 @@ func TestLimitsCostsCancellationRetainsEarlierBuckets(t *testing.T) { require.Equal(t, "canceled", value.Get("source_errors.0.status").Str) require.Equal(t, "0.1", value.Get("costs.buckets.0.results.0.amount.value").Raw) } + +func TestLimitsServiceRejectsUneditableRatesWithoutRequests(t *testing.T) { + for _, test := range []struct { + name, baseline, field string + value int64 + }{ + {"negative", limitsTestRate, "max_requests_per_1_minute", -1}, + {"absent", limitsTestRate, "max_tokens_per_1_day", 1}, + {"null", strings.Replace(limitsTestRate, `200`, `null`, 1), "max_tokens_per_1_minute", 1}, + {"fine tuned", strings.Replace(limitsTestRate, `synthetic-model`, `ft:synthetic-model`, 1), "max_requests_per_1_minute", 1}, + {"shared", strings.Replace(limitsTestRate, `synthetic-model`, `synthetic-shared`, 1), "max_requests_per_1_minute", 1}, + } { + t.Run(test.name, func(t *testing.T) { + var requests atomic.Int32 + backend := limitsTestBackend(t, "proj_demo", 100, func(w http.ResponseWriter, r *http.Request) { requests.Add(1) }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(test.baseline), RateValues: map[string]int64{test.field: test.value}}) + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.False(t, failure.Attempted) + require.Zero(t, requests.Load()) + }) + } +} + +func TestLimitsServiceSparseTokensPerDayConfirmation(t *testing.T) { + const baseline = `{"object":"project.rate_limit","id":"rate_demo","model":"synthetic-model","max_tokens_per_1_day":100,"max_tokens_per_1_minute":null,"future":{"precision":0.1234567890123456789}}` + const updated = `{"object":"project.rate_limit","id":"rate_demo","model":"synthetic-model","max_tokens_per_1_day":9007199254740993,"max_tokens_per_1_minute":null}` + for _, test := range []struct { + name, response string + success bool + }{ + {"matching", updated, true}, + {"different model", strings.Replace(updated, `synthetic-model`, `other-model`, 1), false}, + {"rounded value", strings.Replace(updated, `9007199254740993`, `9007199254740992`, 1), false}, + {"missing value", strings.Replace(updated, `9007199254740993`, `null`, 1), false}, + } { + t.Run(test.name, func(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "proj_demo", 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + limitsTestWrite(t, w, `{"object":"organization.project","id":"proj_demo"}`) + } else if r.Method == http.MethodGet { + limitsTestWrite(t, w, `{"object":"list","data":[`+baseline+`],"has_more":false}`) + } else { + writes.Add(1) + body, err := io.ReadAll(r.Body) + require.NoError(t, err) + require.JSONEq(t, `{"max_tokens_per_1_day":9007199254740993}`, string(body)) + limitsTestWrite(t, w, test.response) + } + }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(baseline), RateValues: map[string]int64{"max_tokens_per_1_day": 9007199254740993}}) + require.Equal(t, int32(1), writes.Load()) + if test.success { + require.NoError(t, err) + } else { + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.True(t, failure.Attempted) + } + }) + } +} diff --git a/pkg/custom/limits_values.go b/pkg/custom/limits_values.go index 6ee70bb4..e3d57fc8 100644 --- a/pkg/custom/limits_values.go +++ b/pkg/custom/limits_values.go @@ -234,14 +234,22 @@ func limitsEditableBaselineContext(ctx context.Context, value gjson.Result, kind value.Get("model").Type != gjson.String || value.Get("model").Str == "" { return errors.New("The returned rate limit has no recognized identity.") } + if limitsRateReadOnly(value) { + return errors.New("This model's rate limit is read-only. Inspect the original API record for its current values.") + } + available := false for _, field := range limitsRateFields { item := value.Get(field) - if item.Exists() || field == "max_requests_per_1_minute" || field == "max_tokens_per_1_minute" { + if item.Exists() && item.Type != gjson.Null { if _, err := limitsInteger(item); err != nil { return err } + available = true } } + if !available { + return errors.New("This record has no editable numeric rate fields. Inspect the original API record.") + } return nil default: return errors.New("Unknown limits edit operation.") @@ -256,6 +264,12 @@ func limitsEditableBaselineContext(ctx context.Context, value gjson.Result, kind var limitsRateFields = []string{ "max_requests_per_1_minute", "max_tokens_per_1_minute", "max_requests_per_1_day", "max_images_per_1_minute", "max_audio_megabytes_per_1_minute", "batch_1_day_max_input_tokens", + "max_tokens_per_1_day", +} + +func limitsRateReadOnly(value gjson.Result) bool { + model := value.Get("model").Str + return strings.HasPrefix(model, "ft:") || strings.HasSuffix(model, "-shared") } func limitsValidateChange(change limitsChange) error { @@ -294,7 +308,7 @@ func limitsValidateChangeContext(ctx context.Context, change limitsChange) error if len(change.RateValues) == 0 { return errors.New("Set at least one rate-limit value.") } - for key := range change.RateValues { + for key, value := range change.RateValues { known := false for _, field := range limitsRateFields { known = known || key == field @@ -302,6 +316,12 @@ func limitsValidateChangeContext(ctx context.Context, change limitsChange) error if !known { return errors.New("This form does not support the requested rate-limit field.") } + if value < 0 { + return errors.New("Rate values cannot be negative. Enter zero or a positive integer.") + } + if change.Before.Get(key).Type != gjson.Number { + return errors.New("This rate field was not returned for this model. Inspect the original API record before editing.") + } } case "limit_delete", "alert_delete": default: diff --git a/pkg/custom/limits_values_test.go b/pkg/custom/limits_values_test.go index 0b66ea14..772527dc 100644 --- a/pkg/custom/limits_values_test.go +++ b/pkg/custom/limits_values_test.go @@ -160,7 +160,7 @@ func TestLimitsMutationConfirmationMatchesReviewedFields(t *testing.T) { require.NoError(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(matching, `"subject_prefix":null`, `"subject_prefix":""`, 1)), change, false)) change.SubjectPrefixSet = false require.NoError(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(matching, `,"subject_prefix":null`, ``, 1)), change, false)) - rateChange := limitsChange{Kind: "rate_update", ID: "rate_demo", RateValues: map[string]int64{"max_requests_per_1_minute": 9007199254740993}} + rateChange := limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(limitsTestRate), RateValues: map[string]int64{"max_requests_per_1_minute": 9007199254740993}} rate := strings.Replace(limitsTestRate, `"max_requests_per_1_minute":100`, `"max_requests_per_1_minute":9007199254740993`, 1) require.NoError(t, limitsMutationConfirmed(gjson.Parse(rate), rateChange, true)) require.Error(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(rate, "9007199254740993", "9007199254740992", 1)), rateChange, true)) diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go index e22e64ab..26c24570 100644 --- a/pkg/custom/limits_workspace.go +++ b/pkg/custom/limits_workspace.go @@ -54,6 +54,7 @@ type limitsWorkspace struct { width, height, section, selection, focus int page, note, formError string details string + detailsLines int busy, quitting, interrupted, locked, apply bool refreshRequired, refreshingDraft, color, dark bool lastSaved gjson.Result @@ -399,6 +400,24 @@ func (m *limitsWorkspace) reconcileDraft() { } change.Before, change.BeforeMissing = current, false m.draft.change, m.refreshRequired, m.formError = change, false, "Baseline refreshed. Review the before and after values." + if change.Kind == "rate_update" { + for i, field := range limitsRateInputs { + if m.draft.inputs[i].Value() == "" || m.draft.inputs[i].Value() == m.draft.initial[i] { + value := "" + if current.Get(field.key).Type == gjson.Number { + value = current.Get(field.key).Raw + } + m.draft.inputs[i].SetValue(value) + m.draft.initial[i] = value + } + } + if m.draft.hasUnavailableEdits() { + m.formError = "Some edited rates are unavailable. Inspect F1, then discard those edits to continue." + m.setFocus(len(m.draft.inputs)) + } else { + m.setFocus(m.focus) + } + } } func (m *limitsWorkspace) menu() []limitsMenuItem { @@ -418,13 +437,23 @@ func (m *limitsWorkspace) menu() []limitsMenuItem { } case 2: for _, rate := range m.snapshot.Rates { - rows = append(rows, limitsMenuItem{label: rate.Get("model").String() + " " + rate.Get("id").String(), kind: "rate_update", value: rate}) + label, kind := rate.Get("model").String()+" "+rate.Get("id").String(), "rate_update" + if limitsRateReadOnly(rate) { + label, kind = label+" (read only)", "rate_inspect" + } + rows = append(rows, limitsMenuItem{label: label, kind: kind, value: rate}) } } return rows } func (m *limitsWorkspace) begin(kind string, value gjson.Result) { + if kind == "rate_inspect" || kind == "rate_update" && limitsRateReadOnly(value) { + m.detailsReturnPage = "home" + m.page, m.detailsHeading = "details", "Read-only model rate limit" + m.setDetails(limitsSafeJSON(value)) + return + } if m.locked { m.note = "Writes are locked after a failed save. Inspect the resource before reopening." return @@ -461,7 +490,17 @@ func (m *limitsWorkspace) setFocus(next int) { if len(m.draft.inputs) == 3 { count++ } + if m.draft.hasUnavailableEdits() { + count++ + } + direction := 1 + if next < m.focus { + direction = -1 + } m.focus = (next + count) % count + for !m.draft.inputAvailable(m.focus) { + m.focus = (m.focus + direction + count) % count + } if m.focus < len(m.draft.inputs) { m.draft.inputs[m.focus].Focus() } @@ -487,17 +526,25 @@ func (m *limitsWorkspace) setReview() { } func (m *limitsWorkspace) detailsHeight() int { + height := max(1, m.panelHeight()-3) if m.page == "review" && m.previewCommand != "" { - return max(1, m.height-6) + height = max(1, m.panelHeight()-6) + } else if m.page == "review" { + height = max(1, m.panelHeight()-5) } - return max(1, m.height-5) + return min(height, max(1, m.detailsLines)) } +func (m *limitsWorkspace) panelWidth() int { return min(m.width, 76) } +func (m *limitsWorkspace) panelHeight() int { return min(m.height, 18) } + func (m *limitsWorkspace) setDetails(content string) { m.details = content - m.viewport.SetWidth(max(1, m.width-2)) + wrapped := ansi.Hardwrap(content, max(1, m.panelWidth()-2), true) + m.detailsLines = strings.Count(wrapped, "\n") + 1 + m.viewport.SetWidth(max(1, m.panelWidth()-2)) m.viewport.SetHeight(m.detailsHeight()) - m.viewport.SetContent(ansi.Hardwrap(content, max(1, m.width-2), true)) + m.viewport.SetContent(wrapped) m.viewport.GotoTop() } @@ -519,7 +566,7 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { m.detailsReturnOffset = 0 m.setDetails(m.details) } - m.viewport.SetWidth(max(1, msg.Width-2)) + m.viewport.SetWidth(max(1, m.panelWidth()-2)) m.viewport.SetHeight(m.detailsHeight()) return m, nil case limitsWorkResult: @@ -653,7 +700,16 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { m.setFocus(m.focus - 1) return m, nil case "enter": - if m.focus == len(m.draft.inputs) && len(m.draft.inputs) == 3 { + if m.focus == len(m.draft.inputs) && m.draft.hasUnavailableEdits() { + for i := range m.draft.inputs { + if !m.draft.inputAvailable(i) { + m.draft.inputs[i].SetValue("") + m.draft.initial[i] = "" + } + } + m.formError = "Unavailable edits discarded. Review the remaining values." + m.setFocus(len(m.draft.inputs)) + } else if m.focus == len(m.draft.inputs) && len(m.draft.inputs) == 3 { m.draft.prefixMode = (m.draft.prefixMode + 1) % 3 } else if m.focus >= len(m.draft.inputs) { if !m.locked { @@ -702,7 +758,7 @@ func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { } return m, command } - if m.page == "form" && m.focus < len(m.draft.inputs) && !m.small() { + if m.page == "form" && m.focus < len(m.draft.inputs) && m.draft.inputAvailable(m.focus) && !m.small() { var command tea.Cmd m.draft.inputs[m.focus], command = m.draft.inputs[m.focus].Update(message) return m, command @@ -870,7 +926,16 @@ func (m *limitsWorkspace) help() string { if invocation == "" { invocation = "openai" } - text := "Spend manages hard monthly USD limits.\nAlerts notify recipients; they do not enforce caps.\nRates manages six project rate fields.\n\nEvery change requires a before/after review.\nRead to the end, then Tab to Apply.\nBack remains the default action.\nCtrl+P prints the reviewed command and exits. Printing sends no change.\n\nUse r to refresh. Use i to inspect original fields.\nUse --max-items -1 to load complete lists.\n\n" + text := "Spend manages hard monthly USD limits.\nAlerts notify recipients; they do not enforce caps.\nRates manages seven project rate fields.\n\nEvery change requires a before/after review.\nRead to the end, then Tab to Apply.\nBack remains the default action.\nCtrl+P prints the reviewed command and exits. Printing sends no change.\n\nUse r to refresh. Use i to inspect original fields.\nUse --max-items -1 to load complete lists.\n\n" + if m.draft.hasUnavailableEdits() { + text += "Unavailable rate drafts remain saved until you select Discard unavailable edits.\n" + for i, field := range limitsRateInputs { + if !m.draft.inputAvailable(i) && m.draft.inputs[i].Value() != "" && m.draft.inputs[i].Value() != m.draft.initial[i] { + text += field.label + ": " + readable.Text(m.draft.inputs[i].Value()) + "\n" + } + } + text += "Other edited fields remain unchanged.\n\n" + } if m.locked { m.mu.Lock() failure := m.mutationError diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go index a19979f0..d96052a6 100644 --- a/pkg/custom/limits_workspace_test.go +++ b/pkg/custom/limits_workspace_test.go @@ -794,23 +794,23 @@ func TestLimitsDraftRatesSendOnlyExplicitChanges(t *testing.T) { } draft.inputs[0].SetValue("") // Blank retains the existing value. draft.inputs[1].SetValue("201") - draft.inputs[4].SetValue("0") // Explicit zero remains a real change. + draft.inputs[3].SetValue("1") change, err := draft.validate() - want := map[string]int64{"max_tokens_per_1_minute": 201, "max_audio_megabytes_per_1_minute": 0} + want := map[string]int64{"max_tokens_per_1_minute": 201, "max_images_per_1_minute": 1} if err != nil || !reflect.DeepEqual(change.RateValues, want) { t.Fatalf("rate changes=%v error=%v; want %v", change.RateValues, err, want) } for _, tc := range []struct { text string value int64 - }{{"-1", -1}, {"-9223372036854775808", math.MinInt64}} { + }{{"0", 0}, {"9223372036854775807", math.MaxInt64}} { draft.inputs[1].SetValue(tc.text) change, err := draft.validate() if err != nil || change.RateValues["max_tokens_per_1_minute"] != tc.value { - t.Errorf("signed rate %q was not preserved: %v, %v", tc.text, change.RateValues, err) + t.Errorf("nonnegative rate %q was not preserved: %v, %v", tc.text, change.RateValues, err) } } - for _, invalid := range []string{"-", "1.5", "1e3", "9223372036854775808", "-9223372036854775809"} { + for _, invalid := range []string{"-1", "-9223372036854775808", "-", "1.5", "1e3", "9223372036854775808", "-9223372036854775809"} { draft.inputs[1].SetValue(invalid) if _, err := draft.validate(); err == nil { t.Errorf("rate draft accepted %q", invalid) @@ -1067,12 +1067,12 @@ func TestLimitsWorkspaceListStatusStaysVisible(t *testing.T) { m.snapshot.CostsTotal, m.snapshot.CostsComplete = "1.23", false for _, section := range []int{1, 2} { m.section = section - status, label, tab := "Incomplete.", "Alerts loaded: 1", "[Alerts]" + status, label, tab := "Incomplete.", "Alerts loaded: 1", "> Alerts" if complete { status = "Complete." } if section == 2 { - label, tab = "Rate records loaded: 1", "[Rates]" + label, tab = "Rate records loaded: 1", "> Rates" } for _, colored := range []bool{false, true} { for _, dark := range []bool{false, true} { @@ -1867,3 +1867,130 @@ func TestLimitsWorkspaceAmbiguousEnforcementStaysUnknown(t *testing.T) { t.Fatalf("workspace selected one conflicting enforcement value: %q", view) } } + +func TestLimitsWorkspaceSparseAndReadOnlyRates(t *testing.T) { + const daily = `{"object":"project.rate_limit","id":"rl_daily","model":"daily-model","max_tokens_per_1_day":100,"max_tokens_per_1_minute":null}` + for _, size := range [][2]int{{40, 12}, {160, 60}} { + m, backend := newLimitsController(t, size[0], size[1]) + m.snapshot.ProjectID = "proj_demo" + m.begin("rate_update", gjson.Parse(daily)) + require.Equal(t, "form", m.page) + require.Equal(t, 6, m.focus, "only the returned numeric field is editable") + require.Contains(t, ansi.Strip(m.View().Content), "Tokens / day") + require.NotContains(t, ansi.Strip(m.View().Content), "Requests / minute") + m.draft.inputs[6].SetValue("9007199254740993") + limitsControllerKey(m, tea.KeyTab) + require.Equal(t, 7, m.focus) + m.Update(tea.KeyPressMsg{Code: tea.KeyTab, Mod: tea.ModShift}) + require.Equal(t, 6, m.focus) + change, err := m.draft.validate() + require.NoError(t, err) + require.Equal(t, map[string]int64{"max_tokens_per_1_day": 9007199254740993}, change.RateValues) + for _, index := range []int{0, 1} { + m.draft.inputs[index].SetValue("1") + _, err := m.draft.validate() + require.ErrorContains(t, err, "unavailable") + m.draft.inputs[index].SetValue("") + } + for _, model := range []string{"ft:synthetic", "synthetic-shared"} { + rate := gjson.Parse(strings.Replace(daily, "daily-model", model, 1)) + m.page, m.section, m.selection = "home", 2, 0 + m.snapshot.Rates = []gjson.Result{rate} + require.Equal(t, "rate_inspect", m.menu()[0].kind) + require.Contains(t, m.menu()[0].label, "read only") + limitsControllerKey(m, tea.KeyEnter) + require.Equal(t, "details", m.page) + require.Contains(t, m.details, model) + m.begin("rate_update", rate) + require.Equal(t, "details", m.page) + } + require.Zero(t, backend.writes.Load()) + require.Zero(t, backend.reads.Load()) + } +} + +func TestLimitsWorkspaceCompactGeometryAndReviewGate(t *testing.T) { + for _, size := range [][2]int{{40, 12}, {80, 24}, {200, 80}} { + m, backend := newLimitsController(t, size[0], size[1]) + m.snapshot.ProjectID = "proj_demo" + m.snapshot.Limit = gjson.Parse(limitsTestLimit) + m.snapshot.Alerts = []gjson.Result{gjson.Parse(limitsControllerAlert)} + m.snapshot.Rates = []gjson.Result{gjson.Parse(limitsControllerRate)} + rows, columns := 0, 0 + for section := 0; section < 3; section++ { + m.section, m.selection = section, 0 + view := m.View() + require.False(t, view.AltScreen) + require.NotContains(t, view.Content, "\x1b[1m") + lines := strings.Split(ansi.Strip(view.Content), "\n") + if section == 0 { + rows, columns = len(lines), ansi.StringWidth(lines[1]) + } + require.Len(t, lines, rows, "tab changes must preserve panel height") + require.Equal(t, columns, ansi.StringWidth(lines[1]), "tab markers must reserve the same cells") + require.LessOrEqual(t, len(lines), min(size[1], 18)) + for _, line := range lines { + require.LessOrEqual(t, ansi.StringWidth(line), min(size[0], 76)) + } + } + m.begin("limit_set", m.snapshot.Limit) + m.draft.inputs[0].SetValue("100.01") + m.review() + require.Equal(t, "review", m.page) + require.Equal(t, min(size[0], 76)-2, m.viewport.Width()) + require.LessOrEqual(t, m.viewport.Height(), min(size[1], 18)-5) + require.False(t, m.viewport.AtBottom(), "clamped review must not approve unread content") + limitsControllerKey(m, tea.KeyTab) + require.False(t, m.apply) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + require.True(t, m.apply) + m.Update(tea.WindowSizeMsg{Width: 180, Height: 70}) + require.False(t, m.apply) + require.False(t, m.viewport.AtBottom()) + require.Zero(t, backend.writes.Load()) + } +} + +func TestLimitsWorkspaceRefreshRetainsUnavailableRateDraftUntilDiscarded(t *testing.T) { + for _, current := range []string{ + `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_tokens_per_1_minute":300}`, + `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":null,"max_tokens_per_1_minute":300}`, + } { + m, backend := newLimitsController(t, 40, 12) + m.begin("rate_update", gjson.Parse(limitsControllerRate)) + m.draft.inputs[0].SetValue("101") + m.draft.inputs[1].SetValue("201") + m.snapshot.Rates = []gjson.Result{gjson.Parse(current)} + m.refreshRequired = true + m.reconcileDraft() + require.False(t, m.refreshRequired) + require.True(t, m.draft.hasUnavailableEdits()) + require.Equal(t, "101", m.draft.inputs[0].Value(), "refresh must not silently discard an edit") + require.Contains(t, m.help(), "Requests / minute: 101") + require.Equal(t, len(m.draft.inputs), m.focus) + require.Contains(t, ansi.Strip(m.View().Content), "> Discard unavailable edits") + require.LessOrEqual(t, len(strings.Split(m.View().Content, "\n")), 12) + _, err := m.draft.validate() + require.ErrorContains(t, err, "unavailable") + limitsControllerKey(m, tea.KeyF1) + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, tea.KeyEnter) + require.False(t, m.draft.hasUnavailableEdits()) + require.Empty(t, m.draft.inputs[0].Value()) + require.Equal(t, "201", m.draft.inputs[1].Value()) + change, err := m.draft.validate() + require.NoError(t, err) + require.Equal(t, map[string]int64{"max_tokens_per_1_minute": 201}, change.RateValues) + require.Zero(t, backend.writes.Load()) + } +} + +func TestLimitsWorkspaceShortDetailsFitContent(t *testing.T) { + m, _ := newLimitsController(t, 120, 50) + m.page = "details" + m.setDetails("One line\nTwo lines") + require.Equal(t, 2, m.viewport.Height()) + require.True(t, m.viewport.AtBottom()) + require.Len(t, strings.Split(ansi.Strip(m.View().Content), "\n"), 5) +} diff --git a/pkg/custom/limits_workspace_view.go b/pkg/custom/limits_workspace_view.go index af652861..2b93f0e2 100644 --- a/pkg/custom/limits_workspace_view.go +++ b/pkg/custom/limits_workspace_view.go @@ -12,7 +12,6 @@ import ( func (m *limitsWorkspace) View() tea.View { view := tea.NewView("") - view.AltScreen = true if m.quitting { return view } @@ -20,8 +19,8 @@ func (m *limitsWorkspace) View() tea.View { view.Content = "Resize to at least 40 x 12.\nCtrl+C: quit" return view } - width := m.width - 2 - accent := lipgloss.NewStyle().Bold(true) + width, height := m.panelWidth()-2, m.panelHeight() + accent := lipgloss.NewStyle() if m.color { color := "#3159BC" if m.dark { @@ -46,11 +45,16 @@ func (m *limitsWorkspace) View() tea.View { sections = append(sections, "Rates") } for i := range sections { + prefix := " " if i == m.section { - sections[i] = accent.Render("[" + sections[i] + "]") + prefix = "> " + sections[i] = accent.Render(prefix + sections[i]) + } else { + sections[i] = prefix + sections[i] } } body = append(body, strings.Join(sections, " "), line(m.sourceSummary()), line(m.costsSummary())) + enforcement := "" if m.section == 0 && m.snapshot.Limit.Exists() { state := m.snapshot.Limit.Get("enforcement.status") status := "not reported" @@ -59,13 +63,20 @@ func (m *limitsWorkspace) View() tea.View { } else if state.Type == gjson.String { status = state.String() } - body = append(body, line("Returned enforcement: "+status)) + enforcement = line("Returned enforcement: " + status) } + body = append(body, enforcement) rows := m.menu() - room := max(1, m.height-6-len(body)) + // Reserve the same menu rows across tabs, without filling the terminal. + room := max(2, len(m.snapshot.Alerts)+1) + if m.snapshot.ProjectID != "" { + room = max(room, len(m.snapshot.Rates)) + } + budget := height - 10 if m.locked { - room = max(1, room-1) + budget-- } + room = max(1, min(room, budget)) start := max(0, m.selection-room+1) for i := start; i < min(len(rows), start+room); i++ { body = append(body, mark(rows[i].label, i == m.selection)) @@ -73,11 +84,14 @@ func (m *limitsWorkspace) View() tea.View { if len(rows) == 0 { body = append(body, "No editable records loaded.") } + for i := max(1, min(len(rows), room)); i < room; i++ { + body = append(body, "") + } body = append(body, line(m.note)) if m.locked { body = append(body, "Writes locked. Inspect before retrying.") } - footer = "←/→: section ↑/↓: select Enter: edit\nr: refresh i: inspect d: delete alert\nq: quit ?: help Ctrl+C: interrupt" + footer = "←/→ section ↑/↓ select Enter open\nr refresh i inspect d delete alert\nq quit ? help Ctrl+C interrupt" case "form": body = append(body, "Tab: next field · Shift+Tab: previous") count := len(m.draft.inputs) @@ -90,27 +104,54 @@ func (m *limitsWorkspace) View() tea.View { } } // Reserve navigation, Review, diagnostics, and the optional prefix action before fields. - fieldRows := m.height - 4 - 2 - len(notice) + fieldRows := height - 4 - 2 - len(notice) if count == 3 { fieldRows-- } + unavailable := m.draft.hasUnavailableEdits() + if unavailable { + fieldRows-- + } + var fields []int + position := 0 + for i := range m.draft.inputs { + if m.draft.inputAvailable(i) { + if i <= m.focus { + position = len(fields) + } + fields = append(fields, i) + } + } + if len(fields) < count { + fieldRows-- + } visible := max(1, fieldRows/2) - start := max(0, min(m.focus, count-1)-visible+1) - end := min(count, start+visible) - for i := start; i < end; i++ { + start := max(0, position-visible+1) + end := min(len(fields), start+visible) + for _, i := range fields[start:end] { m.draft.inputs[i].SetWidth(max(1, width-4)) - marker, label := " ", line(m.draft.labels[i]) + text := m.draft.labels[i] + if m.draft.change.Kind == "rate_update" { + text = limitsRateInputs[i].label + " (blank: unchanged)" + } + marker, label := " ", line(text) if i == m.focus { marker = "> " label = accent.Render(label) } body = append(body, label, marker+ansi.Truncate(m.draft.inputs[i].View(), width, "…")) } + if len(fields) < count { + body = append(body, line("Other rate fields were not returned.")) + } if count == 3 { modes := []string{"Keep unless edited", "Replace (empty is allowed)", "Clear to null"} body = append(body, mark("Prefix: "+modes[m.draft.prefixMode], m.focus == count)) } - body = append(body, mark("Review change", m.focus >= count && !(count == 3 && m.focus == count))) + if unavailable { + body = append(body, mark("Discard unavailable edits", m.focus == count)) + } + body = append(body, mark("Review change", m.focus >= count && !((count == 3 || unavailable) && m.focus == count))) body = append(body, notice...) footer = "Enter: next / select\nF1: help Esc: back Ctrl+C: quit" if m.refreshRequired { @@ -144,13 +185,17 @@ func (m *limitsWorkspace) View() tea.View { } // The action row and controls occupy reserved rows. Long data stays in the viewport. footerRows := strings.Count(footer, "\n") + 1 - available := max(0, m.height-2-footerRows) + available := max(0, height-2-footerRows) text := strings.Join(body, "\n") rows := strings.Split(text, "\n") if len(rows) > available { rows = rows[:available] } - view.Content = header + "\n" + strings.Join(rows, "\n") + strings.Repeat("\n", max(1, available-len(rows)+1)) + footer + controls := strings.Split(footer, "\n") + for i := range controls { + controls[i] = line(controls[i]) + } + view.Content = header + "\n" + strings.Join(rows, "\n") + "\n\n" + strings.Join(controls, "\n") return view } diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index 97ce1d29..f45d67fa 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -40,7 +40,7 @@ The isolated child environment includes only fake credentials and the fixture en | `alert-print` | Organization | Print the complete reviewed alert command after terminal restoration; perform no mutation | | `alert-edit` | Project | Change the alert to USD 150.25 and clear its prefix to null | | `alert-delete` | Organization | Delete the selected synthetic alert | -| `rate-update` | Project | Change requests per minute to 250; preserve all other dimensions | +| `rate-update` | Project | Set requests per minute to 250 and tokens per day to 9007199254740993; preserve other dimensions | | `uncertain` | Organization | Apply a synthetic write, lose its response, and verify that repetition stays blocked | | `slow-cancel` | Organization | Cancel a held read with Ctrl+C; perform no mutation | | `preflight-400` | Organization | Reject the preflight read, retain the draft, explain recovery, and exit with failure without mutation | @@ -97,3 +97,10 @@ Check clipping, stale rows, focus markers, controls, and final state after exit. Generated artifacts are terminal replays, not native terminal appearance evidence. Do not commit binary media or claim these planned scenes ran before execution. The earlier spend-threshold recorder remains the separate before/after comparison. + +The workspace now stays inline and preserves shell scrollback. +Selection keeps fixed geometry and font weight. +The panel uses at most 76 columns and 18 rows. +The recorder requires inline readiness, cursor restoration, and restored terminal modes. +Historical full-screen recordings do not validate this revision. +The rate scene exercises the seventh field and exact tokens-per-day request bytes. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh index 5cd383c3..2c03d332 100755 --- a/scripts/demos/limits-workspace/record.sh +++ b/scripts/demos/limits-workspace/record.sh @@ -29,7 +29,8 @@ exec "$DEMO_PYTHON" -I -B "$DEMO_DRIVER" SCENE cp "$demo_runtime/scene.sh" "$demo_output/scene.sh" { - echo 'feature: supported limits workspace workflows' + echo 'feature: compact inline limits workspace workflows' + echo 'presentation: stable selection geometry; maximum76 columns and18rows; preserved shell scrollback' echo "candidate source commit: $demo_after_sha" echo "candidate source state: ${DEMO_SOURCE_STATE:-committed source supplied by caller}" echo 'baseline: no before comparison; this records the supplied workspace binary only' diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index 0c6af77e..e42c095d 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -34,6 +34,15 @@ ''' +def restored_offset(raw): + """Require inline readiness and cursor restoration after the last rendered frame.""" + assert b"\x1b[?1049h" not in raw and b"\x1b[?1049l" not in raw, "workspace entered an alternate screen" + ready = raw.find(b"Website controls and source errors: ?") + hidden, restored = raw.find(b"\x1b[?25l"), raw.rfind(b"\x1b[?25h") + assert ready >= 0 and hidden >= 0 and restored > max(ready, hidden), "inline workspace did not restore its cursor" + return restored + len(b"\x1b[?25h") + + def write_text(path, value): with Path(path).open("x", encoding="utf-8") as destination: if destination.write(value) != len(value): @@ -79,8 +88,7 @@ def finish(self, expected): initial[3] &= ~termios.PENDIN if restored != initial: raise AssertionError("workspace did not restore terminal input") - if self.raw.rfind(b"\x1b[?1049l") <= self.raw.rfind(b"\x1b[?1049h"): - raise AssertionError("workspace did not leave its alternate screen") + restored_offset(self.raw) if self.raw.rfind(b"\x1b[?25h") <= self.raw.rfind(b"\x1b[?25l"): raise AssertionError("workspace did not restore cursor visibility") @@ -279,6 +287,9 @@ def run_scene(terminal, scene, evidence, output): replace(terminal, evidence, "Requests per minute", "250") for _ in range(6): key(terminal, evidence, "Next rate field", b"\t") + wait_screen(terminal, ["Tokens / day", "Review change"]) + replace(terminal, evidence, "Tokens per day", "9007199254740993") + key(terminal, evidence, "Review rate action", b"\t") checkpoint(terminal, evidence, output, "form") key(terminal, evidence, "Review", b"\r") else: @@ -304,7 +315,7 @@ def run_scene(terminal, scene, evidence, output): output_flags = terminal.terminal_modes["initial"][1] newline = "\r\n" if output_flags & termios.OPOST and output_flags & termios.ONLCR else "\n" expected = (command + newline).encode() - restored = terminal.raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + restored = restored_offset(terminal.raw) start = len(terminal.raw) - len(expected) if (start < restored or terminal.raw[start:] != expected or terminal.raw[restored:].count(expected) != 1 @@ -340,7 +351,7 @@ def run_scene(terminal, scene, evidence, output): raise AssertionError("failed preflight sent a mutation") if b"synthetic-private-preflight" in terminal.raw: raise AssertionError("preflight recovery exposed private response data") - restored = terminal.raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + restored = restored_offset(terminal.raw) diagnostic = terminal.raw[restored:] for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"Reopen the limits workspace", b"review current settings", @@ -433,10 +444,8 @@ def main(): if str(binary).encode() in terminal.raw: raise AssertionError("capture exposed the private binary source path") if terminal.child.returncode != 0: - restored = terminal.raw.rfind(b"\x1b[?1049l") - if restored < 0: - raise AssertionError("final diagnostic lacks terminal restoration") - diagnostic = b" ".join(terminal.raw[restored + len(b"\x1b[?1049l"):].lower().split()) + restored = restored_offset(terminal.raw) + diagnostic = b" ".join(terminal.raw[restored:].lower().split()) for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): if action in diagnostic: diff --git a/scripts/demos/limits-workspace/server.py b/scripts/demos/limits-workspace/server.py index 0d08c552..afd55f1b 100755 --- a/scripts/demos/limits-workspace/server.py +++ b/scripts/demos/limits-workspace/server.py @@ -35,7 +35,7 @@ def initial_state(project): "rates": [{"object": "project.rate_limit", "id": "rl_demo", "model": "synthetic-model", "max_requests_per_1_minute": 100, "max_tokens_per_1_minute": 20000, "max_images_per_1_minute": 5, "max_requests_per_1_day": 1000, - "max_audio_megabytes_per_1_minute": 10, "batch_1_day_max_input_tokens": 50000}], + "max_audio_megabytes_per_1_minute": 10, "batch_1_day_max_input_tokens": 50000, "max_tokens_per_1_day": 100000}], } @@ -146,8 +146,8 @@ def handle_request(self): if self.command == "GET": result = page(state["rates"]) elif self.command == "POST" and tail[1:] == ["rl_demo"]: - if payload != {"max_requests_per_1_minute": 250}: - raise AssertionError("rate update must change only the selected dimension") + if payload != {"max_requests_per_1_minute": 250, "max_tokens_per_1_day": 9007199254740993}: + raise AssertionError("rate update must change only the two selected dimensions") state["rates"][0].update(payload) result = state["rates"][0] if result is None: diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 8dfe0c13..3afc78fb 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -14,6 +14,15 @@ "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") +def restored_offset(raw): + """Require inline readiness and cursor restoration after the last rendered frame.""" + assert b"\x1b[?1049h" not in raw and b"\x1b[?1049l" not in raw, "workspace entered an alternate screen" + ready = raw.find(b"Website controls and source errors: ?") + hidden, restored = raw.find(b"\x1b[?25l"), raw.rfind(b"\x1b[?25h") + assert ready >= 0 and hidden >= 0 and restored > max(ready, hidden), "inline workspace did not restore its cursor" + return restored + len(b"\x1b[?25h") + + def validate_rendered_checkpoints(root): expected_rows = [] expected_pngs = {scene + ".png" for scene in SCENES} @@ -77,6 +86,7 @@ def main(): assert staged["removed"] is True and not staged_path.parent.exists(), (scene, "staged CLI cleanup incomplete") assert (evidence["width"], evidence["height"]) in {(80, 24), (40, 12)} raw = (root / (scene + "-evidence.tty")).read_bytes() + restored_offset(raw) child = [json.loads(line) for line in (root / (scene + "-evidence.child.cast")).read_text().splitlines()] child_bytes = "".join(event[2] for event in child[1:] if event[1] == "o").encode() outer = [json.loads(line) for line in (root / (scene + ".cast")).read_text().splitlines()] @@ -89,9 +99,8 @@ def main(): assert relay[:-len(prompt)] == raw == child_bytes, (scene, "capture changed CLI bytes") assert hashlib.sha256(raw).hexdigest() == evidence["relayed_sha256"] if expected_status != 0: - restored = raw.rfind(b"\x1b[?1049l") - assert restored >= 0, (scene, "final diagnostic lacks terminal restoration") - diagnostic = b" ".join(raw[restored + len(b"\x1b[?1049l"):].lower().split()) + restored = restored_offset(raw) + diagnostic = b" ".join(raw[restored:].lower().split()) for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): assert action not in diagnostic, (scene, "final diagnostic names an action from the closed workspace") @@ -129,8 +138,8 @@ def main(): expected = (command + newline).encode() printed = (root / (scene + "-evidence.printed-command.raw.txt")).read_bytes() start = len(raw) - len(expected) - restored_end = raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") - assert raw.rfind(b"\x1b[?1049l") > raw.rfind(b"\x1b[?1049h") >= 0 + restored_end = restored_offset(raw) + assert restored_offset(raw) <= start assert raw.rfind(b"\x1b[?25h") > raw.rfind(b"\x1b[?25l") >= 0 assert restored_end <= start and raw.rfind(b"\x1b[?25h") < start assert printed == expected == raw[start:], (scene, "printed command bytes changed") @@ -157,7 +166,7 @@ def main(): compact = "".join(screen.split()) assert all("".join(message.split()) in compact for message in messages), (scene, name, "missing recovery evidence") assert b"synthetic-private-preflight" not in raw, (scene, "private response leaked") - restored_end = raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + restored_end = restored_offset(raw) diagnostic = raw[restored_end:] assert all(message in diagnostic for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"Reopen the limits workspace", b"review current settings", @@ -191,7 +200,7 @@ def main(): elif scene == "alert-delete": assert request["method"] == "DELETE" and request["path"].endswith("/spend_alerts/alert_demo") elif scene == "rate-update": - assert request["body"] == {"max_requests_per_1_minute": 250} + assert request["body"] == {"max_requests_per_1_minute": 250, "max_tokens_per_1_day": 9007199254740993} if scene == "uncertain": assert request["status"] == "response_lost_after_write" else: From 31b28529e08fc4a8eaf47f3c69fe67bab3b1c475 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 00:54:49 -0700 Subject: [PATCH 29/44] Verify loading readiness when canceling inline Limits --- scripts/demos/limits-workspace/scene.py | 12 ++++++------ scripts/demos/limits-workspace/validate.py | 14 +++++++------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index e42c095d..74fd65eb 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -34,10 +34,10 @@ ''' -def restored_offset(raw): +def restored_offset(raw, loading=False): """Require inline readiness and cursor restoration after the last rendered frame.""" assert b"\x1b[?1049h" not in raw and b"\x1b[?1049l" not in raw, "workspace entered an alternate screen" - ready = raw.find(b"Website controls and source errors: ?") + ready = raw.find(b"Loading" if loading else b"Website controls and source errors: ?") hidden, restored = raw.find(b"\x1b[?25l"), raw.rfind(b"\x1b[?25h") assert ready >= 0 and hidden >= 0 and restored > max(ready, hidden), "inline workspace did not restore its cursor" return restored + len(b"\x1b[?25h") @@ -88,7 +88,7 @@ def finish(self, expected): initial[3] &= ~termios.PENDIN if restored != initial: raise AssertionError("workspace did not restore terminal input") - restored_offset(self.raw) + restored_offset(self.raw, loading=expected == 130) if self.raw.rfind(b"\x1b[?25h") <= self.raw.rfind(b"\x1b[?25l"): raise AssertionError("workspace did not restore cursor visibility") @@ -315,7 +315,7 @@ def run_scene(terminal, scene, evidence, output): output_flags = terminal.terminal_modes["initial"][1] newline = "\r\n" if output_flags & termios.OPOST and output_flags & termios.ONLCR else "\n" expected = (command + newline).encode() - restored = restored_offset(terminal.raw) + restored = restored_offset(terminal.raw, loading=scene == "slow-cancel") start = len(terminal.raw) - len(expected) if (start < restored or terminal.raw[start:] != expected or terminal.raw[restored:].count(expected) != 1 @@ -351,7 +351,7 @@ def run_scene(terminal, scene, evidence, output): raise AssertionError("failed preflight sent a mutation") if b"synthetic-private-preflight" in terminal.raw: raise AssertionError("preflight recovery exposed private response data") - restored = restored_offset(terminal.raw) + restored = restored_offset(terminal.raw, loading=scene == "slow-cancel") diagnostic = terminal.raw[restored:] for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"Reopen the limits workspace", b"review current settings", @@ -444,7 +444,7 @@ def main(): if str(binary).encode() in terminal.raw: raise AssertionError("capture exposed the private binary source path") if terminal.child.returncode != 0: - restored = restored_offset(terminal.raw) + restored = restored_offset(terminal.raw, loading=scene == "slow-cancel") diagnostic = b" ".join(terminal.raw[restored:].lower().split()) for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 3afc78fb..5cf93c93 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -14,10 +14,10 @@ "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") -def restored_offset(raw): +def restored_offset(raw, loading=False): """Require inline readiness and cursor restoration after the last rendered frame.""" assert b"\x1b[?1049h" not in raw and b"\x1b[?1049l" not in raw, "workspace entered an alternate screen" - ready = raw.find(b"Website controls and source errors: ?") + ready = raw.find(b"Loading" if loading else b"Website controls and source errors: ?") hidden, restored = raw.find(b"\x1b[?25l"), raw.rfind(b"\x1b[?25h") assert ready >= 0 and hidden >= 0 and restored > max(ready, hidden), "inline workspace did not restore its cursor" return restored + len(b"\x1b[?25h") @@ -86,7 +86,7 @@ def main(): assert staged["removed"] is True and not staged_path.parent.exists(), (scene, "staged CLI cleanup incomplete") assert (evidence["width"], evidence["height"]) in {(80, 24), (40, 12)} raw = (root / (scene + "-evidence.tty")).read_bytes() - restored_offset(raw) + restored_offset(raw, loading=scene == "slow-cancel") child = [json.loads(line) for line in (root / (scene + "-evidence.child.cast")).read_text().splitlines()] child_bytes = "".join(event[2] for event in child[1:] if event[1] == "o").encode() outer = [json.loads(line) for line in (root / (scene + ".cast")).read_text().splitlines()] @@ -99,7 +99,7 @@ def main(): assert relay[:-len(prompt)] == raw == child_bytes, (scene, "capture changed CLI bytes") assert hashlib.sha256(raw).hexdigest() == evidence["relayed_sha256"] if expected_status != 0: - restored = restored_offset(raw) + restored = restored_offset(raw, loading=scene == "slow-cancel") diagnostic = b" ".join(raw[restored:].lower().split()) for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): @@ -138,8 +138,8 @@ def main(): expected = (command + newline).encode() printed = (root / (scene + "-evidence.printed-command.raw.txt")).read_bytes() start = len(raw) - len(expected) - restored_end = restored_offset(raw) - assert restored_offset(raw) <= start + restored_end = restored_offset(raw, loading=scene == "slow-cancel") + assert restored_offset(raw, loading=scene == "slow-cancel") <= start assert raw.rfind(b"\x1b[?25h") > raw.rfind(b"\x1b[?25l") >= 0 assert restored_end <= start and raw.rfind(b"\x1b[?25h") < start assert printed == expected == raw[start:], (scene, "printed command bytes changed") @@ -166,7 +166,7 @@ def main(): compact = "".join(screen.split()) assert all("".join(message.split()) in compact for message in messages), (scene, name, "missing recovery evidence") assert b"synthetic-private-preflight" not in raw, (scene, "private response leaked") - restored_end = restored_offset(raw) + restored_end = restored_offset(raw, loading=scene == "slow-cancel") diagnostic = raw[restored_end:] assert all(message in diagnostic for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", b"Reopen the limits workspace", b"review current settings", From 1947689e960524b98a0686e2e320df2698db0b66 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 00:56:27 -0700 Subject: [PATCH 30/44] Preserve rate flag aliases and narrow review controls --- cmd/openai/limits_workflow_test.go | 96 +++++++++++++++------------ internal/app/limits_commands.go | 36 ++++++---- internal/app/limits_workspace_test.go | 3 + internal/app/limits_workspace_view.go | 4 +- 4 files changed, 79 insertions(+), 60 deletions(-) diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go index 2efab4ed..2b3aef86 100644 --- a/cmd/openai/limits_workflow_test.go +++ b/cmd/openai/limits_workflow_test.go @@ -612,51 +612,59 @@ func TestMainLimitsWorkflowCostsPrecisionAndPartialFailure(t *testing.T) { } func TestMainLimitsTokensPerDayFlagAndStdin(t *testing.T) { - for _, test := range []struct{ name, input, value, want string }{ - {"flag zero", "", "0", "0"}, - {"flag signed", "", "-1", "-1"}, - {"flag exact", "", "9223372036854775807", "9223372036854775807"}, - {"JSON", `{"max_tokens_per_1_day":9007199254740993,"future":{"kept":true}}`, "", "9007199254740993"}, - {"YAML", "max_tokens_per_1_day: 9007199254740993\nfuture:\n kept: true\n", "", "9007199254740993"}, - {"explicit override", `{"max_tokens_per_1_day":9007199254740993,"future":{"kept":true}}`, "0", "0"}, + for _, route := range [][]string{ + {"projects", "rate-limits", "update-rate-limit"}, + {"admin", "organization", "projects", "rate-limits", "update-rate-limit"}, + {"admin:organization:projects:rate-limits", "update-rate-limit"}, } { - t.Run(test.name, func(t *testing.T) { - var requests atomic.Int32 - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - requests.Add(1) - if r.Method != http.MethodPost || r.URL.Path != "/organization/projects/proj_synthetic/rate_limits/rl_synthetic" { - t.Errorf("unexpected rate route: %s %s", r.Method, r.URL.Path) - } - if r.Header.Get("Authorization") != "Bearer sk-fake-spend-admin" { - t.Error("rate update did not use the admin credential") - } - body, err := io.ReadAll(r.Body) - if err != nil { - t.Error(err) - } - value := gjson.ParseBytes(body) - if value.Get("max_tokens_per_1_day").Raw != test.want { - t.Errorf("daily tokens changed: %s", body) - } - if test.input != "" && value.Get("future.kept").Type != gjson.True { - t.Error("stdin lost an unfamiliar field") + for _, test := range []struct{ name, input, value, want string }{ + {"flag zero", "", "0", "0"}, + {"flag signed", "", "-1", "-1"}, + {"flag exact", "", "9223372036854775807", "9223372036854775807"}, + {"JSON", `{"max_tokens_per_1_day":9007199254740993,"future":{"kept":true}}`, "", "9007199254740993"}, + {"YAML", "max_tokens_per_1_day: 9007199254740993\nfuture:\n kept: true\n", "", "9007199254740993"}, + {"explicit override", `{"max_tokens_per_1_day":9007199254740993,"future":{"kept":true}}`, "0", "0"}, + } { + t.Run(strings.Join(route, "_")+"/"+test.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodPost || r.URL.Path != "/organization/projects/proj_synthetic/rate_limits/rl_synthetic" { + t.Errorf("unexpected rate route: %s %s", r.Method, r.URL.Path) + } + if r.Header.Get("Authorization") != "Bearer sk-fake-spend-admin" { + t.Error("rate update did not use the admin credential") + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + value := gjson.ParseBytes(body) + if value.Get("max_tokens_per_1_day").Raw != test.want { + t.Errorf("daily tokens changed: %s", body) + } + if test.input != "" && value.Get("future.kept").Type != gjson.True { + t.Error("stdin lost an unfamiliar field") + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprintf(w, `{"id":"rl_synthetic","object":"project.rate_limit","model":"synthetic-model","max_tokens_per_1_day":%s}`, test.want) + })) + defer server.Close() + args := append([]string{"openai", "--format", "json"}, route...) + args = append(args, "--project-id", "proj_synthetic", "--rate-limit-id", "rl_synthetic") + if test.value != "" { + args = append(args, "--max-tokens-per-1-day="+test.value) + } + got := runMainDispatchWithStdin(t, "bash", []string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "NO_COLOR=1"}, shellFileInput(t, []byte(test.input)), args...) + if got.code != 0 || got.stderr != "" || requests.Load() != 1 || gjson.Get(got.stdout, "max_tokens_per_1_day").Raw != test.want { + t.Fatalf("daily token command failed: code=%d stderr=%q requests=%d stdout=%s", got.code, got.stderr, requests.Load(), got.stdout) } - w.Header().Set("Content-Type", "application/json") - fmt.Fprintf(w, `{"id":"rl_synthetic","object":"project.rate_limit","model":"synthetic-model","max_tokens_per_1_day":%s}`, test.want) - })) - defer server.Close() - args := []string{"openai", "--format", "json", "projects", "rate-limits", "update-rate-limit", "--project-id", "proj_synthetic", "--rate-limit-id", "rl_synthetic"} - if test.value != "" { - args = append(args, "--max-tokens-per-1-day="+test.value) - } - got := runMainDispatchWithStdin(t, "bash", []string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "NO_COLOR=1"}, shellFileInput(t, []byte(test.input)), args...) - if got.code != 0 || got.stderr != "" || requests.Load() != 1 || gjson.Get(got.stdout, "max_tokens_per_1_day").Raw != test.want { - t.Fatalf("daily token command failed: code=%d stderr=%q requests=%d stdout=%s", got.code, got.stderr, requests.Load(), got.stdout) - } - }) - } - got := runMainDispatch(t, "bash", "openai", "projects", "rate-limits", "update-rate-limit", "--help") - if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "--max-tokens-per-1-day") { - t.Fatalf("daily token flag missing from help: %+v", got) + }) + } + helpArgs := append([]string{"openai"}, route...) + got := runMainDispatch(t, "bash", append(helpArgs, "--help")...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "--max-tokens-per-1-day") { + t.Fatalf("daily token flag missing from help: %+v", got) + } } } diff --git a/internal/app/limits_commands.go b/internal/app/limits_commands.go index 9418e3d6..4e72cd72 100644 --- a/internal/app/limits_commands.go +++ b/internal/app/limits_commands.go @@ -38,24 +38,32 @@ func registerLimitsCommands(root *cli.Command) { return } organization.Commands = append(organization.Commands, newLimitsCommand(false)) + routes := []*cli.Command{root.Command("admin:organization:projects:rate-limits")} if projects := organization.Command("projects"); projects != nil { projects.Commands = append(projects.Commands, newLimitsCommand(true)) - if rates := projects.Command("rate-limits"); rates != nil { - if update := rates.Command("update-rate-limit"); update != nil { - found := false - for _, flag := range update.Flags { - for _, name := range flag.Names() { - found = found || name == "max-tokens-per-1-day" - } - } - if !found { - update.Flags = append(update.Flags, &requestflag.Flag[int64]{ - Name: "max-tokens-per-1-day", BodyPath: "max_tokens_per_1_day", - Usage: "The maximum number of tokens per day for this model.", - }) - } + routes = append(routes, projects.Command("rate-limits")) + } + // Subgroup cloning already created the preserved colon route. + for _, rates := range routes { + if rates == nil { + continue + } + update := rates.Command("update-rate-limit") + if update == nil { + continue + } + found := false + for _, flag := range update.Flags { + for _, name := range flag.Names() { + found = found || name == "max-tokens-per-1-day" } } + if !found { + update.Flags = append(update.Flags, &requestflag.Flag[int64]{ + Name: "max-tokens-per-1-day", BodyPath: "max_tokens_per_1_day", + Usage: "The maximum number of tokens per day for this model.", + }) + } } } diff --git a/internal/app/limits_workspace_test.go b/internal/app/limits_workspace_test.go index c5c37b26..ebdc9007 100644 --- a/internal/app/limits_workspace_test.go +++ b/internal/app/limits_workspace_test.go @@ -573,6 +573,8 @@ func TestLimitsWorkspacePrintedCommandNativeShellPreservesArguments(t *testing.T want: []string{"admin", "organization", "projects", "spend-alerts", "delete", "--project-id", "proj target's", "--alert-id", "alert owner's"}}, {name: "rate-update", projectID: "proj target's", change: limitsChange{Kind: "rate_update", ID: "rl owner's", RateValues: map[string]int64{"max_requests_per_1_minute": -1}}, want: []string{"admin", "organization", "projects", "rate-limits", "update-rate-limit", "--project-id", "proj target's", "--rate-limit-id", "rl owner's", "--max-requests-per-1-minute", "-1"}}, + {name: "tokens-per-day", projectID: "proj target's", change: limitsChange{Kind: "rate_update", ID: "rl owner's", RateValues: map[string]int64{"max_tokens_per_1_day": 9007199254740993}}, + want: []string{"admin", "organization", "projects", "rate-limits", "update-rate-limit", "--project-id", "proj target's", "--rate-limit-id", "rl owner's", "--max-tokens-per-1-day", "9007199254740993"}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -1937,6 +1939,7 @@ func TestLimitsWorkspaceCompactGeometryAndReviewGate(t *testing.T) { m.draft.inputs[0].SetValue("100.01") m.review() require.Equal(t, "review", m.page) + require.Contains(t, ansi.Strip(m.View().Content), "Enter: select") require.Equal(t, min(size[0], 76)-2, m.viewport.Width()) require.LessOrEqual(t, m.viewport.Height(), min(size[1], 18)-5) require.False(t, m.viewport.AtBottom(), "clamped review must not approve unread content") diff --git a/internal/app/limits_workspace_view.go b/internal/app/limits_workspace_view.go index 6cc6375f..107722ef 100644 --- a/internal/app/limits_workspace_view.go +++ b/internal/app/limits_workspace_view.go @@ -168,9 +168,9 @@ func (m *limitsWorkspace) View() tea.View { apply = " Read to the end before applying" } body = append(body, line(back+" "+apply)) - footer = "↑/↓: scroll Tab: action Enter: select\nF1: help Esc: back Ctrl+C: quit" + footer = "↑/↓ scroll Tab action Enter: select\nF1: help Esc: back Ctrl+C: quit" if m.previewCommand != "" { - footer = "↑/↓: scroll Tab: action Enter: select\nCtrl+P: print command and exit\nF1: help Esc: back Ctrl+C: quit" + footer = "↑/↓ scroll Tab action Enter: select\nCtrl+P: print command and exit\nF1: help Esc: back Ctrl+C: quit" } } else { footer = "↑/↓: scroll Esc: back Ctrl+C: quit" From 8d655da6d5387d437cbc802bfc9123eef8cc24ae Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 01:30:07 -0700 Subject: [PATCH 31/44] test(limits): cover inline terminal recovery and mode restoration --- scripts/demos/limits-workspace/README.md | 26 +- scripts/demos/limits-workspace/record.sh | 7 +- scripts/demos/limits-workspace/scene.py | 279 ++++++++++++++++++--- scripts/demos/limits-workspace/server.py | 5 +- scripts/demos/limits-workspace/validate.py | 110 +++++++- 5 files changed, 375 insertions(+), 52 deletions(-) diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index f45d67fa..1a54b0b9 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -5,11 +5,13 @@ It makes no live API calls or billing changes. The recorder uses the shared `capture_and_render.sh` lifecycle and existing PTY harness. No Go build occurs during recording. -Obtain the source-build and PTY reservations through the current coordinator. +Follow the current local resource policy before source builds or PTY replay. Read `/Users/vguvvala/code/cli-work/DESIGN.md`, `STATUS.md`, and `PRE_PUSH_REVIEW.md` before presentation changes or publication. Preserve generated ownership and obtain independent adversarial review before every feature push. Use existing Python, asciinema, agg, ffmpeg, ffprobe, and Menlo installations. +Use pinned observer dependencies from `scripts/demos/audio-workbench/setup_observer.py`. +Set `DEMO_OBSERVER_SOURCES` to that verified package directory. Supply a verified binary and its full source commit. Keep each output directory empty and outside the repository. Set `DEMO_SOURCE_STATE` when recording an uncommitted candidate based on that source commit. @@ -17,11 +19,11 @@ Binary hashes and source manifests establish recording identity; they do not est ```sh # Run only during the granted PTY reservation. -DEMO_WIDTH=80 DEMO_NO_COLOR=1 \ +DEMO_OBSERVER_SOURCES=/absolute/path/to/verified-observer DEMO_WIDTH=80 DEMO_NO_COLOR=1 \ bash scripts/demos/limits-workspace/record.sh --run-authorized-pty-slot \ /absolute/path/to/immutable/openai SOURCE_SHA /absolute/path/to/evidence/limits-80 -DEMO_WIDTH=40 DEMO_NO_COLOR=1 \ +DEMO_OBSERVER_SOURCES=/absolute/path/to/verified-observer DEMO_WIDTH=40 DEMO_NO_COLOR=1 \ bash scripts/demos/limits-workspace/record.sh --run-authorized-pty-slot \ /absolute/path/to/immutable/openai SOURCE_SHA /absolute/path/to/evidence/limits-40 ``` @@ -35,7 +37,7 @@ The isolated child environment includes only fake credentials and the fixture en | --- | --- | --- | | `org-limit` | Organization | Replace the hard limit with USD 125.25 per month | | `org-limit-delete` | Organization | Remove the hard limit and retain its confirmed receipt after a missing-record refresh | -| `project-limit` | Project | Replace the hard limit with USD 125.25 per month | +| `project-limit` | Project | Replace the hard limit with USD 125.25 and preserve seeded shell scrollback | | `alert-create` | Organization | Correct an invalid amount, retain the draft through help and Back, then create a USD 75.50 alert | | `alert-print` | Organization | Print the complete reviewed alert command after terminal restoration; perform no mutation | | `alert-edit` | Project | Change the alert to USD 150.25 and clear its prefix to null | @@ -43,6 +45,7 @@ The isolated child environment includes only fake credentials and the fixture en | `rate-update` | Project | Set requests per minute to 250 and tokens per day to 9007199254740993; preserve other dimensions | | `uncertain` | Organization | Apply a synthetic write, lose its response, and verify that repetition stays blocked | | `slow-cancel` | Organization | Cancel a held read with Ctrl+C; perform no mutation | +| `preflight-recover` | Organization | Fail preflight once, refresh the retained draft, then confirm exactly one write | | `preflight-400` | Organization | Reject the preflight read, retain the draft, explain recovery, and exit with failure without mutation | Each mutation scene opens a form or deletion review. @@ -51,8 +54,11 @@ It checks that the fixture receives no mutation before confirmation. The create-alert scene checks visible validation errors and complete help text. Repeated help preserves the draft, prefix mode, and selected Review action. Escape from review returns to the draft. Help from review clears Apply selection. +The alert scene resizes help below minimum size, then returns through both supported sizes. +Resize clears Apply and requires rereading. The scene inspects the retained amount before a fresh review. The final request verifies the retained amount, recipient, and prefix. -The uncertain scene checks one mutation, retained failure guidance, and exit status 1. +The uncertain scene refreshes after failure and checks that writes remain locked. +It requires one mutation, retained failure guidance, and exit status 1. The cancellation scene waits for the fixture's request marker and requires exit status 130. Successful mutation scenes require exit status 0 and one mutation each. The print scene uses Ctrl+P and requires exit status 0 with no mutation. @@ -72,6 +78,13 @@ The scene driver restores terminal output processing and closes every owned chil The recorder preserves raw child bytes, child casts, outer casts, request logs, hashes, and actual process statuses. The validator checks byte identity across all three capture forms. +It independently replays child bytes and resize events through pinned pyte and wcwidth sources. +The observer answers cursor queries at their actual positions, including fragmented and repeated queries. +The recorder probes query fragmentation, keyboard controls, cursor bounds, and history before recording. +The project scene checks retained scrollback markers in emulator history. +Limits supplies a resizable observer because the shared observer supports fixed audio sizes only. +The driver retains shared PTY allocation and cleanup. +The observer clips during resize; it does not model native terminal reflow. Checkpoint renderers receive no stdin. The recorder logs their exact names, timestamps, and output filenames. A second validation pass compares that inventory with source checkpoints and all PNG and frame-GIF filenames. It rejects missing, duplicate, altered, or unexpected checkpoint artifacts. @@ -98,9 +111,10 @@ Generated artifacts are terminal replays, not native terminal appearance evidenc Do not commit binary media or claim these planned scenes ran before execution. The earlier spend-threshold recorder remains the separate before/after comparison. -The workspace now stays inline and preserves shell scrollback. +The workspace uses an inline panel. The project scene verifies prior shell scrollback. Selection keeps fixed geometry and font weight. The panel uses at most 76 columns and 18 rows. The recorder requires inline readiness, cursor restoration, and restored terminal modes. +Paste disable, keyboard reset, and keyboard-stack restoration must precede printed commands and final diagnostics. Historical full-screen recordings do not validate this revision. The rate scene exercises the seventh field and exact tokens-per-day request bytes. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh index 6263a258..df437552 100755 --- a/scripts/demos/limits-workspace/record.sh +++ b/scripts/demos/limits-workspace/record.sh @@ -17,6 +17,8 @@ source "$record_source/../capture_and_render.sh" # The shared cleanup also owns synthetic executable paths shown in command previews. TMPDIR=/private/tmp demo_prepare_capture "$record_root" "$1" "$1" "$2" "$2" "$3" "$record_source/server.py" demo_python="$(command -v python3)" +: "${DEMO_OBSERVER_SOURCES:?Set the verified observer package directory.}" +"$demo_python" -I -B "$record_source/scene.py" --check-observer > "$demo_output/observer-validation.txt" record_digest="$(shasum -a 256 "$demo_after" | cut -d ' ' -f 1)" demo_start_api "$demo_output/requests.jsonl" "$demo_output/fixtures.json" "$demo_runtime/gates" cat > "$demo_runtime/scene.sh" <<'SCENE' @@ -44,6 +46,7 @@ cp "$demo_runtime/scene.sh" "$demo_output/scene.sh" shasum -a 256 "$record_source/record.sh" "$record_source/server.py" "$record_source/scene.py" \ "$record_source/validate.py" "$record_source/README.md" \ "$record_root/scripts/demos/capture_and_render.sh" "$record_root/scripts/image_picker_harness.py" \ + "$record_root/scripts/demos/audio-workbench/observer.py" "$record_root/scripts/demos/audio-workbench/observer_sources.json" \ "$record_root/internal/app/limits_commands.go" "$record_root/internal/app/limits_service.go" \ "$record_root/internal/app/limits_values.go" "$record_root/internal/app/limits_draft.go" \ "$record_root/internal/app/limits_workspace.go" "$record_root/internal/app/limits_workspace_view.go" \ @@ -51,14 +54,14 @@ shasum -a 256 "$record_source/record.sh" "$record_source/server.py" "$record_sou demo_window_size="${record_width}x$((record_height+4))" demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ --theme asciinema --fps-cap 15 --last-frame-duration 2) -record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel alert-print preflight-400) +record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel alert-print preflight-400 preflight-recover) for record_scene in "${record_scenes[@]}"; do demo_capture_scene "$record_scene" 0 "$demo_runtime/after" "$demo_api_url/$record_scene/v1" \ "$record_scene: ${record_width}x${record_height}" \ "DEMO_PYTHON=$demo_python" "DEMO_ASCIINEMA=$demo_asciinema" "DEMO_DRIVER=$record_source/scene.py" \ "DEMO_SCENE_NAME=$record_scene" "DEMO_EVIDENCE_BASE=$demo_output/$record_scene-evidence" \ "DEMO_WIDTH=$record_width" "DEMO_HEIGHT=$record_height" "DEMO_NO_COLOR=$record_no_color" \ - "DEMO_RUNTIME_DIRECTORY=$demo_runtime" \ + "DEMO_RUNTIME_DIRECTORY=$demo_runtime" "DEMO_OBSERVER_SOURCES=${DEMO_OBSERVER_SOURCES:?Set the verified observer package directory.}" \ "DEMO_GATE_DIRECTORY=$demo_runtime/gates" "DEMO_REQUEST_LOG=$demo_output/requests.jsonl" done demo_stop_api diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index 74fd65eb..d1cb10e6 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -1,10 +1,14 @@ #!/usr/bin/env python3 """Drive the real workspace with deterministic synthetic input and retain raw bytes.""" +import codecs +import errno import hashlib import importlib.util import json import os +import re +import select from pathlib import Path import shutil import subprocess @@ -19,7 +23,11 @@ spec.loader.exec_module(harness) # Trapped signals retain their default disposition in the foreground CLI child. -BASH_WORKSPACE = r'''trap ':' HUP INT TERM +BASH_WORKSPACE = r'''for ((limits_seed=1; limits_seed<=DEMO_SCROLLBACK_ROWS; limits_seed++)); do + printf "Prior shell output %03d\n" "$limits_seed" +done +printf "F40 workspace start\n" +trap ':' HUP INT TERM "$@" limits_scene_status=$? printf '%s\n' "$limits_scene_status" > "$DEMO_CLI_EXIT" || exit 74 @@ -34,13 +42,136 @@ ''' -def restored_offset(raw, loading=False): - """Require inline readiness and cursor restoration after the last rendered frame.""" - assert b"\x1b[?1049h" not in raw and b"\x1b[?1049l" not in raw, "workspace entered an alternate screen" - ready = raw.find(b"Loading" if loading else b"Website controls and source errors: ?") - hidden, restored = raw.find(b"\x1b[?25l"), raw.rfind(b"\x1b[?25h") - assert ready >= 0 and hidden >= 0 and restored > max(ready, hidden), "inline workspace did not restore its cursor" - return restored + len(b"\x1b[?25h") +validator_spec = importlib.util.spec_from_file_location("limits_demo_validator", Path(__file__).with_name("validate.py")) +validator = importlib.util.module_from_spec(validator_spec) +validator_spec.loader.exec_module(validator) +restored_offset = validator.restored_offset + +observer_path = Path(__file__).parents[1] / "audio-workbench" / "observer.py" +observer_spec = importlib.util.spec_from_file_location("limits_verified_observer_sources", observer_path) +observer_sources = importlib.util.module_from_spec(observer_spec) +observer_spec.loader.exec_module(observer_sources) +pyte = observer_sources._verified_packages(os.environ.get("DEMO_OBSERVER_SOURCES")) + + +class TerminalObserver: + """Observe synthetic output and reply at the actual terminal query boundary.""" + def __init__(self, width, height, reply): + class Screen(pyte.HistoryScreen): + def write_process_input(self, data): + reply(data.encode()) + + def report_device_status(self, mode, **kwargs): + if not kwargs.get("private"): + column = self.cursor.x + self.cursor.x = min(column, self.columns - 1) + try: + super().report_device_status(mode) + finally: + self.cursor.x = column + + def report_device_attributes(self, mode=0, **kwargs): + if mode == 0 and not kwargs.get("private"): + self.write_process_input("\x1b[?1;2c") + + self.screen = Screen(width, height, history=observer_sources.HISTORY_ROWS) + self.stream = pyte.ByteStream(self.screen) + self.pending = bytearray() + self.bytes_observed = 0 + + def feed(self, data): + self.bytes_observed += len(data) + self.pending.extend(data) + while self.pending: + if self.pending[0] != 27: + end = self.pending.find(b"\x1b") + end = len(self.pending) if end < 0 else end + elif len(self.pending) < 2: + break + elif self.pending[1] == ord("["): + end = next((i + 1 for i in range(2, len(self.pending)) + if 0x40 <= self.pending[i] <= 0x7e), None) + if end is None: + break + elif self.pending[1] in b"]PX^_": + terminators = ((b"\x07", 1), (b"\x1b\\", 2)) if self.pending[1] == ord("]") else ((b"\x1b\\", 2),) + endings = [position + size for marker, size in terminators + if (position := self.pending.find(marker, 2)) >= 0] + if not endings: + break + end = min(endings) + else: + end = 1 + while end < len(self.pending) and 0x20 <= self.pending[end] <= 0x2f: + end += 1 + if end == len(self.pending): + break + assert 0x30 <= self.pending[end] <= 0x7e, "malformed terminal escape sequence" + end += 1 + token = bytes(self.pending[:end]) + del self.pending[:end] + if token in (b"\x1b]11;?\x07", b"\x1b]11;?\x1b\\"): + self.screen.write_process_input("\x1b]11;rgb:1818/1818/1818\x07") + elif token[:2] in (b"\x1bP", b"\x1bX", b"\x1b^", b"\x1b_") or re.fullmatch(rb"\x1b\[>[0-9;]*c", token): + pass # Ignore unsupported string controls and secondary device queries. + elif not (re.fullmatch(rb"\x1b\[[<>=][0-9;]*u", token) + or re.fullmatch(rb"\x1b\[>4(?:;[0-2])?m", token)): + self.stream.feed(token) + assert len(self.screen.history.top) < observer_sources.HISTORY_ROWS, "observer history reached its bound" + + def resize(self, width, height): + self.screen.resize(lines=height, columns=width) + + def text(self): + return "\n".join(self.screen.display) + + def history(self): + return ["".join(line.get(column, self.screen.default_char).data + for column in range(self.screen.columns)).rstrip() + for line in self.screen.history.top] + + +def probe_observer(): + payload = (b"before\r\n\x1b[6n\x1b]11;?\x07\x1b[c" + b"\x1bP+q5443\x1b\\\x1b_hidden\x1b\\\x1b[>0c" + b"\x1b[>4;2m\x1b[>1u\x1b[<1uafter\x1b[6n") + expected = [b"\x1b[2;1R", b"\x1b]11;rgb:1818/1818/1818\x07", b"\x1b[?1;2c", b"\x1b[2;6R"] + for boundary in range(len(payload) + 1): + replies = [] + observer = TerminalObserver(40, 12, replies.append) + observer.feed(payload[:boundary]) + observer.feed(payload[boundary:]) + assert replies == expected and not observer.pending, "split query replies differ" + assert observer.screen.display[:2] == ["before".ljust(40), "after".ljust(40)] + replies = [] + observer = TerminalObserver(40, 12, replies.append) + for byte in payload: + observer.feed(bytes([byte])) + assert replies == expected, "single-byte query replies differ" + observer = TerminalObserver(40, 12, replies.append) + observer.feed(b"x" * 40 + b"\x1b[6n") + assert replies[-1] == b"\x1b[1;40R", "cursor report exceeded the right margin" + observer = TerminalObserver(40, 12, lambda _: None) + observer.feed(b"".join(f"seed {number:02d}\r\n".encode() for number in range(30))) + assert observer.history()[:3] == ["seed 00", "seed 01", "seed 02"] + for prefix, final in ((b"\x1b(", b"B"), (b"\x1b#", b"8"), (b"\x1bPq", b"\x1b\\")): + observer = TerminalObserver(40, 12, lambda _: None) + observer.feed(prefix) + assert observer.pending, "incomplete escape was accepted" + observer.feed(final) + assert not observer.pending, "complete escape remained pending" + restored = (b"\x1b[?25l\x1b[?2004h\x1b[>4;2m\x1b[>1u\x1b[?u" + b"Website controls and source errors: ?\x1b[?2004l\x1b[>4m\x1b[<1u\x1b[?25h") + assert restored_offset(restored) == len(restored) + for late in (b"\x1b[>5u", b"\x1b[>4;1m", b"\x1b[>04;1m", b"\x1b[?25;2004h", + b"\x1b[?1049;25h", b"\x1b[2;0J", b"\x1b[?25l", b"\x1b[<2u"): + try: + restored_offset(restored + late) + except AssertionError: + pass + else: + raise AssertionError("late terminal mode change was accepted") + print("PASS: observer split/coalesced queries; keyboard controls; cursor bounds; retained history.") def write_text(path, value): @@ -52,16 +183,37 @@ def write_text(path, value): class RelayTerminal(harness.Terminal): def __init__(self, *args, terminal_snapshot, **kwargs): self.terminal_snapshot = terminal_snapshot + self.protocol_replies = [] + self.decoder = codecs.getincrementaldecoder("utf-8")("strict") + self.observer = TerminalObserver(kwargs["width"], kwargs["height"], self.reply) super().__init__(*args, **kwargs) def read(self, duration=0.05): - offset = len(self.raw) - super().read(duration) - chunk = self.raw[offset:] + if not select.select([self.master], [], [], duration)[0]: + return + try: + chunk = os.read(self.master, 65536) + except OSError as error: + if error.errno == errno.EIO: + return + raise + self.raw.extend(chunk) + decoded = self.decoder.decode(chunk) + if decoded: + self.events.append([time.monotonic() - self.started, "o", decoded]) + self.observer.feed(chunk) if sys.stdout.buffer.write(chunk) != len(chunk): raise OSError("short terminal relay write") sys.stdout.buffer.flush() + def resize(self, width, height, notify=True): + self.observer.resize(width, height) + super().resize(width, height, notify=notify) + + def reply(self, value): + self.protocol_replies.append(value.decode("ascii")) + self.send(value) + def send(self, value): if os.write(self.master, value) != len(value): raise OSError("short workspace key write") @@ -88,6 +240,7 @@ def finish(self, expected): initial[3] &= ~termios.PENDIN if restored != initial: raise AssertionError("workspace did not restore terminal input") + self.decoder.decode(b"", final=True) restored_offset(self.raw, loading=expected == 130) if self.raw.rfind(b"\x1b[?25h") <= self.raw.rfind(b"\x1b[?25l"): raise AssertionError("workspace did not restore cursor visibility") @@ -100,15 +253,7 @@ def drain(terminal, seconds): def screen_text(terminal): - with tempfile.TemporaryDirectory(prefix="limits-demo-screen-") as temporary: - recording, transcript = Path(temporary) / "screen.cast", Path(temporary) / "screen.txt" - header = {"version": 2, "width": terminal.initial_size[0], "height": terminal.initial_size[1]} - write_text(recording, "\n".join(json.dumps(item) for item in [header, *terminal.events]) + "\n") - subprocess.run([os.environ["DEMO_ASCIINEMA"], "convert", "-f", "txt", str(recording), str(transcript)], - capture_output=True, check=True, timeout=10, - env={"PATH": "/usr/bin:/bin", "HOME": temporary, - "ASCIINEMA_STATE_HOME": temporary + "/state", "ASCIINEMA_CONFIG_HOME": temporary + "/config"}) - return transcript.read_text() + return terminal.observer.text() def wait_screen(terminal, required, forbidden=(), timeout=10): @@ -153,16 +298,16 @@ def replace(terminal, evidence, label, value): key(terminal, evidence, label, b"\x01\x0b\x1b[200~" + value.encode() + b"\x1b[201~") -def review(terminal, evidence, scene, output): +def read_review(terminal, evidence, scene, output, name): wait_screen(terminal, ["> Back", "Enter: select"]) - checkpoint(terminal, evidence, output, "review-top") + checkpoint(terminal, evidence, output, name + "-top") preview_seen = False for _ in range(40): screen = screen_text(terminal) if "Command printing needs" in screen or "Command preview omitted" in screen: raise AssertionError("review omitted the real command preview") if not preview_seen and "Command preview (inspection only)" in screen: - checkpoint(terminal, evidence, output, "review-command", + checkpoint(terminal, evidence, output, name + "-command", ("Command preview (inspection only)",)) preview_seen = True if "Apply change" in screen and "Read to the end" not in screen: @@ -177,9 +322,13 @@ def review(terminal, evidence, scene, output): raise AssertionError("review reached Apply without a command preview checkpoint") if mutations(scene): raise AssertionError("fixture received a mutation before explicit confirmation") + + +def review(terminal, evidence, scene, output, name="review"): + read_review(terminal, evidence, scene, output, name) if scene == "alert-print": wait_screen(terminal, ["Ctrl+P"]) - checkpoint(terminal, evidence, output, "review-print", ("Ctrl+P",)) + checkpoint(terminal, evidence, output, name + "-print", ("Ctrl+P",)) key(terminal, evidence, "Ctrl+P print and exit", b"\x10") return key(terminal, evidence, "Tab to Apply", b"\t") @@ -189,15 +338,36 @@ def review(terminal, evidence, scene, output): key(terminal, evidence, "Review help", b"\x1bOP") wait_screen(terminal, ["Spend manages hard monthly"]) checkpoint(terminal, evidence, output, "review-help") + terminal.resize(39, 11) + wait_screen(terminal, ["Resize to at least 40 x 12."]) + checkpoint(terminal, evidence, output, "review-help-small") + target = (40, 12) if terminal.initial_size == (80, 24) else (80, 24) + terminal.resize(*target) + wait_screen(terminal, ["Spend manages hard monthly"], ["Resize to at least"]) + checkpoint(terminal, evidence, output, "review-help-resized") key(terminal, evidence, "Return to review", b"\x1b") - wait_screen(terminal, ["> Back", "Apply change"], ["> Apply change"]) - checkpoint(terminal, evidence, output, "review-return", ("> Back",)) + wait_screen(terminal, ["> Back", "Read to the end"], ["> Apply change"]) + checkpoint(terminal, evidence, output, "review-return", ("> Back", "Read to the end")) + key(terminal, evidence, "Apply blocked until reread", b"\t") + wait_screen(terminal, ["> Back", "Read to the end"], ["> Apply change"]) + terminal.resize(*terminal.initial_size) + wait_screen(terminal, ["> Back", "Read to the end"], ["> Apply change"]) + checkpoint(terminal, evidence, output, "review-resize-return") if mutations(scene): - raise AssertionError("returning from help applied the draft") + raise AssertionError("resizing or returning from help applied the draft") + key(terminal, evidence, "Inspect draft after resize", b"\x1b") + for _ in range(4): + key(terminal, evidence, "Previous resized alert field", b"\x1b[Z") + wait_screen(terminal, ["Monthly USD", "75.50", "Review change"]) + checkpoint(terminal, evidence, output, "review-resize-draft", ("Monthly USD", "75.50")) + for _ in range(4): + key(terminal, evidence, "Next resized alert field", b"\t") + key(terminal, evidence, "Fresh review after resize", b"\r") + read_review(terminal, evidence, scene, output, "review-resized") key(terminal, evidence, "Select Apply again", b"\t") wait_screen(terminal, ["> Apply change"]) - checkpoint(terminal, evidence, output, "review-apply") - key(terminal, evidence, "Apply", b"\r") + checkpoint(terminal, evidence, output, name + "-apply") + key(terminal, evidence, "Apply" if name == "review" else "Apply refreshed", b"\r") def run_scene(terminal, scene, evidence, output): @@ -327,6 +497,24 @@ def run_scene(terminal, scene, evidence, output): evidence["printed_command"] = {"byte_offset": start, "byte_length": len(expected), "sha256": hashlib.sha256(expected).hexdigest()} checkpoint(terminal, evidence, output, "printed-command") + elif scene == "preflight-recover": + wait_screen(terminal, ["HTTP 400", "125.25", "Review change"], ["Saving"]) + checkpoint(terminal, evidence, output, "recover-failed", ("HTTP 400", "125.25")) + if mutations(scene): + raise AssertionError("failed preflight sent a mutation") + key(terminal, evidence, "Refresh retained draft", b"r") + wait_screen(terminal, ["Baseline refreshed.", "125.25", "Review change"], ["Loading"]) + checkpoint(terminal, evidence, output, "recover-refreshed", ("125.25", "Baseline refreshed.")) + if mutations(scene): + raise AssertionError("refresh applied the retained draft") + key(terminal, evidence, "Review recovered draft", b"\r") + review(terminal, evidence, scene, output, "refreshed-review") + wait_screen(terminal, ["Change confirmed."], ["Loading", "Saving"]) + checkpoint(terminal, evidence, output, "saved") + key(terminal, evidence, "Quit", b"q") + terminal.finish(0) + if len(mutations(scene)) != 1: + raise AssertionError("recovered preflight did not produce exactly one mutation") elif scene == "preflight-400": wait_screen(terminal, ["HTTP 400", "125.25", "Review change"], ["Saving"]) checkpoint(terminal, evidence, output, "preflight", ("HTTP 400", "125.25")) @@ -369,6 +557,12 @@ def run_scene(terminal, scene, evidence, output): raise AssertionError("uncertain write repeated automatically") key(terminal, evidence, "Back", b"\x1b") wait_screen(terminal, ["Writes locked"]) + key(terminal, evidence, "Refresh uncertain outcome", b"r") + wait_screen(terminal, ["Writes locked", "125.25"], ["Loading"]) + checkpoint(terminal, evidence, output, "uncertain-refreshed", ("Writes locked", "125.25")) + key(terminal, evidence, "Blocked change after refresh", b"\r") + if len(mutations(scene)) != 1: + raise AssertionError("refresh unlocked an uncertain write") key(terminal, evidence, "Quit", b"q") terminal.finish(1) else: @@ -421,7 +615,8 @@ def main(): "GOMAXPROCS": "2", "PAGER": "cat", "OPENAI_API_KEY": "synthetic-demo-key", "OPENAI_ADMIN_KEY": "synthetic-demo-admin-key", "OPENAI_BASE_URL": os.environ["OPENAI_BASE_URL"], "DEMO_CLI_EXIT": str(exit_receipt), "DEMO_TERMIOS_PYTHON": sys.executable, - "DEMO_TERMIOS_SNAPSHOT": str(terminal_snapshot)} + "DEMO_TERMIOS_SNAPSHOT": str(terminal_snapshot), + "DEMO_SCROLLBACK_ROWS": str(height * 2 + 4 if scene == "project-limit" else 0)} if os.environ.get("DEMO_NO_COLOR", "1") == "1": env.update(NO_COLOR="1", FORCE_COLOR="0") terminal = None @@ -437,6 +632,23 @@ def main(): raise AssertionError("Bash did not preserve the actual CLI exit status") evidence["cli_exit_status"] = int(status_text) evidence["terminal_modes"] = terminal.terminal_modes + newline = b"\r\n" if terminal.initial[1] & termios.OPOST and terminal.initial[1] & termios.ONLCR else b"\n" + marker = b"F40 workspace start" + newline + assert terminal.raw.count(marker) == 1, "missing or repeated prelude boundary" + evidence["prelude_boundary"] = terminal.raw.index(marker) + len(marker) + assert not terminal.observer.pending, "terminal ended inside a control sequence" + evidence["observer"] = {"bytes_observed": terminal.observer.bytes_observed, + "source_manifest_sha256": observer_sources.MANIFEST_SHA256, + "package_commits": dict(observer_sources.COMMITS), + "limitation": "Offline viewport model; resize clips without terminal reflow."} + history = terminal.observer.history() + evidence["prior_scrollback"] = {"seeded": scene == "project-limit", "retained": history} + if scene == "project-limit": + expected = [f"Prior shell output {number:03d}" for number in range(1, 4)] + positions = [history.index(line) for line in expected] + assert positions == sorted(positions), "prior shell scrollback changed order" + evidence["protocol_replies"] = terminal.protocol_replies + evidence["resize_events"] = [event for event in terminal.events if event[1] == "r"] evidence["staged_binary"]["sha256_after"] = hashlib.sha256(staged.read_bytes()).hexdigest() if (evidence["staged_binary"]["sha256_after"] != evidence["binary_sha256"] or hashlib.sha256(binary.read_bytes()).hexdigest() != evidence["binary_sha256"]): @@ -472,4 +684,9 @@ def main(): if __name__ == "__main__": - main() + if sys.argv[1:] == ["--check-observer"]: + probe_observer() + elif len(sys.argv) == 1: + main() + else: + raise SystemExit("usage: scene.py [--check-observer]") diff --git a/scripts/demos/limits-workspace/server.py b/scripts/demos/limits-workspace/server.py index afd55f1b..2330e440 100755 --- a/scripts/demos/limits-workspace/server.py +++ b/scripts/demos/limits-workspace/server.py @@ -15,7 +15,7 @@ SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", - "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400", "preflight-recover") def write_text(path, value): @@ -109,7 +109,8 @@ def handle_request(self): return limit_reads[scene] += 1 result = state["limit"] - if scene == "preflight-400" and limit_reads[scene] > 1: + if ((scene == "preflight-400" and limit_reads[scene] > 1) + or (scene == "preflight-recover" and limit_reads[scene] == 2)): status, result = 400, {"error": { "message": "synthetic-private-preflight-message\u001b[2J", "param": "synthetic-private-preflight-field\nline", diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 5cf93c93..8f172385 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -2,8 +2,10 @@ """Check recorded bytes, actual child statuses, and synthetic mutation bodies.""" import hashlib +import importlib.util import json import math +import re from pathlib import Path import sys import termios @@ -11,16 +13,35 @@ SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", - "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400", "preflight-recover") def restored_offset(raw, loading=False): - """Require inline readiness and cursor restoration after the last rendered frame.""" - assert b"\x1b[?1049h" not in raw and b"\x1b[?1049l" not in raw, "workspace entered an alternate screen" + """Require readiness and every emitted terminal reset before ordinary output.""" ready = raw.find(b"Loading" if loading else b"Website controls and source errors: ?") - hidden, restored = raw.find(b"\x1b[?25l"), raw.rfind(b"\x1b[?25h") - assert ready >= 0 and hidden >= 0 and restored > max(ready, hidden), "inline workspace did not restore its cursor" - return restored + len(b"\x1b[?25h") + assert ready >= 0, "workspace never reached its expected ready state" + states, seen = {}, set() + for match in re.finditer(rb"\x1b\[\?([0-9;]+)([hl])", raw): + enabled = match[2] == b"h" + for value in match[1].split(b";"): + if not value: + continue + mode = int(value) + assert mode not in (47, 1047, 1049), "workspace used an alternate screen" + if mode in (25, 2004): + states[mode] = enabled, match.end() + seen.add((mode, enabled)) + for match in re.finditer(rb"\x1b\[\??([0-9;]*)J", raw): + assert int(match[1].split(b";")[0] or b"0") not in (2, 3), "workspace cleared the screen or scrollback" + assert (25, False) in seen and (2004, True) in seen, "workspace did not initialize expected terminal modes" + assert states.get(25, (False, 0))[0] is True, "workspace left its cursor hidden" + assert states.get(2004, (True, 0))[0] is False, "workspace left bracketed paste enabled" + controls = list(re.finditer(rb"\x1b\[(?:[<>=?][ -?]*u|>[ -?]*m)", raw)) + expected = [b"\x1b[>4;2m", b"\x1b[>1u", b"\x1b[?u", b"\x1b[>4m", b"\x1b[<1u"] + assert [item[0] for item in controls] == expected, "workspace keyboard controls differ from the pinned lifecycle" + endings = [states[25][1], states[2004][1], controls[-2].end(), controls[-1].end()] + assert min(endings) > ready, "terminal modes were not restored after workspace readiness" + return max(endings) def validate_rendered_checkpoints(root): @@ -54,6 +75,37 @@ def validate_rendered_checkpoints(root): print("PASS: rendered checkpoint names, exact times, PNGs, and frame GIFs match source evidence without missing or extra files.") +def observed_scene(root, scene, evidence, child): + spec = importlib.util.spec_from_file_location("limits_replay_driver", Path(__file__).with_name("scene.py")) + driver = importlib.util.module_from_spec(spec) + spec.loader.exec_module(driver) + replies = [] + observer = driver.TerminalObserver(evidence["width"], evidence["height"], replies.append) + points = iter(evidence["checkpoints"]) + point = next(points, None) + for event in child[1:]: + while point is not None and point["seconds"] < event[0]: + expected = (root / (scene + "-evidence." + point["name"] + ".txt")).read_text() + assert observer.text() == expected, (scene, point["name"], "viewport differs from captured bytes") + point = next(points, None) + if event[1] == "o": + observer.feed(event[2].encode()) + elif event[1] == "r": + width, height = map(int, event[2].split("x")) + observer.resize(width, height) + else: + raise AssertionError("unexpected child capture event") + while point is not None: + expected = (root / (scene + "-evidence." + point["name"] + ".txt")).read_text() + assert observer.text() == expected, (scene, point["name"], "final viewport differs from capture") + point = next(points, None) + assert not observer.pending, (scene, "unfinished terminal control") + assert observer.text() == (root / (scene + "-evidence.exit.txt")).read_text() + assert observer.history() == evidence["prior_scrollback"]["retained"] + assert [reply.decode("ascii") for reply in replies] == evidence["protocol_replies"] + return observer + + def main(): if len(sys.argv) not in {2, 3} or len(sys.argv) == 3 and sys.argv[2] != "--rendered-checkpoints": raise SystemExit("usage: validate.py OUTPUT_DIRECTORY [--rendered-checkpoints]") @@ -61,6 +113,7 @@ def main(): if len(sys.argv) == 3: validate_rendered_checkpoints(root) return + assert (root / "observer-validation.txt").read_text() == "PASS: observer split/coalesced queries; keyboard controls; cursor bounds; retained history.\n" requests = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] for scene in SCENES: evidence = json.loads((root / (scene + "-evidence.json")).read_text()) @@ -88,6 +141,24 @@ def main(): raw = (root / (scene + "-evidence.tty")).read_bytes() restored_offset(raw, loading=scene == "slow-cancel") child = [json.loads(line) for line in (root / (scene + "-evidence.child.cast")).read_text().splitlines()] + observer = observed_scene(root, scene, evidence, child) + assert observer.bytes_observed == evidence["observer"]["bytes_observed"] == len(raw) + assert evidence["observer"]["source_manifest_sha256"] == "9d048014421425452323fff126d2b460d33357aef64692d07f9e142b6b9888ad" + newline = "\r\n" if initial[1] & termios.OPOST and initial[1] & termios.ONLCR else "\n" + rows = evidence["height"] * 2 + 4 if scene == "project-limit" else 0 + prelude = "".join(f"Prior shell output {number:03d}" + newline for number in range(1, rows + 1)) + prelude += "F40 workspace start" + newline + assert raw.startswith(prelude.encode()) and evidence["prelude_boundary"] == len(prelude.encode()) + assert evidence["prior_scrollback"]["seeded"] == (scene == "project-limit") + if scene == "project-limit": + expected = [f"Prior shell output {number:03d}" for number in range(1, 4)] + positions = [observer.history().index(line) for line in expected] + assert positions == sorted(positions), "prior scrollback changed order" + resize_events = [event for event in child[1:] if event[1] == "r"] + assert resize_events == evidence["resize_events"] + target = "40x12" if evidence["width"] == 80 else "80x24" + sizes = [event[2] for event in resize_events] + assert sizes == (["39x11", target, f'{evidence["width"]}x{evidence["height"]}'] if scene == "alert-create" else []) child_bytes = "".join(event[2] for event in child[1:] if event[1] == "o").encode() outer = [json.loads(line) for line in (root / (scene + ".cast")).read_text().splitlines()] outer_bytes = "".join(event[2] for event in outer[1:] if event[1] == "o").encode() @@ -176,18 +247,30 @@ def main(): continue if scene == "alert-create": required = ("validation", "validation-help", "draft-return", "review-back", - "review-top", "review-before-help", "review-help", "review-return", "review-apply", "saved") + "review-top", "review-before-help", "review-help", "review-help-small", "review-help-resized", + "review-return", "review-resize-return", "review-resize-draft", "review-resized-top", + "review-resized-command", "review-apply", "saved") assert all(name in checkpoints for name in required) assert all(checkpoints[left]["seconds"] < checkpoints[right]["seconds"] for left, right in zip(required, required[1:])) for name, messages in (("validation", ("Enter a USD amount", "Review change")), ("validation-help", ("Draft details", "two decimal places.")), ("draft-return", ("> Review change", "Demo")), ("review-back", ("Monthly USD", "75.50")), - ("review-return", ("> Back", "Apply change"))): + ("review-return", ("> Back", "Read to the end"))): screen = (root / (scene + "-evidence." + name + ".txt")).read_text() assert all(message in screen for message in messages), (scene, name, "missing draft recovery evidence") + if scene == "preflight-recover": + required = ("review-command", "review-apply", "recover-failed", "recover-refreshed", + "refreshed-review-command", "refreshed-review-apply", "saved") + assert all(name in checkpoints for name in required) + assert all(checkpoints[left]["seconds"] < checkpoints[right]["seconds"] for left, right in zip(required, required[1:])) + reads = [entry for entry in requests if entry["scene"] == scene and entry["path"].endswith("/spend_limit") and entry["method"] == "GET"] + assert len(reads) >= 4 and [entry["status"] for entry in reads[:2]] == [200, 400] + assert all(entry["status"] == 200 for entry in reads[2:]) + assert (root / (scene + "-evidence.recover-refreshed.txt")).read_text().count("125.25") >= 1 + assert [event["key"] for event in evidence["keys"]].count("Apply refreshed") == 1 request = mutations[0] - if scene in {"org-limit", "project-limit", "uncertain"}: + if scene in {"org-limit", "project-limit", "uncertain", "preflight-recover"}: assert request["body"] == {"currency": "USD", "interval": "month", "threshold_amount": 12525} elif scene in {"alert-create", "alert-edit"}: assert request["body"] == {"currency": "USD", "interval": "month", @@ -202,18 +285,23 @@ def main(): elif scene == "rate-update": assert request["body"] == {"max_requests_per_1_minute": 250, "max_tokens_per_1_day": 9007199254740993} if scene == "uncertain": + assert "uncertain-refreshed" in checkpoints + assert [event["key"] for event in evidence["keys"]].count("Refresh uncertain outcome") == 1 assert request["status"] == "response_lost_after_write" else: assert request["status"] == 200 assert "review-command" in checkpoints and "review-apply" in checkpoints assert checkpoints["review-command"]["seconds"] < checkpoints["review-apply"]["seconds"] assert [event["key"] for event in evidence["keys"]].count("Apply") == 1 - print("PASS: eleven real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") + print("PASS: twelve real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") print("PASS: one mutation per confirmed action; no retry after uncertain outcome; slow load canceled without mutation.") - print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; ten inspection previews; staged files removed.") + print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; eleven inspection previews; staged files removed.") print("PASS: all seven terminal mode fields restored before Bash exits; only Darwin PENDIN normalized.") print("PASS: Ctrl+P exits without mutation; the complete command prints byte-for-byte after terminal restoration.") print("PASS: preflight HTTP 400 retains the draft; safe F1 recovery; zero mutations; normal exit preserves failure.") + print("PASS: prior shell scrollback retained; child resize events reset Apply; draft survives help and resizing.") + print("PASS: failed preflight refresh recovers before one write; uncertain writes remain locked after refresh.") + print("PASS: paste, keyboard modes, and cursor reset before ordinary output; viewport checkpoints replay from exact bytes.") print("LIMIT: inspect all checkpoint images and GIF frames before claiming visual acceptance.") From e87682d6a5972d0010b5530f40eb25b1bf5907c0 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 02:22:06 -0700 Subject: [PATCH 32/44] test: align Limits footer assertion with compact layout --- internal/app/limits_workspace_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/limits_workspace_test.go b/internal/app/limits_workspace_test.go index ebdc9007..a747d18a 100644 --- a/internal/app/limits_workspace_test.go +++ b/internal/app/limits_workspace_test.go @@ -1083,7 +1083,7 @@ func TestLimitsWorkspaceListStatusStaysVisible(t *testing.T) { require.Contains(t, view, status+" "+label) require.Contains(t, view, tab) require.Contains(t, view, "Reported Costs (incomplete): USD 1.23") - require.Contains(t, view, "q: quit") + require.Contains(t, view, "q quit") require.NotContains(t, view, "true") require.NotContains(t, view, "false") require.LessOrEqual(t, len(strings.Split(view, "\n")), m.height) From 87fee817a3ec223a92012c37301a6a9c36c576b1 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 02:28:30 -0700 Subject: [PATCH 33/44] test: pass generated commands to Limits process startup --- cmd/openai/limits_workflow_unix_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go index 71e9fe2f..1bf8099d 100644 --- a/cmd/openai/limits_workflow_unix_test.go +++ b/cmd/openai/limits_workflow_unix_test.go @@ -58,7 +58,7 @@ func TestMainLimitsBorrowedWriterProcess(t *testing.T) { commands.Command.Writer, commands.Command.Reader = borrowed, os.Stdin ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) defer cancel() - code := app.Run(ctx, os.Args[separator+1:]) + code := app.Run(ctx, os.Args[separator+1:], commands.Command) if err := ctx.Err(); err != nil { t.Fatal("workspace required fallback cancellation", err) } From 87e4ba55b52ae7bb987274171bd929aafd226e77 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 03:02:11 -0700 Subject: [PATCH 34/44] fix: preserve limits capture timing and daily token labels --- cmd/openai/limits_workflow_test.go | 16 +++-- .../transform/project_limits_overview.go | 1 + .../transform/project_limits_overview_test.go | 6 +- scripts/demos/limits-workspace/README.md | 3 + scripts/demos/limits-workspace/scene.py | 3 + .../demos/limits-workspace/test_validate.py | 61 +++++++++++++++++++ scripts/demos/limits-workspace/validate.py | 24 ++++++++ 7 files changed, 109 insertions(+), 5 deletions(-) create mode 100644 scripts/demos/limits-workspace/test_validate.py diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go index 2b3aef86..35694377 100644 --- a/cmd/openai/limits_workflow_test.go +++ b/cmd/openai/limits_workflow_test.go @@ -16,7 +16,7 @@ import ( ) const limitsWorkflowProject = `{"id":"proj_synthetic","object":"organization.project","name":"Synthetic project","status":"active","future":0.100000000000000001}` -const limitsWorkflowRate = `{"id":"rl_synthetic","object":"project.rate_limit","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993,"max_audio_megabytes_per_1_minute":1.25,"future":{"exact":123456789012345678901234567890}}` +const limitsWorkflowRate = `{"id":"rl_synthetic","object":"project.rate_limit","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993,"max_tokens_per_1_day":9007199254740995,"max_audio_megabytes_per_1_minute":1.25,"future":{"exact":123456789012345678901234567890}}` func limitsWorkflowArgs(project bool, show bool) []string { args := []string{"admin", "limits"} @@ -200,7 +200,11 @@ func TestMainLimitsWorkflowReadableAndFormats(t *testing.T) { switch mode { case "bare", "show": case "extraction": - args = append([]string{"--transform", "spend_limit.threshold_amount"}, args...) + field := "spend_limit.threshold_amount" + if project { + field = "project_rate_limits.0.max_tokens_per_1_day" + } + args = append([]string{"--transform", field}, args...) case "raw-output": args = append([]string{"--raw-output"}, args...) default: @@ -228,7 +232,7 @@ func TestMainLimitsWorkflowReadableAndFormats(t *testing.T) { } } if project { - for _, text := range []string{"Max requests per minute: 0", "Max tokens per minute: 9007199254740993", "Max audio megabytes per minute: 1.25"} { + for _, text := range []string{"Max requests per minute: 0", "Max tokens per minute: 9007199254740993", "Max tokens per day: 9007199254740995", "Max audio megabytes per minute: 1.25"} { if !strings.Contains(got.stdout, text) { t.Errorf("rate units lost %q: %q", text, got.stdout) } @@ -241,7 +245,11 @@ func TestMainLimitsWorkflowReadableAndFormats(t *testing.T) { } } case "extraction": - if got.stdout != "9007199254740993\n" { + wantExtraction := "9007199254740993\n" + if project { + wantExtraction = "9007199254740995\n" + } + if got.stdout != wantExtraction { t.Errorf("overview extraction changed: %q", got.stdout) } default: diff --git a/internal/output/transform/project_limits_overview.go b/internal/output/transform/project_limits_overview.go index 1d0ee0be..2e753c44 100644 --- a/internal/output/transform/project_limits_overview.go +++ b/internal/output/transform/project_limits_overview.go @@ -99,6 +99,7 @@ var limitsOverviewRateLabels = map[string]string{ "max_images_per_1_minute": "max_images_per_minute", "max_audio_megabytes_per_1_minute": "max_audio_megabytes_per_minute", "max_requests_per_1_day": "max_requests_per_day", + "max_tokens_per_1_day": "max_tokens_per_day", "batch_1_day_max_input_tokens": "max_batch_input_tokens_per_day", } diff --git a/internal/output/transform/project_limits_overview_test.go b/internal/output/transform/project_limits_overview_test.go index cadc2d95..ff6298a1 100644 --- a/internal/output/transform/project_limits_overview_test.go +++ b/internal/output/transform/project_limits_overview_test.go @@ -33,7 +33,7 @@ func TestProjectLimitsOverviewSpendScopes(t *testing.T) { } func TestProjectLimitsOverviewExactRateUnits(t *testing.T) { - const rate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993123456789,"max_images_per_1_minute":-1,"max_audio_megabytes_per_1_minute":0.1234567890123456789,"max_requests_per_1_day":1e9,"batch_1_day_max_input_tokens":9223372036854775807,"future":{"count":9007199254740993}}` + const rate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993123456789,"max_images_per_1_minute":-1,"max_audio_megabytes_per_1_minute":0.1234567890123456789,"max_requests_per_1_day":1e9,"max_tokens_per_1_day":9007199254740995,"batch_1_day_max_input_tokens":9223372036854775807,"future":{"count":9007199254740993}}` raw := `{"scope":"project","project_id":"proj_synthetic","project":{"id":"proj_synthetic","future":1e100},"spend_limit":null,"spend_limit_configured":false,"spend_alerts":[],"spend_alerts_complete":true,"project_rate_limits":[` + rate + `],"project_rate_limits_complete":true,"source_errors":[]}` value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) if err != nil { @@ -44,6 +44,7 @@ func TestProjectLimitsOverviewExactRateUnits(t *testing.T) { "max_requests_per_minute": "0", "max_tokens_per_minute": "9007199254740993123456789", "max_images_per_minute": "-1", "max_audio_megabytes_per_minute": "0.1234567890123456789", "max_requests_per_day": "1e9", "max_batch_input_tokens_per_day": "9223372036854775807", + "max_tokens_per_day": "9007199254740995", } { if projected.Get(label).Raw != want { t.Errorf("rate %s changed units or precision: %s", label, projected.Raw) @@ -65,6 +66,9 @@ func TestProjectLimitsOverviewRatePresenceAndAmbiguity(t *testing.T) { for _, raw := range []string{ `{"object":"project.rate_limit","id":"rl_synthetic"}`, `{"object":"project.rate_limit","max_requests_per_1_minute":null,"max_tokens_per_1_minute":"100"}`, + `{"object":"project.rate_limit","max_tokens_per_1_day":null}`, + `{"object":"project.rate_limit","max_tokens_per_1_day":"100"}`, + `{"object":"project.rate_limit","max_tokens_per_1_day":100,"max_tokens_per_day":null}`, `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_minute":null}`, `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_1_minute":900}`, `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_1_\u006dinute":900}`, diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index 1a54b0b9..f0606f8a 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -78,6 +78,9 @@ The scene driver restores terminal output processing and closes every owned chil The recorder preserves raw child bytes, child casts, outer casts, request logs, hashes, and actual process statuses. The validator checks byte identity across all three capture forms. +The child cast ends with an empty output event at the actual observation-end timestamp. +This event preserves observed quiet time without adding terminal bytes or changing checkpoint times. +The validator requires finite, monotonic event times and checkpoints within that observation interval. It independently replays child bytes and resize events through pinned pyte and wcwidth sources. The observer answers cursor queries at their actual positions, including fragmented and repeated queries. The recorder probes query fragmentation, keyboard controls, cursor bounds, and history before recording. diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index d1cb10e6..b65dff8a 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -627,6 +627,9 @@ def main(): terminal = RelayTerminal("/bin/bash", shell_args, env, width=width, height=height, controlling_terminal=True, terminal_snapshot=terminal_snapshot) run_scene(terminal, scene, evidence, output) + # Preserve observed quiet time without adding terminal bytes or moving checkpoints. + evidence["observation_end_seconds"] = time.monotonic() - terminal.started + terminal.events.append([evidence["observation_end_seconds"], "o", ""]) status_text = exit_receipt.read_text() if status_text != str(terminal.child.returncode) + "\n": raise AssertionError("Bash did not preserve the actual CLI exit status") diff --git a/scripts/demos/limits-workspace/test_validate.py b/scripts/demos/limits-workspace/test_validate.py new file mode 100644 index 00000000..377a95a4 --- /dev/null +++ b/scripts/demos/limits-workspace/test_validate.py @@ -0,0 +1,61 @@ +#!/usr/bin/env python3 +"""Check quiet-tail timing without a terminal or renderer.""" + +import copy +import importlib.util +from pathlib import Path +import unittest + + +spec = importlib.util.spec_from_file_location("limits_capture_validation", Path(__file__).with_name("validate.py")) +validator = importlib.util.module_from_spec(spec) +spec.loader.exec_module(validator) + + +class TimelineContracts(unittest.TestCase): + def setUp(self): + self.evidence = {"width": 80, "height": 24, "observation_end_seconds": 1.25, + "checkpoints": [{"name": "printed-command", "seconds": 1.0}]} + self.child = [{"version": 2, "width": 80, "height": 24}, + [0.25, "o", "\x1b[?25hprinted command\r\n"], [1.25, "o", ""]] + + def test_quiet_checkpoint_preserves_exact_terminal_bytes(self): + validator.validate_child_timeline(self.evidence, self.child) + raw = "".join(event[2] for event in self.child[1:] if event[1] == "o").encode() + self.assertEqual(raw, b"\x1b[?25hprinted command\r\n") + self.assertLess(self.child[1][0], self.evidence["checkpoints"][0]["seconds"]) + + def test_missing_quiet_tail_is_rejected(self): + with self.assertRaises(AssertionError): + validator.validate_child_timeline(self.evidence, self.child[:-1]) + + def test_checkpoint_beyond_observation_is_rejected(self): + self.evidence["checkpoints"][0]["seconds"] = 1.251 + with self.assertRaises(AssertionError): + validator.validate_child_timeline(self.evidence, self.child) + + def test_invalid_or_reversed_times_are_rejected(self): + for location in ("end", "event", "checkpoint"): + for value in (float("nan"), float("inf"), -0.1, True): + with self.subTest(location=location, value=value): + evidence, child = copy.deepcopy(self.evidence), copy.deepcopy(self.child) + if location == "end": + evidence["observation_end_seconds"] = child[-1][0] = value + elif location == "event": + child[1][0] = value + else: + evidence["checkpoints"][0]["seconds"] = value + with self.assertRaises(AssertionError): + validator.validate_child_timeline(evidence, child) + self.child.insert(2, [0.1, "o", "late event"]) + with self.assertRaises(AssertionError): + validator.validate_child_timeline(self.evidence, self.child) + + def test_renderer_dimensions_must_match_observation(self): + self.child[0]["width"] = 40 + with self.assertRaises(AssertionError): + validator.validate_child_timeline(self.evidence, self.child) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 8f172385..f51c9def 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -16,6 +16,29 @@ "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400", "preflight-recover") +def validate_child_timeline(evidence, child): + """Bind checkpoints and the byte-free final event to actual observation time.""" + end = evidence["observation_end_seconds"] + valid_time = lambda value: type(value) in {int, float} and math.isfinite(value) and value >= 0 + assert valid_time(end), "invalid observation end" + assert len(child) >= 2 and child[0].get("version") == 2, "missing child cast header" + assert (child[0].get("width"), child[0].get("height")) == (evidence["width"], evidence["height"]) + assert child[-1] == [end, "o", ""], "missing byte-free observation-end event" + previous = 0 + for index, event in enumerate(child[1:], 1): + assert type(event) is list and len(event) == 3, "invalid child event" + seconds, kind, data = event + assert valid_time(seconds) and previous <= seconds <= end, "invalid child event time" + assert kind in {"o", "r"} and type(data) is str, "invalid child event payload" + assert data or index == len(child) - 1 and kind == "o", "unexpected empty child event" + previous = seconds + previous = 0 + for point in evidence["checkpoints"]: + seconds = point["seconds"] + assert valid_time(seconds) and previous <= seconds <= end, "checkpoint exceeds its observation interval" + previous = seconds + + def restored_offset(raw, loading=False): """Require readiness and every emitted terminal reset before ordinary output.""" ready = raw.find(b"Loading" if loading else b"Website controls and source errors: ?") @@ -141,6 +164,7 @@ def main(): raw = (root / (scene + "-evidence.tty")).read_bytes() restored_offset(raw, loading=scene == "slow-cancel") child = [json.loads(line) for line in (root / (scene + "-evidence.child.cast")).read_text().splitlines()] + validate_child_timeline(evidence, child) observer = observed_scene(root, scene, evidence, child) assert observer.bytes_observed == evidence["observer"]["bytes_observed"] == len(raw) assert evidence["observer"]["source_manifest_sha256"] == "9d048014421425452323fff126d2b460d33357aef64692d07f9e142b6b9888ad" From a2ec77604fd7751bc9a834d2528f243a44ebeadf Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 03:43:36 -0700 Subject: [PATCH 35/44] fix(demos): replay resized Limits viewports before rendering --- scripts/demos/limits-workspace/README.md | 17 +- scripts/demos/limits-workspace/record.sh | 17 +- .../limits-workspace/render_viewports.py | 176 ++++++++++++++++++ .../limits-workspace/test_render_viewports.py | 137 ++++++++++++++ scripts/demos/limits-workspace/validate.py | 113 +++++++++++ 5 files changed, 446 insertions(+), 14 deletions(-) create mode 100644 scripts/demos/limits-workspace/render_viewports.py create mode 100644 scripts/demos/limits-workspace/test_render_viewports.py diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index f0606f8a..fae22105 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -88,9 +88,24 @@ The project scene checks retained scrollback markers in emulator history. Limits supplies a resizable observer because the shared observer supports fixed audio sizes only. The driver retains shared PTY allocation and cleanup. The observer clips during resize; it does not model native terminal reflow. +The viewport exporter replays original output and resize events at their original timestamps. +It creates separately labeled rendering casts from observed cells, colors, attributes, and cursor state. +Original child bytes, casts, checkpoint times, and process evidence remain unchanged. +Checkpoint PNGs use their actual viewport dimensions, including temporary resize dimensions. +Animations use a fixed-font, letterboxed canvas that contains every observed viewport. +These animations represent observer-derived child viewports, rather than the outer PTY mirror. +The comparison cast preserves the outer replay. The comparison GIF uses separately bound child viewport projections. +The manifest binds rendering casts to original captures, snapshots, dimensions, and exporter hashes. Checkpoint renderers receive no stdin. The recorder logs their exact names, timestamps, and output filenames. A second validation pass compares that inventory with source checkpoints and all PNG and frame-GIF filenames. -It rejects missing, duplicate, altered, or unexpected checkpoint artifacts. +It rejects missing, duplicate, or unexpected checkpoint artifacts. +It verifies derived-cast content and raster dimensions. It does not compare every decoded pixel. +It compares projected cells, styles, cursor state, and letterboxing against original event replay. +It also rejects stale viewport dimensions and inconsistent animation canvases. +Actual renderer tests compare shrink and enlargement against independently specified pixel controls. +Pixel controls and full-frame visual inspection remain required. +Use fresh output directories when rerendering preserved captures. +Derived casts reject replacement. Media exports replace legacy GIFs and PNGs inside that fresh copy. It checks exact mutation bodies and guards against automatic retry after an uncertain write. Terminal captures merge stdout and stderr. They do not prove separate stream routing. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh index df437552..0fb86899 100755 --- a/scripts/demos/limits-workspace/record.sh +++ b/scripts/demos/limits-workspace/record.sh @@ -40,11 +40,12 @@ cp "$demo_runtime/scene.sh" "$demo_output/scene.sh" echo "child terminal: ${record_width}x${record_height}; outer terminal: ${record_width}x$((record_height+4))" echo "NO_COLOR: $record_no_color; GOMAXPROCS=2; isolated HOME; CI unset" echo 'render: Menlo 18px, asciinema theme, shared capture_and_render lifecycle' + echo 'render projection: observer-derived child viewports; active-grid PNGs; fixed-font letterboxed animations' demo_capture_metadata "$demo_python" --version } > "$demo_output/metadata.txt" shasum -a 256 "$record_source/record.sh" "$record_source/server.py" "$record_source/scene.py" \ - "$record_source/validate.py" "$record_source/README.md" \ + "$record_source/validate.py" "$record_source/render_viewports.py" "$record_source/README.md" \ "$record_root/scripts/demos/capture_and_render.sh" "$record_root/scripts/image_picker_harness.py" \ "$record_root/scripts/demos/audio-workbench/observer.py" "$record_root/scripts/demos/audio-workbench/observer_sources.json" \ "$record_root/internal/app/limits_commands.go" "$record_root/internal/app/limits_service.go" \ @@ -68,19 +69,9 @@ demo_stop_api test "$record_digest" = "$(shasum -a 256 "$demo_after" | cut -d ' ' -f 1)" shasum -a 256 -c "$demo_output/source-sha256.txt" > "$demo_output/source-validation.txt" "$demo_python" -I -B "$record_source/validate.py" "$demo_output" > "$demo_output/validation.txt" +# agg uses a fixed terminal grid. Replay resizes before producing derived viewports. +"$demo_python" -I -B "$record_source/render_viewports.py" "$demo_output" --agg "$demo_agg" --ffmpeg "$demo_ffmpeg" demo_assemble_capture 150 "${record_scenes[@]}" -# Select checkpoint frames from the exact child captures. No UI text is reconstructed. -: > "$demo_output/checkpoint-frames.txt" -for record_scene in "${record_scenes[@]}"; do - while IFS=$'\t' read -r record_name record_seconds; do - "$demo_agg" --quiet "${demo_render_options[@]}" --select "$record_seconds" \ - "$demo_output/$record_scene-evidence.child.cast" "$demo_output/$record_scene-$record_name-frame.gif" < /dev/null - "$demo_ffmpeg" -nostdin -hide_banner -loglevel error -y -i "$demo_output/$record_scene-$record_name-frame.gif" \ - -frames:v 1 "$demo_output/$record_scene-$record_name.png" < /dev/null - printf '%s\t%s\t%s\t%s\t%s\n' "$record_scene" "$record_name" "$record_seconds" \ - "$record_scene-$record_name-frame.gif" "$record_scene-$record_name.png" >> "$demo_output/checkpoint-frames.txt" - done < <("$demo_python" -I -B -c 'import json,sys; data=json.load(open(sys.argv[1])); [print(item["name"]+"\t"+str(item["seconds"])) for item in data["checkpoints"]]' "$demo_output/$record_scene-evidence.json") -done "$demo_python" -I -B "$record_source/validate.py" "$demo_output" --rendered-checkpoints >> "$demo_output/validation.txt" ( cd "$demo_output" diff --git a/scripts/demos/limits-workspace/render_viewports.py b/scripts/demos/limits-workspace/render_viewports.py new file mode 100644 index 00000000..8717f8f9 --- /dev/null +++ b/scripts/demos/limits-workspace/render_viewports.py @@ -0,0 +1,176 @@ +#!/usr/bin/env python3 +"""Render observer-derived viewports while preserving immutable child captures.""" + +import argparse +import hashlib +import importlib.util +import json +from pathlib import Path +import re +import subprocess + + +COLORS = ("black", "red", "green", "brown", "blue", "magenta", "cyan", "white") +SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400", "preflight-recover") + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def color_codes(value, background=False): + if value == "default": + return [49 if background else 39] + if value in COLORS: + return [(40 if background else 30) + COLORS.index(value)] + if value.startswith("bright") and value[6:] in COLORS: + return [(100 if background else 90) + COLORS.index(value[6:])] + assert re.fullmatch("[0-9a-fA-F]{6}", value), ("unsupported observed color", value) + return [48 if background else 38, 2, *bytes.fromhex(value)] + + +def cell_style(cell): + codes = [0, *color_codes(cell.fg), *color_codes(cell.bg, True)] + for field, code in (("bold", 1), ("italics", 3), ("underscore", 4), + ("blink", 5), ("reverse", 7), ("strikethrough", 9)): + if getattr(cell, field): + codes.append(code) + return "\x1b[" + ";".join(map(str, codes)) + "m" + + +def viewport_bytes(screen): + """Serialize observed cells, including wide-cell placeholders and combining marks.""" + output = ["\x1b[?25l\x1b[0m\x1b[2J\x1b[H"] + style = None + for row in range(screen.lines): + output.append(f"\x1b[{row + 1};1H") + for column in range(screen.columns): + cell = screen.buffer[row][column] + if not cell.data: + continue # A wide glyph already occupies this continuation cell. + wanted = cell_style(cell) + if wanted != style: + output.append(wanted) + style = wanted + output.append(cell.data) + output.append("\x1b[0m") + output.append(f"\x1b[{screen.cursor.y + 1};{min(screen.cursor.x, screen.columns - 1) + 1}H") + output.append("\x1b[?25l" if screen.cursor.hidden else "\x1b[?25h") + return "".join(output) + + +def write_cast(path, width, height, events, provenance): + assert not path.exists() and not path.is_symlink(), ("use a fresh rendering directory", path) + header = {"version": 2, "width": width, "height": height, + "title": "Observer-derived viewport; original captured bytes remain separate", + "limits_viewport": provenance} + path.write_text("\n".join(json.dumps(item, ensure_ascii=False) for item in [header, *events]) + "\n") + + +def driver_module(): + spec = importlib.util.spec_from_file_location("limits_viewport_driver", Path(__file__).with_name("scene.py")) + driver = importlib.util.module_from_spec(spec) + spec.loader.exec_module(driver) + return driver + + +def prepare(root): + driver = driver_module() + captures = [] + for scene in SCENES: + evidence_path = root / (scene + "-evidence.json") + child_path = root / (scene + "-evidence.child.cast") + evidence = json.loads(evidence_path.read_text()) + child = [json.loads(line) for line in child_path.read_text().splitlines()] + driver.validator.validate_child_timeline(evidence, child) + captures.append((scene, evidence_path, child_path, evidence, child)) + sizes = [(item[3]["width"], item[3]["height"]) for item in captures] + sizes += [tuple(map(int, event[2].split("x"))) for item in captures for event in item[4][1:] if event[1] == "r"] + canvas = (max(size[0] for size in sizes), max(size[1] for size in sizes)) + manifest = {"renderer_sha256": digest(__file__), "observer_sha256": digest(Path(__file__).with_name("scene.py")), + "animation_canvas": list(canvas), "projection": "observer-derived, fixed-font, letterboxed replay", + "limitation": "Offline viewport model; resize clips without terminal reflow.", "scenes": []} + for scene, evidence_path, child_path, evidence, child in captures: + source = {"child_cast_sha256": digest(child_path), "evidence_sha256": digest(evidence_path), + "renderer_sha256": manifest["renderer_sha256"]} + observer = driver.TerminalObserver(evidence["width"], evidence["height"], lambda data: None) + points = iter(evidence["checkpoints"]) + point = next(points, None) + frames, frame_sources, entries = [], [], [] + + def snapshot(name, seconds, expected_path): + assert re.fullmatch("[a-z0-9-]+", name), "invalid checkpoint name" + assert observer.text() == expected_path.read_text(), (scene, name, "source viewport differs") + width, height = observer.screen.columns, observer.screen.lines + path = root / (scene + "-" + name + "-viewport.cast") + provenance = {**source, "checkpoint": name, "seconds": seconds, "active_size": [width, height], + "snapshot_sha256": digest(expected_path)} + write_cast(path, width, height, [[seconds, "o", viewport_bytes(observer.screen)], + [evidence["observation_end_seconds"], "o", ""]], provenance) + entries.append({"name": name, "seconds": seconds, "active_size": [width, height], + "cast": path.name, "sha256": digest(path), "source": provenance}) + + for event_index, event in enumerate(child[1:], 1): + while point is not None and point["seconds"] < event[0]: + snapshot(point["name"], point["seconds"], root / (scene + "-evidence." + point["name"] + ".txt")) + point = next(points, None) + if event[1] == "r": + observer.resize(*map(int, event[2].split("x"))) + else: + observer.feed(event[2].encode()) + if event[1] == "r" or event[2]: + frames.append([event[0], "o", viewport_bytes(observer.screen)]) + frame_sources.append({"source_event_index": event_index, "seconds": event[0], + "active_size": [observer.screen.columns, observer.screen.lines]}) + while point is not None: + snapshot(point["name"], point["seconds"], root / (scene + "-evidence." + point["name"] + ".txt")) + point = next(points, None) + snapshot("exit", evidence["observation_end_seconds"], root / (scene + "-evidence.exit.txt")) + animation = root / (scene + "-viewport.cast") + frames.append([evidence["observation_end_seconds"], "o", ""]) + write_cast(animation, *canvas, frames, {**source, "projection": manifest["projection"]}) + manifest["scenes"].append({"scene": scene, "source": source, "animation": animation.name, + "animation_sha256": digest(animation), "animation_events": frame_sources, + "checkpoints": entries}) + (root / "viewport-manifest.json").write_text(json.dumps(manifest, indent=2) + "\n") + return manifest + + +def render(root, manifest, agg, ffmpeg): + # Derived stills retain absolute checkpoint times, including their initial quiet interval. + observation_end = max(point["seconds"] for entry in manifest["scenes"] for point in entry["checkpoints"]) + options = [agg, "--quiet", "--renderer", "resvg", "--font-family", "Menlo", "--font-size", "18", + "--line-height", "1.2", "--theme", "asciinema", "--fps-cap", "15", "--last-frame-duration", "2", + "--idle-time-limit", str(observation_end)] + + def run(argv): + subprocess.run(argv, check=True, stdin=subprocess.DEVNULL, timeout=30) + + rows = [] + for entry in manifest["scenes"]: + scene = entry["scene"] + run([*options, str(root / entry["animation"]), str(root / (scene + ".gif"))]) + for point in entry["checkpoints"]: + base = scene if point["name"] == "exit" else scene + "-" + point["name"] + frame, png = base + "-frame.gif", base + ".png" + run([*options, "--select", str(point["seconds"]), str(root / point["cast"]), str(root / frame)]) + run([ffmpeg, "-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", str(root / frame), + "-frames:v", "1", str(root / png)]) + if point["name"] != "exit": + rows.append("\t".join((scene, point["name"], str(point["seconds"]), frame, png))) + (root / "checkpoint-frames.txt").write_text("\n".join(rows) + "\n") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + parser.add_argument("--agg", required=True) + parser.add_argument("--ffmpeg", required=True) + args = parser.parse_args() + manifest = prepare(args.directory) + render(args.directory, manifest, args.agg, args.ffmpeg) + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/limits-workspace/test_render_viewports.py b/scripts/demos/limits-workspace/test_render_viewports.py new file mode 100644 index 00000000..00d28fa3 --- /dev/null +++ b/scripts/demos/limits-workspace/test_render_viewports.py @@ -0,0 +1,137 @@ +#!/usr/bin/env python3 +"""Check viewport attributes and real renderer behavior across shrink and enlargement.""" + +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + + +spec = importlib.util.spec_from_file_location("limits_viewport_export", Path(__file__).with_name("render_viewports.py")) +renderer = importlib.util.module_from_spec(spec) +spec.loader.exec_module(renderer) + + +class ViewportContracts(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.driver = renderer.driver_module() + + def test_default_named_bright_and_rgb_colors(self): + self.assertEqual(renderer.color_codes("default"), [39]) + self.assertEqual(renderer.color_codes("default", True), [49]) + self.assertEqual(renderer.color_codes("brown"), [33]) + self.assertEqual(renderer.color_codes("brightblue", True), [104]) + self.assertEqual(renderer.color_codes("8ebaff"), [38, 2, 142, 186, 255]) + with self.assertRaises(AssertionError): + renderer.color_codes("invalid-color") + + def test_styles_wide_cells_and_combining_marks_round_trip(self): + original = self.driver.TerminalObserver(20, 4, lambda data: None) + original.feed("\x1b[1;3;4;5;7;9;38;2;142;186;255;104m界e\u0301\x1b[0m".encode()) + projected = self.driver.TerminalObserver(20, 4, lambda data: None) + projected.feed(renderer.viewport_bytes(original.screen).encode()) + self.assertEqual(projected.text(), original.text()) + for column in range(20): + self.assertEqual(projected.screen.buffer[0][column], original.screen.buffer[0][column]) + + def test_cursor_visibility_and_position_round_trip(self): + for hidden in (False, True): + with self.subTest(hidden=hidden): + original = self.driver.TerminalObserver(20, 4, lambda data: None) + original.feed(b"\x1b[3;7H") + original.screen.cursor.hidden = hidden + projected = self.driver.TerminalObserver(20, 4, lambda data: None) + projected.feed(renderer.viewport_bytes(original.screen).encode()) + self.assertEqual((projected.screen.cursor.x, projected.screen.cursor.y, projected.screen.cursor.hidden), + (6, 2, hidden)) + + def test_letterbox_clears_prior_cells_without_scaling(self): + original = self.driver.TerminalObserver(20, 4, lambda data: None) + original.feed(b"\x1b[?25lshort") + projected = self.driver.TerminalObserver(40, 12, lambda data: None) + projected.feed(b"stale content\x1b[10;30Hstale") + projected.feed(renderer.viewport_bytes(original.screen).encode()) + self.assertEqual(projected.screen.display[0], "short".ljust(40)) + self.assertTrue(all(not line.strip() for line in projected.screen.display[1:])) + + def test_static_cast_preserves_checkpoint_and_observation_times(self): + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / "frame.cast" + events = [[1.5, "o", "payload"], [2.25, "o", ""]] + renderer.write_cast(path, 40, 12, events, {"seconds": 1.5}) + actual = [json.loads(line) for line in path.read_text().splitlines()] + self.assertEqual((actual[0]["width"], actual[0]["height"]), (40, 12)) + self.assertEqual(actual[1:], events) + with self.assertRaises(AssertionError): + renderer.write_cast(path, 40, 12, events, {}) + + +class ActualRendererResizeContracts(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.driver = renderer.driver_module() + cls.agg = os.environ["LIMITS_DEMO_AGG"] + cls.ffmpeg = os.environ["LIMITS_DEMO_FFMPEG"] + + def render(self, directory, name, size, events): + cast, gif = directory / (name + ".cast"), directory / (name + ".gif") + renderer.write_cast(cast, *size, events, {}) + subprocess.run([self.agg, "--quiet", "--renderer", "resvg", "--font-family", "Menlo", "--font-size", "18", + "--line-height", "1.2", "--theme", "asciinema", "--select", "2", str(cast), str(gif)], + check=True, stdin=subprocess.DEVNULL, timeout=15) + pixels = subprocess.check_output([self.ffmpeg, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(gif), + "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgba", "-"], + stdin=subprocess.DEVNULL, timeout=15) + return hashlib.sha256(pixels).hexdigest(), gif.read_bytes()[6:10] + + def test_shrink_and_enlarge_checkpoints_match_independent_pixels(self): + for initial, target in (((80, 24), (40, 12)), ((40, 12), (80, 24))): + with self.subTest(initial=initial, target=target), tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + payload = "\x1b[?25l\x1b[0m\x1b[2J\x1b[H\x1b[38;2;142;186;255mResize correct\x1b[0m" + observer = self.driver.TerminalObserver(*initial, lambda data: None) + observer.feed(b"old content that must disappear") + observer.resize(*target) + observer.feed(payload.encode()) + actual = self.render(directory, "projected", target, [[1, "o", renderer.viewport_bytes(observer.screen)], [3, "o", ""]]) + expected = self.render(directory, "expected", target, [[1, "o", payload], [3, "o", ""]]) + legacy = self.render(directory, "legacy-fixed-grid", initial, + [[0.1, "o", "old content"], [0.5, "r", f"{target[0]}x{target[1]}"], [1, "o", payload], [3, "o", ""]]) + self.assertEqual(actual, expected) + self.assertNotEqual(legacy[1], expected[1]) + + def test_shrink_and_enlarge_animations_preserve_wrapping_and_letterbox(self): + for initial, target in (((80, 24), (40, 12)), ((40, 12), (80, 24))): + with self.subTest(initial=initial, target=target), tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + prefix = "\x1b[?25l\x1b[0m\x1b[2J\x1b[H\x1b[38;2;142;186;255m" + column = target[0] - 5 + observer = self.driver.TerminalObserver(*initial, lambda data: None) + observer.feed(b"old viewport") + observer.resize(*target) + observer.feed((prefix + f"\x1b[1;{column}H" + "X" * 10 + "\x1b[0m").encode()) + expected = prefix + f"\x1b[1;{column}H" + "X" * 6 + "\x1b[2;1H" + "X" * 4 + "\x1b[0m" + actual = self.render(directory, "animation-projection", (80, 24), + [[1, "o", renderer.viewport_bytes(observer.screen)], [3, "o", ""]]) + self.assertEqual(actual, self.render(directory, "independent-wrap", (80, 24), [[1, "o", expected], [3, "o", ""]])) + + def test_public_export_preserves_late_checkpoint_after_initial_quiet(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + events = [[13, "o", "\x1b[?25lquiet checkpoint"], [13, "o", ""]] + renderer.write_cast(directory / "sample-viewport.cast", 40, 12, events, {}) + renderer.write_cast(directory / "sample-exit-viewport.cast", 40, 12, events, {}) + manifest = {"scenes": [{"scene": "sample", "animation": "sample-viewport.cast", + "checkpoints": [{"name": "exit", "seconds": 13, "cast": "sample-exit-viewport.cast"}]}]} + renderer.render(directory, manifest, self.agg, self.ffmpeg) + self.assertTrue((directory / "sample.png").stat().st_size) + self.assertTrue((directory / "sample-frame.gif").stat().st_size) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index f51c9def..4d6c71b6 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -6,6 +6,7 @@ import json import math import re +import struct from pathlib import Path import sys import termios @@ -68,6 +69,7 @@ def restored_offset(raw, loading=False): def validate_rendered_checkpoints(root): + validate_viewport_projections(root) expected_rows = [] expected_pngs = {scene + ".png" for scene in SCENES} expected_frames = {scene + "-frame.gif" for scene in SCENES} @@ -98,6 +100,117 @@ def validate_rendered_checkpoints(root): print("PASS: rendered checkpoint names, exact times, PNGs, and frame GIFs match source evidence without missing or extra files.") +def image_size(path): + data = path.read_bytes() + if path.suffix == ".png": + assert data[:8] == b"\x89PNG\r\n\x1a\n", "invalid PNG header" + return struct.unpack(">II", data[16:24]) + assert data[:6] in (b"GIF87a", b"GIF89a"), "invalid GIF header" + return struct.unpack("= source.lines or column >= source.columns: + assert projected.buffer[row][column] == projected.default_char, "letterbox contains stale cells" + + for entry in manifest["scenes"]: + scene = entry["scene"] + child_path, evidence_path = root / (scene + "-evidence.child.cast"), root / (scene + "-evidence.json") + child = [json.loads(line) for line in child_path.read_text().splitlines()] + evidence = json.loads(evidence_path.read_text()) + sizes.append((evidence["width"], evidence["height"])) + sizes.extend(tuple(map(int, event[2].split("x"))) for event in child[1:] if event[1] == "r") + assert entry["source"] == {"child_cast_sha256": digest(child_path), "evidence_sha256": digest(evidence_path), + "renderer_sha256": manifest["renderer_sha256"]} + assert entry["animation"] == scene + "-viewport.cast" + animation_path = root / entry["animation"] + casts.add(animation_path.name) + assert digest(animation_path) == entry["animation_sha256"] + animation = [json.loads(line) for line in animation_path.read_text().splitlines()] + assert (animation[0]["width"], animation[0]["height"]) == canvas + assert animation[0]["limits_viewport"] == {**entry["source"], "projection": manifest["projection"]} + original_events = [(index, event) for index, event in enumerate(child[1:], 1) if event[1] == "r" or event[2]] + assert len(animation) == len(original_events) + 2 + animated_source = driver.TerminalObserver(evidence["width"], evidence["height"], lambda data: None) + animated = driver.TerminalObserver(*canvas, lambda data: None) + for (index, event), projected, provenance in zip(original_events, animation[1:-1], entry["animation_events"], strict=True): + if event[1] == "r": + animated_source.resize(*map(int, event[2].split("x"))) + else: + animated_source.feed(event[2].encode()) + assert projected[:2] == [event[0], "o"], "animation event order or time changed" + assert provenance == {"source_event_index": index, "seconds": event[0], + "active_size": [animated_source.screen.columns, animated_source.screen.lines]} + animated.feed(projected[2].encode()) + same_viewport(animated_source.screen, animated.screen, True) + original = driver.TerminalObserver(evidence["width"], evidence["height"], lambda data: None) + original_index = 1 + points = [*evidence["checkpoints"], {"name": "exit", "seconds": evidence["observation_end_seconds"]}] + assert len(entry["checkpoints"]) == len(points) + for point, projection in zip(points, entry["checkpoints"], strict=True): + name, seconds = point["name"], point["seconds"] + assert (projection["name"], projection["seconds"]) == (name, seconds) + while original_index < len(child) and child[original_index][0] <= seconds: + event = child[original_index] + if event[1] == "r": + original.resize(*map(int, event[2].split("x"))) + else: + original.feed(event[2].encode()) + original_index += 1 + size = (original.screen.columns, original.screen.lines) + assert tuple(projection["active_size"]) == size + expected = root / (scene + "-evidence." + name + ".txt") + assert original.text() == expected.read_text(), "original checkpoint changed" + assert projection["cast"] == scene + "-" + name + "-viewport.cast" + path = root / projection["cast"] + casts.add(path.name) + assert digest(path) == projection["sha256"] + static = [json.loads(line) for line in path.read_text().splitlines()] + assert (static[0]["width"], static[0]["height"]) == size + assert static[0]["limits_viewport"] == projection["source"] + assert projection["source"] == {**entry["source"], "checkpoint": name, "seconds": seconds, + "active_size": list(size), "snapshot_sha256": digest(expected)} + assert len(static) == 3 and static[1][:2] == [seconds, "o"] + assert static[2] == [evidence["observation_end_seconds"], "o", ""] + still = driver.TerminalObserver(*size, lambda data: None) + still.feed(static[1][2].encode()) + same_viewport(original.screen, still.screen) + base = scene if name == "exit" else scene + "-" + name + pixels = image_size(root / (base + ".png")) + assert pixels == image_size(root / (base + "-frame.gif")), "PNG and GIF dimensions differ" + assert pixel_sizes.setdefault(size, pixels) == pixels, "same grid has inconsistent dimensions" + assert animation[-1] == [evidence["observation_end_seconds"], "o", ""] + assert {path.name for path in root.glob("*-viewport.cast")} == casts + assert canvas == (max(size[0] for size in sizes), max(size[1] for size in sizes)), "animation canvas omits an observed grid" + for first, first_pixels in pixel_sizes.items(): + for second, second_pixels in pixel_sizes.items(): + for dimension in (0, 1): + if first[dimension] < second[dimension]: + assert first_pixels[dimension] < second_pixels[dimension], "resized viewport retained stale pixel dimensions" + for scene in SCENES: + assert image_size(root / (scene + ".gif")) == pixel_sizes[canvas], "animation canvas differs" + + def observed_scene(root, scene, evidence, child): spec = importlib.util.spec_from_file_location("limits_replay_driver", Path(__file__).with_name("scene.py")) driver = importlib.util.module_from_spec(spec) From f4f2e765c81697da64b8770024dade7a8e5d223c Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 04:11:39 -0700 Subject: [PATCH 36/44] fix(demos): select static snapshots inside a stable rendering interval --- scripts/demos/limits-workspace/README.md | 7 ++- .../limits-workspace/render_viewports.py | 19 ++++--- .../limits-workspace/test_render_viewports.py | 54 ++++++++++++++----- scripts/demos/limits-workspace/validate.py | 8 +-- 4 files changed, 64 insertions(+), 24 deletions(-) diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md index fae22105..f0b67625 100644 --- a/scripts/demos/limits-workspace/README.md +++ b/scripts/demos/limits-workspace/README.md @@ -88,9 +88,12 @@ The project scene checks retained scrollback markers in emulator history. Limits supplies a resizable observer because the shared observer supports fixed audio sizes only. The driver retains shared PTY allocation and cleanup. The observer clips during resize; it does not model native terminal reflow. -The viewport exporter replays original output and resize events at their original timestamps. +The viewport exporter preserves original animation event times, including resize events. It creates separately labeled rendering casts from observed cells, colors, attributes, and cursor state. Original child bytes, casts, checkpoint times, and process evidence remain unchanged. +Static rendering casts use a separate 0–2 second interval and select its midpoint. +Their metadata retains exact source checkpoint times. This avoids selection at fractional timestamp boundaries. +Animation rendering casts retain original event times. Checkpoint PNGs use their actual viewport dimensions, including temporary resize dimensions. Animations use a fixed-font, letterboxed canvas that contains every observed viewport. These animations represent observer-derived child viewports, rather than the outer PTY mirror. @@ -102,7 +105,7 @@ It rejects missing, duplicate, or unexpected checkpoint artifacts. It verifies derived-cast content and raster dimensions. It does not compare every decoded pixel. It compares projected cells, styles, cursor state, and letterboxing against original event replay. It also rejects stale viewport dimensions and inconsistent animation canvases. -Actual renderer tests compare shrink and enlargement against independently specified pixel controls. +Actual renderer tests compare shrink, enlargement, fractional checkpoints, late checkpoints, and exits against independent decoded-pixel controls. Pixel controls and full-frame visual inspection remain required. Use fresh output directories when rerendering preserved captures. Derived casts reject replacement. Media exports replace legacy GIFs and PNGs inside that fresh copy. diff --git a/scripts/demos/limits-workspace/render_viewports.py b/scripts/demos/limits-workspace/render_viewports.py index 8717f8f9..7aff50ce 100644 --- a/scripts/demos/limits-workspace/render_viewports.py +++ b/scripts/demos/limits-workspace/render_viewports.py @@ -13,6 +13,7 @@ COLORS = ("black", "red", "green", "brown", "blue", "magenta", "cyan", "white") SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400", "preflight-recover") +STATIC_RENDER_TIMELINE = {"output_seconds": 0, "selection_seconds": 1, "end_seconds": 2} def digest(path): @@ -90,7 +91,8 @@ def prepare(root): canvas = (max(size[0] for size in sizes), max(size[1] for size in sizes)) manifest = {"renderer_sha256": digest(__file__), "observer_sha256": digest(Path(__file__).with_name("scene.py")), "animation_canvas": list(canvas), "projection": "observer-derived, fixed-font, letterboxed replay", - "limitation": "Offline viewport model; resize clips without terminal reflow.", "scenes": []} + "limitation": "Offline viewport model; resize clips without terminal reflow.", + "static_render_timeline": dict(STATIC_RENDER_TIMELINE), "scenes": []} for scene, evidence_path, child_path, evidence, child in captures: source = {"child_cast_sha256": digest(child_path), "evidence_sha256": digest(evidence_path), "renderer_sha256": manifest["renderer_sha256"]} @@ -105,9 +107,11 @@ def snapshot(name, seconds, expected_path): width, height = observer.screen.columns, observer.screen.lines path = root / (scene + "-" + name + "-viewport.cast") provenance = {**source, "checkpoint": name, "seconds": seconds, "active_size": [width, height], - "snapshot_sha256": digest(expected_path)} - write_cast(path, width, height, [[seconds, "o", viewport_bytes(observer.screen)], - [evidence["observation_end_seconds"], "o", ""]], provenance) + "snapshot_sha256": digest(expected_path), + "static_render_timeline": manifest["static_render_timeline"]} + # This clock renders a static snapshot. Provenance retains the original checkpoint time. + write_cast(path, width, height, [[STATIC_RENDER_TIMELINE["output_seconds"], "o", viewport_bytes(observer.screen)], + [STATIC_RENDER_TIMELINE["end_seconds"], "o", ""]], provenance) entries.append({"name": name, "seconds": seconds, "active_size": [width, height], "cast": path.name, "sha256": digest(path), "source": provenance}) @@ -138,11 +142,11 @@ def snapshot(name, seconds, expected_path): def render(root, manifest, agg, ffmpeg): - # Derived stills retain absolute checkpoint times, including their initial quiet interval. + # Select inside the static interval, away from fractional source timestamp boundaries. observation_end = max(point["seconds"] for entry in manifest["scenes"] for point in entry["checkpoints"]) options = [agg, "--quiet", "--renderer", "resvg", "--font-family", "Menlo", "--font-size", "18", "--line-height", "1.2", "--theme", "asciinema", "--fps-cap", "15", "--last-frame-duration", "2", - "--idle-time-limit", str(observation_end)] + "--idle-time-limit", str(max(observation_end, STATIC_RENDER_TIMELINE["end_seconds"]))] def run(argv): subprocess.run(argv, check=True, stdin=subprocess.DEVNULL, timeout=30) @@ -154,7 +158,8 @@ def run(argv): for point in entry["checkpoints"]: base = scene if point["name"] == "exit" else scene + "-" + point["name"] frame, png = base + "-frame.gif", base + ".png" - run([*options, "--select", str(point["seconds"]), str(root / point["cast"]), str(root / frame)]) + run([*options, "--select", str(manifest["static_render_timeline"]["selection_seconds"]), + str(root / point["cast"]), str(root / frame)]) run([ffmpeg, "-nostdin", "-hide_banner", "-loglevel", "error", "-y", "-i", str(root / frame), "-frames:v", "1", str(root / png)]) if point["name"] != "exit": diff --git a/scripts/demos/limits-workspace/test_render_viewports.py b/scripts/demos/limits-workspace/test_render_viewports.py index 00d28fa3..9933eb22 100644 --- a/scripts/demos/limits-workspace/test_render_viewports.py +++ b/scripts/demos/limits-workspace/test_render_viewports.py @@ -7,6 +7,7 @@ import os from pathlib import Path import subprocess +import sys import tempfile import unittest @@ -59,7 +60,7 @@ def test_letterbox_clears_prior_cells_without_scaling(self): self.assertEqual(projected.screen.display[0], "short".ljust(40)) self.assertTrue(all(not line.strip() for line in projected.screen.display[1:])) - def test_static_cast_preserves_checkpoint_and_observation_times(self): + def test_cast_keeps_explicit_times_and_rejects_replacement(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) / "frame.cast" events = [[1.5, "o", "payload"], [2.25, "o", ""]] @@ -84,10 +85,13 @@ def render(self, directory, name, size, events): subprocess.run([self.agg, "--quiet", "--renderer", "resvg", "--font-family", "Menlo", "--font-size", "18", "--line-height", "1.2", "--theme", "asciinema", "--select", "2", str(cast), str(gif)], check=True, stdin=subprocess.DEVNULL, timeout=15) - pixels = subprocess.check_output([self.ffmpeg, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(gif), + return self.pixel_hash(gif), gif.read_bytes()[6:10] + + def pixel_hash(self, path): + pixels = subprocess.check_output([self.ffmpeg, "-nostdin", "-hide_banner", "-loglevel", "error", "-i", str(path), "-frames:v", "1", "-f", "rawvideo", "-pix_fmt", "rgba", "-"], stdin=subprocess.DEVNULL, timeout=15) - return hashlib.sha256(pixels).hexdigest(), gif.read_bytes()[6:10] + return hashlib.sha256(pixels).hexdigest() def test_shrink_and_enlarge_checkpoints_match_independent_pixels(self): for initial, target in (((80, 24), (40, 12)), ((40, 12), (80, 24))): @@ -120,17 +124,43 @@ def test_shrink_and_enlarge_animations_preserve_wrapping_and_letterbox(self): [[1, "o", renderer.viewport_bytes(observer.screen)], [3, "o", ""]]) self.assertEqual(actual, self.render(directory, "independent-wrap", (80, 24), [[1, "o", expected], [3, "o", ""]])) - def test_public_export_preserves_late_checkpoint_after_initial_quiet(self): + def test_public_export_fractional_late_and_exit_checkpoints_match_pixels(self): with tempfile.TemporaryDirectory() as temporary: directory = Path(temporary) - events = [[13, "o", "\x1b[?25lquiet checkpoint"], [13, "o", ""]] - renderer.write_cast(directory / "sample-viewport.cast", 40, 12, events, {}) - renderer.write_cast(directory / "sample-exit-viewport.cast", 40, 12, events, {}) - manifest = {"scenes": [{"scene": "sample", "animation": "sample-viewport.cast", - "checkpoints": [{"name": "exit", "seconds": 13, "cast": "sample-exit-viewport.cast"}]}]} - renderer.render(directory, manifest, self.agg, self.ffmpeg) - self.assertTrue((directory / "sample.png").stat().st_size) - self.assertTrue((directory / "sample-frame.gif").stat().st_size) + # Fractional values reproduce timestamps from the rejected synthetic exports. + cases = ((0.9312871660222299, 5.30981912498828), + (0.9358217500266619, 7.4997836670372635), (13.25, 15.125)) + payloads, times = {}, {} + for index, scene in enumerate(renderer.SCENES): + checkpoint, end = cases[index % len(cases)] + payload = "\x1b[?25l\x1b[0m\x1b[2J\x1b[Hcheckpoint " + scene + payloads[scene], times[scene] = payload, (checkpoint, end) + observer = self.driver.TerminalObserver(40, 12, lambda data: None) + observer.feed(payload.encode()) + evidence = {"width": 40, "height": 12, "observation_end_seconds": end, + "checkpoints": [{"name": "overview", "seconds": checkpoint}]} + (directory / (scene + "-evidence.json")).write_text(json.dumps(evidence)) + child = [{"version": 2, "width": 40, "height": 12}, [0.1, "o", payload], [end, "o", ""]] + (directory / (scene + "-evidence.child.cast")).write_text("\n".join(map(json.dumps, child)) + "\n") + for point in ("overview", "exit"): + (directory / (scene + "-evidence." + point + ".txt")).write_text(observer.text()) + originals = {path.name: hashlib.sha256(path.read_bytes()).hexdigest() for path in directory.iterdir()} + subprocess.run([sys.executable, "-I", "-B", str(Path(renderer.__file__)), str(directory), + "--agg", self.agg, "--ffmpeg", self.ffmpeg], + check=True, stdin=subprocess.DEVNULL, timeout=30) + self.driver.validator.validate_viewport_projections(directory) + manifest = json.loads((directory / "viewport-manifest.json").read_text()) + self.assertEqual(manifest["static_render_timeline"], {"output_seconds": 0, "selection_seconds": 1, "end_seconds": 2}) + for entry in manifest["scenes"]: + scene = entry["scene"] + self.assertEqual(tuple(point["seconds"] for point in entry["checkpoints"]), times[scene]) + expected = self.render(directory, scene + "-independent", (40, 12), + [[1, "o", payloads[scene]], [3, "o", ""]])[0] + for name in (scene + "-overview.png", scene + ".png"): + with self.subTest(scene=scene, frame=name): + self.assertEqual(self.pixel_hash(directory / name), expected) + for name, expected in originals.items(): + self.assertEqual(hashlib.sha256((directory / name).read_bytes()).hexdigest(), expected) if __name__ == "__main__": diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py index 4d6c71b6..bee517c6 100755 --- a/scripts/demos/limits-workspace/validate.py +++ b/scripts/demos/limits-workspace/validate.py @@ -115,6 +115,7 @@ def validate_viewport_projections(root): driver = importlib.util.module_from_spec(spec) spec.loader.exec_module(driver) manifest = json.loads((root / "viewport-manifest.json").read_text()) + assert manifest["static_render_timeline"] == {"output_seconds": 0, "selection_seconds": 1, "end_seconds": 2} digest = lambda path: hashlib.sha256(path.read_bytes()).hexdigest() assert manifest["renderer_sha256"] == digest(Path(__file__).with_name("render_viewports.py")) assert manifest["observer_sha256"] == digest(Path(__file__).with_name("scene.py")) @@ -189,9 +190,10 @@ def same_viewport(source, projected, padded=False): assert (static[0]["width"], static[0]["height"]) == size assert static[0]["limits_viewport"] == projection["source"] assert projection["source"] == {**entry["source"], "checkpoint": name, "seconds": seconds, - "active_size": list(size), "snapshot_sha256": digest(expected)} - assert len(static) == 3 and static[1][:2] == [seconds, "o"] - assert static[2] == [evidence["observation_end_seconds"], "o", ""] + "active_size": list(size), "snapshot_sha256": digest(expected), + "static_render_timeline": manifest["static_render_timeline"]} + assert len(static) == 3 and static[1][:2] == [0, "o"] + assert static[2] == [2, "o", ""] still = driver.TerminalObserver(*size, lambda data: None) still.feed(static[1][2].encode()) same_viewport(original.screen, still.screen) From 21210c5e56193d17963e14594892182c62015ebd Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 04:50:07 -0700 Subject: [PATCH 37/44] fix: share daily token flags across compatible rate routes --- cmd/openai/limits_workflow_test.go | 4 ++++ internal/app/limits_commands.go | 9 +++++---- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go index 35694377..f29e6194 100644 --- a/cmd/openai/limits_workflow_test.go +++ b/cmd/openai/limits_workflow_test.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "testing" + "github.com/openai/openai-cli/internal/app" "github.com/tidwall/gjson" ) @@ -426,6 +427,9 @@ func TestMainLimitsWorkflowEnvironmentHeadersAndDebug(t *testing.T) { if r.Header.Get("X-Limits-Env") != "synthetic-private-header" { t.Error("overview lost configured environment headers") } + if r.Header.Get("User-Agent") != "OpenAI/CLI "+app.Version || r.Header.Get("X-Stainless-Package-Version") != app.Version { + t.Error("overview lost the CLI version metadata") + } return limitsWorkflowSuccess(t, false, r) }) env := []string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_API_KEY=sk-fake-spend-test", "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "OPENAI_CUSTOM_HEADERS=X-Limits-Env: synthetic-private-header", "FORCE_COLOR=0"} diff --git a/internal/app/limits_commands.go b/internal/app/limits_commands.go index 4e72cd72..4ad84a51 100644 --- a/internal/app/limits_commands.go +++ b/internal/app/limits_commands.go @@ -44,6 +44,10 @@ func registerLimitsCommands(root *cli.Command) { routes = append(routes, projects.Command("rate-limits")) } // Subgroup cloning already created the preserved colon route. + dailyTokens := &requestflag.Flag[int64]{ + Name: "max-tokens-per-1-day", BodyPath: "max_tokens_per_1_day", + Usage: "The maximum number of tokens per day for this model.", + } for _, rates := range routes { if rates == nil { continue @@ -59,10 +63,7 @@ func registerLimitsCommands(root *cli.Command) { } } if !found { - update.Flags = append(update.Flags, &requestflag.Flag[int64]{ - Name: "max-tokens-per-1-day", BodyPath: "max_tokens_per_1_day", - Usage: "The maximum number of tokens per day for this model.", - }) + update.Flags = append(update.Flags, dailyTokens) } } } From 83d1f067b31726529c3a11e4583a234f6238b930 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 04:53:03 -0700 Subject: [PATCH 38/44] fix: retain spend mutation outcomes across presentation failures --- cmd/openai/spend_error_recovery_test.go | 42 +++++++++++++++++++++++++ internal/app/cmdutil.go | 10 ++++++ internal/app/limits_workspace.go | 3 ++ internal/app/limits_workspace_test.go | 28 +++++++++++++++++ internal/app/spend_error_guidance.go | 23 +++++++++----- 5 files changed, 98 insertions(+), 8 deletions(-) diff --git a/cmd/openai/spend_error_recovery_test.go b/cmd/openai/spend_error_recovery_test.go index f2f0ec1a..6fdc1cdf 100644 --- a/cmd/openai/spend_error_recovery_test.go +++ b/cmd/openai/spend_error_recovery_test.go @@ -195,6 +195,48 @@ func TestMainSpendErrorRecoveryResponseConversion(t *testing.T) { } } +func TestMainSpendMutationInvalidResponseFormats(t *testing.T) { + for _, target := range []struct { + name string + args []string + }{ + {"organization limit", []string{"admin", "organization", "spend-limit", "update", "--currency", "USD", "--interval", "month", "--threshold-amount", "10000"}}, + {"project alert", []string{"projects", "spend-alerts", "create", "--project-id", "proj_synthetic", "--currency", "USD", "--interval", "month", "--threshold-amount", "10000", "--notification-channel", `{"type":"email","recipients":["synthetic@example.invalid"]}`}}, + {"project rate", []string{"projects", "rate-limits", "update-rate-limit", "--project-id", "proj_synthetic", "--rate-limit-id", "rl_synthetic", "--max-tokens-per-1-day", "0"}}, + } { + for _, body := range []string{"", `{"synthetic-private-response": broken}`} { + t.Run(fmt.Sprintf("%s/empty=%t", target.name, body == ""), func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + if r.Method != http.MethodPost { + t.Errorf("unexpected mutation method: %s", r.Method) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + for _, output := range [][]string{ + nil, {"--format", "auto"}, {"--format", "text"}, {"--format", "json"}, + {"--format", "jsonl"}, {"--format", "raw"}, {"--format", "pretty"}, {"--format", "yaml"}, + {"--format", "text", "--transform", "synthetic-private-response"}, + } { + t.Run(strings.Join(output, " "), func(t *testing.T) { + args := append([]string{"openai", "--format-error", "text"}, target.args...) + before := requests.Load() + got := runMainDispatchWithEnv(t, "bash", spendRecoveryEnvironment(t, server.URL), append(args, output...)...) + checkSpendRecoveryFailure(t, got, "The API may have received the change.", "Check its current state before repeating") + if requests.Load() != before+1 { + t.Fatal("invalid response retried the mutation") + } + }) + } + }) + } + } +} + func TestMainSpendErrorRecoveryInputDecodeDoesNotClaimMutation(t *testing.T) { for _, test := range []struct{ name, input string }{ {"syntax", `[synthetic-private-input not closed`}, diff --git a/internal/app/cmdutil.go b/internal/app/cmdutil.go index 9c490afb..7a4749c0 100644 --- a/internal/app/cmdutil.go +++ b/internal/app/cmdutil.go @@ -305,6 +305,16 @@ func showJSON(res gjson.Result, opts ShowJSONOpts, selectTransformer transformer if err := opts.Context.Err(); err != nil { return err } + // Validate before projections or extraction can conceal an invalid response. + if isSpendMutationResponse(opts) && !gjson.Valid(res.Raw) { + var raw json.RawMessage + if err := json.Unmarshal([]byte(res.Raw), &raw); err != nil { + return spendMutationResponseFailure(err, opts) + } + } + if err := opts.Context.Err(); err != nil { + return err + } if handled, err := filecommands.ShowResult(res, filecommands.ResultOptions{ReadableOptions: fileResultOptions(opts), Quiet: quietOutput(opts.Context)}); handled { return err } diff --git a/internal/app/limits_workspace.go b/internal/app/limits_workspace.go index f33009ce..8f95fe57 100644 --- a/internal/app/limits_workspace.go +++ b/internal/app/limits_workspace.go @@ -207,6 +207,9 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error if m.output != nil { outputErr = m.output.Err() } + if outputErr != nil && m.lastSaved.Exists() && message == "" { + message = "A limits change was confirmed, but terminal output failed. Inspect its current state before another change.\n" + m.readRecovery() + } interrupted := m.interrupted || errors.Is(runErr, tea.ErrInterrupted) if ctx.Err() != nil { causes = append(causes, ctx.Err()) diff --git a/internal/app/limits_workspace_test.go b/internal/app/limits_workspace_test.go index a747d18a..0291af05 100644 --- a/internal/app/limits_workspace_test.go +++ b/internal/app/limits_workspace_test.go @@ -1758,6 +1758,34 @@ func TestLimitsWorkspaceFinalizationPreservesIndependentFailures(t *testing.T) { } } +func TestLimitsWorkspaceConfirmedSaveSurvivesTerminalFailure(t *testing.T) { + for _, kind := range []string{"limit_set", "alert_create", "limit_delete"} { + t.Run(kind, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month"}`) + if kind == "alert_create" { + m.workerSaved = gjson.Parse(limitsControllerAlert) + } else if kind == "limit_delete" { + m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit.deleted","deleted":true}`) + } + m.workerSavedOperation = limitsChangeOperation(kind, false) + m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} + var out bytes.Buffer + err := finishLimitsWorkspace(t.Context(), m, tea.ErrProgramKilled, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard}) + var safe *limitsWorkflowFailure + if !errors.As(err, &safe) || !errors.Is(err, syscall.EPIPE) || clioutput.IsBrokenPipe(err) { + t.Fatalf("terminal failure hid the confirmed outcome: %v", err) + } + if !strings.Contains(safe.Error(), "change was confirmed") || !strings.Contains(safe.Error(), "terminal output failed") || !strings.Contains(safe.Error(), "before another change") { + t.Fatalf("missing confirmed-save recovery: %v", err) + } + if out.Len() != 0 || m.mutationPending || m.lastSaved.Raw != m.workerSaved.Raw { + t.Fatal("finalization repeated output or lost the confirmed result") + } + }) + } +} + func TestLimitsWorkspaceReceiptUsesMutationProjection(t *testing.T) { for _, format := range []string{"text", "json"} { t.Run(format, func(t *testing.T) { diff --git a/internal/app/spend_error_guidance.go b/internal/app/spend_error_guidance.go index 222a600f..1c2ae268 100644 --- a/internal/app/spend_error_guidance.go +++ b/internal/app/spend_error_guidance.go @@ -28,9 +28,21 @@ func (e *spendMutationResponseError) Error() string { return e.cause.Error() } func (e *spendMutationResponseError) Unwrap() error { return e.cause } func spendMutationResponseFailure(err error, opts ShowJSONOpts) error { - if opts.OutputKind != OutputResponse { + if !isSpendMutationResponse(opts) { return err } + var syntaxError *json.SyntaxError + var typeError *json.UnmarshalTypeError + if errors.As(err, &syntaxError) || errors.As(err, &typeError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return &spendMutationResponseError{cause: err} + } + return err +} + +func isSpendMutationResponse(opts ShowJSONOpts) bool { + if opts.OutputKind != OutputResponse { + return false + } switch opts.Operation { case "(resource) admin.organization.spend_limit > (method) update", "(resource) admin.organization.spend_limit > (method) delete", @@ -43,15 +55,10 @@ func spendMutationResponseFailure(err error, opts ShowJSONOpts) error { "(resource) admin.organization.projects.spend_alerts > (method) update", "(resource) admin.organization.projects.spend_alerts > (method) delete", "(resource) admin.organization.projects.rate_limits > (method) update_rate_limit": + return true default: - return err + return false } - var syntaxError *json.SyntaxError - var typeError *json.UnmarshalTypeError - if errors.As(err, &syntaxError) || errors.As(err, &typeError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { - return &spendMutationResponseError{cause: err} - } - return err } // A status alone cannot establish configuration at an arbitrary API address. From a80de10bc2662e7c9e9e3c1ad90c311823e6e2f5 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 04:54:40 -0700 Subject: [PATCH 39/44] test: preserve cancellation during mutation response validation --- internal/app/cmdutil.go | 2 +- internal/app/spend_error_guidance_test.go | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/internal/app/cmdutil.go b/internal/app/cmdutil.go index 7a4749c0..9740f79f 100644 --- a/internal/app/cmdutil.go +++ b/internal/app/cmdutil.go @@ -309,7 +309,7 @@ func showJSON(res gjson.Result, opts ShowJSONOpts, selectTransformer transformer if isSpendMutationResponse(opts) && !gjson.Valid(res.Raw) { var raw json.RawMessage if err := json.Unmarshal([]byte(res.Raw), &raw); err != nil { - return spendMutationResponseFailure(err, opts) + return errors.Join(spendMutationResponseFailure(err, opts), opts.Context.Err()) } } if err := opts.Context.Err(); err != nil { diff --git a/internal/app/spend_error_guidance_test.go b/internal/app/spend_error_guidance_test.go index 179b3f08..44ece7c5 100644 --- a/internal/app/spend_error_guidance_test.go +++ b/internal/app/spend_error_guidance_test.go @@ -82,6 +82,26 @@ func TestSpendErrorGuidanceResponseConversionProvenance(t *testing.T) { require.False(t, errors.As(failure, &marker), "pre-conversion cancellation must retain its classification") } +func TestSpendMutationResponseValidationPreservesCancellation(t *testing.T) { + for _, body := range []string{"", `{"synthetic-private-response": broken}`, `{"id":"alert_synthetic"}`} { + t.Run(body, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + checking := &cancelListTableContext{Context: ctx, cancel: cancel, after: 2} + var out bytes.Buffer + err := ShowJSON(gjson.Parse(body), ShowJSONOpts{Context: checking, Format: "text", + Operation: "(resource) admin.organization.spend_alerts > (method) create", + OutputKind: OutputResponse, Stdout: &out, Stderr: io.Discard}) + require.ErrorIs(t, err, context.Canceled) + require.Empty(t, out.String()) + if !gjson.Valid(body) { + var syntaxFailure *json.SyntaxError + require.ErrorAs(t, err, &syntaxFailure) + } + }) + } +} + func TestSpendErrorGuidanceAdminMetadata(t *testing.T) { root := readableErrorTestCommand(t) root.Metadata["help-invocation"] = "'/tmp/synthetic app/openai'" From a8f9becef9ace2883f6fd7af1ec2b678610870f9 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 04:55:39 -0700 Subject: [PATCH 40/44] fix: keep confirmed-save recovery independent of unsent drafts --- internal/app/limits_workspace.go | 13 +++++++++++-- internal/app/limits_workspace_test.go | 18 ++++++++++++++++-- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/internal/app/limits_workspace.go b/internal/app/limits_workspace.go index 8f95fe57..c1f0d1c7 100644 --- a/internal/app/limits_workspace.go +++ b/internal/app/limits_workspace.go @@ -208,7 +208,7 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error outputErr = m.output.Err() } if outputErr != nil && m.lastSaved.Exists() && message == "" { - message = "A limits change was confirmed, but terminal output failed. Inspect its current state before another change.\n" + m.readRecovery() + message = "A limits change was confirmed, but terminal output failed. Inspect its current state before another change.\n" + m.readRecoveryFor("overview") } interrupted := m.interrupted || errors.Is(runErr, tea.ErrInterrupted) if ctx.Err() != nil { @@ -295,6 +295,10 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error } func (m *limitsWorkspace) readRecovery() string { + return m.readRecoveryFor(m.change.Kind) +} + +func (m *limitsWorkspace) readRecoveryFor(kind string) string { invocation := m.invocation if invocation == "" { invocation = "openai" @@ -303,7 +307,12 @@ func (m *limitsWorkspace) readRecovery() string { if m.snapshot.ProjectID != "" { resource += "projects " } - switch m.change.Kind { + switch kind { + case "overview": + resource = "admin limits show" + if m.snapshot.ProjectID != "" { + resource = "projects limits show" + } case "alert_create", "alert_update", "alert_delete": resource += "spend-alerts list" case "rate_update": diff --git a/internal/app/limits_workspace_test.go b/internal/app/limits_workspace_test.go index 0291af05..f8f737aa 100644 --- a/internal/app/limits_workspace_test.go +++ b/internal/app/limits_workspace_test.go @@ -1759,7 +1759,7 @@ func TestLimitsWorkspaceFinalizationPreservesIndependentFailures(t *testing.T) { } func TestLimitsWorkspaceConfirmedSaveSurvivesTerminalFailure(t *testing.T) { - for _, kind := range []string{"limit_set", "alert_create", "limit_delete"} { + for _, kind := range []string{"limit_set", "alert_create", "limit_delete", "rate_update"} { t.Run(kind, func(t *testing.T) { m, _ := newLimitsController(t, 80, 24) m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month"}`) @@ -1767,8 +1767,15 @@ func TestLimitsWorkspaceConfirmedSaveSurvivesTerminalFailure(t *testing.T) { m.workerSaved = gjson.Parse(limitsControllerAlert) } else if kind == "limit_delete" { m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit.deleted","deleted":true}`) + } else if kind == "rate_update" { + m.snapshot.ProjectID = "proj_synthetic" + m.workerSaved = gjson.Parse(limitsControllerRate) + } + m.workerSavedOperation = limitsChangeOperation(kind, m.snapshot.ProjectID != "") + m.change.Kind = "limit_set" + if kind == "limit_set" { + m.change.Kind = "alert_create" } - m.workerSavedOperation = limitsChangeOperation(kind, false) m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} var out bytes.Buffer err := finishLimitsWorkspace(t.Context(), m, tea.ErrProgramKilled, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard}) @@ -1779,6 +1786,13 @@ func TestLimitsWorkspaceConfirmedSaveSurvivesTerminalFailure(t *testing.T) { if !strings.Contains(safe.Error(), "change was confirmed") || !strings.Contains(safe.Error(), "terminal output failed") || !strings.Contains(safe.Error(), "before another change") { t.Fatalf("missing confirmed-save recovery: %v", err) } + readOptions := "help admin limits show" + if m.snapshot.ProjectID != "" { + readOptions = "help projects limits show" + } + if !strings.Contains(safe.Error(), readOptions) || strings.Contains(safe.Error(), "spend-limit retrieve") || strings.Contains(safe.Error(), "spend-alerts list") { + t.Fatalf("unsent draft redirected confirmed-save recovery: %v", err) + } if out.Len() != 0 || m.mutationPending || m.lastSaved.Raw != m.workerSaved.Raw { t.Fatal("finalization repeated output or lost the confirmed result") } From 0d5a6dcbe10c73e011d166820ffe4605823865ba Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 04:58:37 -0700 Subject: [PATCH 41/44] test: preserve large mutation responses through validation --- cmd/openai/spend_thresholds_test.go | 31 ++++++++++++++++++----------- 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/cmd/openai/spend_thresholds_test.go b/cmd/openai/spend_thresholds_test.go index 0a260c06..92302772 100644 --- a/cmd/openai/spend_thresholds_test.go +++ b/cmd/openai/spend_thresholds_test.go @@ -550,18 +550,25 @@ func TestMainSpendThresholdLargeUnknownField(t *testing.T) { io.WriteString(w, body) })) defer server.Close() - args := spendThresholdArgs(false, "spend-limit", "retrieve") - got := runSpendThresholdCommand(t, server, args...) - if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "Spend threshold: USD 100.00 per month") || !strings.Contains(got.stdout, "Future: "+value+"\n") { - t.Fatalf("large readable response lost data: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) - } - got = runSpendThresholdCommand(t, server, append([]string{"--format", "raw"}, args...)...) - if got.code != 0 || got.stderr != "" || got.stdout != body+"\n" { - t.Fatalf("large raw response changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) - } - got = runSpendThresholdCommand(t, server, append([]string{"--transform", "future", "--raw-output"}, args...)...) - if got.code != 0 || got.stderr != "" || got.stdout != value+"\n" { - t.Fatalf("large extracted field changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + for _, action := range []string{"retrieve", "update"} { + t.Run(action, func(t *testing.T) { + args := spendThresholdArgs(false, "spend-limit", action) + if action == "update" { + args = append(args, "--currency", "USD", "--interval", "month", "--threshold-amount", "10000") + } + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "Spend threshold: USD 100.00 per month") || !strings.Contains(got.stdout, "Future: "+value+"\n") { + t.Fatalf("large readable response lost data: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } + got = runSpendThresholdCommand(t, server, append([]string{"--format", "raw"}, args...)...) + if got.code != 0 || got.stderr != "" || got.stdout != body+"\n" { + t.Fatalf("large raw response changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } + got = runSpendThresholdCommand(t, server, append([]string{"--transform", "future", "--raw-output"}, args...)...) + if got.code != 0 || got.stderr != "" || got.stdout != value+"\n" { + t.Fatalf("large extracted field changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } + }) } } From f166437372fb740374f6e30f63c8513fe339f375 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 05:08:05 -0700 Subject: [PATCH 42/44] style: align mutation validation test options --- internal/app/spend_error_guidance_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/app/spend_error_guidance_test.go b/internal/app/spend_error_guidance_test.go index 44ece7c5..3e3547bc 100644 --- a/internal/app/spend_error_guidance_test.go +++ b/internal/app/spend_error_guidance_test.go @@ -90,7 +90,7 @@ func TestSpendMutationResponseValidationPreservesCancellation(t *testing.T) { checking := &cancelListTableContext{Context: ctx, cancel: cancel, after: 2} var out bytes.Buffer err := ShowJSON(gjson.Parse(body), ShowJSONOpts{Context: checking, Format: "text", - Operation: "(resource) admin.organization.spend_alerts > (method) create", + Operation: "(resource) admin.organization.spend_alerts > (method) create", OutputKind: OutputResponse, Stdout: &out, Stderr: io.Discard}) require.ErrorIs(t, err, context.Canceled) require.Empty(t, out.String()) From a283f7377fdcade119730b7b528e3a7d1037f7f2 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 05:28:04 -0700 Subject: [PATCH 43/44] Remove unused Limits scene import --- scripts/demos/limits-workspace/scene.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py index b65dff8a..4e456f9c 100755 --- a/scripts/demos/limits-workspace/scene.py +++ b/scripts/demos/limits-workspace/scene.py @@ -11,7 +11,6 @@ import select from pathlib import Path import shutil -import subprocess import sys import tempfile import termios From b01004d20ddc926b2ea2ba7ce6cb3387813c8248 Mon Sep 17 00:00:00 2001 From: Sai Guvvala Date: Wed, 21 Oct 2026 05:55:11 -0700 Subject: [PATCH 44/44] Preserve confirmed saves when receipt output fails --- internal/app/limits_workspace.go | 4 ++ internal/app/limits_workspace_test.go | 85 +++++++++++++++------------ 2 files changed, 53 insertions(+), 36 deletions(-) diff --git a/internal/app/limits_workspace.go b/internal/app/limits_workspace.go index c1f0d1c7..5afea812 100644 --- a/internal/app/limits_workspace.go +++ b/internal/app/limits_workspace.go @@ -264,6 +264,10 @@ func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error opts.Operation, opts.OutputKind = m.lastSavedOperation, OutputResponse opts.Title = "Last confirmed result" receiptErr = ShowJSON(m.lastSaved, opts) + if receiptErr != nil { + // Keep the confirmed outcome independent so EPIPE suppression cannot discard it. + receiptErr = errors.Join(&limitsWorkflowFailure{message: "A limits change was confirmed, but its receipt could not be printed. Inspect its current state before another change.\n" + m.readRecoveryFor("overview")}, receiptErr) + } } var overviewErr error if incomplete { diff --git a/internal/app/limits_workspace_test.go b/internal/app/limits_workspace_test.go index f8f737aa..2f706ca9 100644 --- a/internal/app/limits_workspace_test.go +++ b/internal/app/limits_workspace_test.go @@ -1760,43 +1760,56 @@ func TestLimitsWorkspaceFinalizationPreservesIndependentFailures(t *testing.T) { func TestLimitsWorkspaceConfirmedSaveSurvivesTerminalFailure(t *testing.T) { for _, kind := range []string{"limit_set", "alert_create", "limit_delete", "rate_update"} { - t.Run(kind, func(t *testing.T) { - m, _ := newLimitsController(t, 80, 24) - m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month"}`) - if kind == "alert_create" { - m.workerSaved = gjson.Parse(limitsControllerAlert) - } else if kind == "limit_delete" { - m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit.deleted","deleted":true}`) - } else if kind == "rate_update" { - m.snapshot.ProjectID = "proj_synthetic" - m.workerSaved = gjson.Parse(limitsControllerRate) - } - m.workerSavedOperation = limitsChangeOperation(kind, m.snapshot.ProjectID != "") - m.change.Kind = "limit_set" - if kind == "limit_set" { - m.change.Kind = "alert_create" - } - m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} - var out bytes.Buffer - err := finishLimitsWorkspace(t.Context(), m, tea.ErrProgramKilled, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard}) - var safe *limitsWorkflowFailure - if !errors.As(err, &safe) || !errors.Is(err, syscall.EPIPE) || clioutput.IsBrokenPipe(err) { - t.Fatalf("terminal failure hid the confirmed outcome: %v", err) - } - if !strings.Contains(safe.Error(), "change was confirmed") || !strings.Contains(safe.Error(), "terminal output failed") || !strings.Contains(safe.Error(), "before another change") { - t.Fatalf("missing confirmed-save recovery: %v", err) - } - readOptions := "help admin limits show" - if m.snapshot.ProjectID != "" { - readOptions = "help projects limits show" - } - if !strings.Contains(safe.Error(), readOptions) || strings.Contains(safe.Error(), "spend-limit retrieve") || strings.Contains(safe.Error(), "spend-alerts list") { - t.Fatalf("unsent draft redirected confirmed-save recovery: %v", err) - } - if out.Len() != 0 || m.mutationPending || m.lastSaved.Raw != m.workerSaved.Raw { - t.Fatal("finalization repeated output or lost the confirmed result") + for _, phase := range []string{"terminal", "receipt"} { + for _, format := range []string{"text", "json"} { + t.Run(kind+"/"+phase+"/"+format, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month"}`) + if kind == "alert_create" { + m.workerSaved = gjson.Parse(limitsControllerAlert) + } else if kind == "limit_delete" { + m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit.deleted","deleted":true}`) + } else if kind == "rate_update" { + m.snapshot.ProjectID = "proj_synthetic" + m.workerSaved = gjson.Parse(limitsControllerRate) + } + m.workerSavedOperation = limitsChangeOperation(kind, m.snapshot.ProjectID != "") + m.change.Kind = "limit_set" + if kind == "limit_set" { + m.change.Kind = "alert_create" + } + var out bytes.Buffer + var sink io.Writer = &out + var runErr error = tea.ErrProgramKilled + expectedFailure := "terminal output failed" + m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} + if phase == "receipt" { + m.output = &limitsWorkspaceOutput{} + sink = limitsFailingWriter{err: syscall.EPIPE} + runErr = nil + expectedFailure = "receipt could not be printed" + } + err := finishLimitsWorkspace(t.Context(), m, runErr, ShowJSONOpts{Context: t.Context(), Format: format, Stdout: sink, Stderr: io.Discard}) + var safe *limitsWorkflowFailure + if !errors.As(err, &safe) || !errors.Is(err, syscall.EPIPE) || clioutput.IsBrokenPipe(err) { + t.Fatalf("terminal failure hid the confirmed outcome: %v", err) + } + if !strings.Contains(safe.Error(), "change was confirmed") || !strings.Contains(safe.Error(), expectedFailure) || !strings.Contains(safe.Error(), "before another change") { + t.Fatalf("missing confirmed-save recovery: %v", err) + } + readOptions := "help admin limits show" + if m.snapshot.ProjectID != "" { + readOptions = "help projects limits show" + } + if !strings.Contains(safe.Error(), readOptions) || strings.Contains(safe.Error(), "spend-limit retrieve") || strings.Contains(safe.Error(), "spend-alerts list") { + t.Fatalf("unsent draft redirected confirmed-save recovery: %v", err) + } + if out.Len() != 0 || m.mutationPending || m.lastSaved.Raw != m.workerSaved.Raw { + t.Fatal("finalization repeated output or lost the confirmed result") + } + }) } - }) + } } }