diff --git a/cmd/openai/error_coverage_test.go b/cmd/openai/error_coverage_test.go index b8acc92c..55d38d8d 100644 --- a/cmd/openai/error_coverage_test.go +++ b/cmd/openai/error_coverage_test.go @@ -188,7 +188,9 @@ func buildErrorCoverageManifest(root *cli.Command) (errorCoverageManifest, error } if entry.HasChildren && (command.Action != nil || len(command.Arguments) != 0 || command.ArgValidator != nil) { switch name { - case "help", "help setup", "tokenizer": + case "help", "help setup", "tokenizer", + "admin limits", "admin organization limits", + "projects limits", "admin projects limits", "admin organization projects limits": entry.Class += "; reviewed positional-error group" entry.ActionContract = name default: @@ -511,6 +513,7 @@ func declaredErrorCoverageCases(manifest errorCoverageManifest) []errorCoverageC } message += " to see commands." code := 3 + suffix := []string{private} switch path { case "help": message = "Unknown help topic. Run openai help to see commands." @@ -520,8 +523,16 @@ func declaredErrorCoverageCases(manifest errorCoverageManifest) []errorCoverageC // This action-bearing group owns positional input validation. message = "Tokenizer takes no positional input. Open the editor without arguments, or use tokenizer count --text TEXT." code = 1 + case "admin limits", "admin organization limits", + "projects limits", "admin projects limits", "admin organization projects limits": + message = "Limits does not accept positional arguments. Use --help to inspect its options." + code = 1 + if strings.HasSuffix(path, "projects limits") { + // Reach positional validation without masking it behind a missing project identifier. + suffix = []string{"--project-id=proj_synthetic", private} + } } - add("unknown-child", "", []string{private}, code, message) + add("unknown-child", "", suffix, code, message) } var required, requiredAfterInput []string aliasOwners := map[string]string{} diff --git a/cmd/openai/fine_tuning_model_limits_test.go b/cmd/openai/fine_tuning_model_limits_test.go new file mode 100644 index 00000000..aef52997 --- /dev/null +++ b/cmd/openai/fine_tuning_model_limits_test.go @@ -0,0 +1,381 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" + + "github.com/goccy/go-yaml" + "github.com/tidwall/gjson" +) + +const fineTuningModelLimitsResponse = `{"object":"synthetic.future_limits","data":[{"model":"synthetic-model","future":{"exact":9007199254740993,"decimal":0.100000000000000001,"unset":null,"text":"quoted \"value\""}}],"has_more":true,"next_page":"synthetic-unfollowed-cursor"}` + +func fineTuningModelLimitsEnvironment(t *testing.T, endpoint string) []string { + t.Helper() + home := t.TempDir() + return []string{ + "HOME=" + home, "USERPROFILE=" + home, "XDG_CONFIG_HOME=" + home, + "OPENAI_BASE_URL=" + endpoint, "OPENAI_API_KEY=sk-fake-fine-tuning", + "OPENAI_ADMIN_KEY=sk-fake-admin-unused", "OPENAI_ORG_ID=org_synthetic_fine_tuning", + "OPENAI_PROJECT_ID=proj_synthetic_fine_tuning", "OPENAI_CUSTOM_HEADERS=X-Synthetic-Context: retained", + "NO_COLOR=1", "FORCE_COLOR=0", "CI=1", + } +} + +func TestMainFineTuningModelLimitsRequestAndCredentials(t *testing.T) { + for _, tc := range []struct { + name, key, admin, headers, authorization string + flags []string + missing, overrideContext bool + }{ + {name: "ordinary only", key: "sk-fake-fine-tuning", authorization: "Bearer sk-fake-fine-tuning"}, + {name: "both keys", key: "sk-fake-fine-tuning", admin: "sk-fake-admin-unused", authorization: "Bearer sk-fake-fine-tuning"}, + {name: "root overrides", key: "sk-fake-environment-unused", admin: "sk-fake-admin-unused", authorization: "Bearer sk-fake-flag", + flags: []string{"--api-key=sk-fake-flag", "--admin-api-key=sk-fake-admin-flag-unused", "--organization=org_synthetic_override", "--project=proj_synthetic_override"}, overrideContext: true}, + {name: "explicit authorization", key: "sk-fake-fine-tuning-unused", admin: "sk-fake-admin-unused", authorization: "Bearer synthetic-header", + flags: []string{"--header=Authorization: Bearer synthetic-header"}}, + {name: "explicit authorization without API key", admin: "sk-fake-admin-unused", authorization: "Bearer synthetic-header", + flags: []string{"--header=Authorization: Bearer synthetic-header"}}, + {name: "environment authorization", admin: "sk-fake-admin-unused", authorization: "Bearer synthetic-last", + headers: "Authorization: Bearer synthetic-first\nAUTHORIZATION: Bearer synthetic-last"}, + {name: "API key before environment authorization", key: "sk-fake-fine-tuning", admin: "sk-fake-admin-unused", + authorization: "Bearer sk-fake-fine-tuning", headers: "Authorization: Bearer synthetic-header-unused"}, + {name: "empty API key with environment authorization", key: "sk-fake-environment-unused", admin: "sk-fake-admin-unused", + authorization: "Bearer synthetic-header", headers: "Authorization: Bearer synthetic-header", flags: []string{"--api-key="}}, + {name: "admin only rejected", admin: "sk-fake-admin-unused", missing: true}, + {name: "missing credentials", missing: true}, + {name: "empty API key override", key: "sk-fake-environment-unused", admin: "sk-fake-admin-unused", + flags: []string{"--api-key="}, missing: true}, + {name: "empty authorization override", key: "sk-fake-fine-tuning", admin: "sk-fake-admin-unused", + flags: []string{"--header=Authorization:"}, missing: true}, + {name: "empty last environment authorization", admin: "sk-fake-admin-unused", + headers: "Authorization: Bearer synthetic-first\nAuthorization:", missing: true}, + } { + t.Run(tc.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodGet || r.URL.RequestURI() != "/tenant/v1/fine_tuning/model_limits" { + t.Errorf("unexpected read request: %s %s", r.Method, r.URL.RequestURI()) + } + wantOrg, wantProject := "org_synthetic_fine_tuning", "proj_synthetic_fine_tuning" + if tc.overrideContext { + wantOrg, wantProject = "org_synthetic_override", "proj_synthetic_override" + } + if r.Header.Get("Authorization") != tc.authorization || + r.Header.Get("OpenAI-Organization") != wantOrg || r.Header.Get("OpenAI-Project") != wantProject || + r.Header.Get("X-Synthetic-Context") != "retained" { + t.Error("ordinary credential selection or request context changed") + } + body, err := io.ReadAll(r.Body) + if err != nil || len(body) != 0 { + t.Errorf("read request has a body or read error: bytes=%d error=%v", len(body), err) + } + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, fineTuningModelLimitsResponse) + })) + defer server.Close() + env := fineTuningModelLimitsEnvironment(t, server.URL+"/tenant/v1/") + env = spendRecoverySetEnvironment(env, "OPENAI_API_KEY", tc.key) + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", tc.admin) + env = spendRecoverySetEnvironment(env, "OPENAI_CUSTOM_HEADERS", "X-Synthetic-Context: retained\n"+tc.headers) + flags := slices.Clone(tc.flags) + if tc.overrideContext { + env = spendRecoverySetEnvironment(env, "OPENAI_BASE_URL", "http://127.0.0.1:1/unused/") + flags = append(flags, "--base-url="+server.URL+"/tenant/v1/") + } + args := append([]string{"openai", "fine-tuning", "model-limits", "--format=json"}, flags...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if tc.missing { + if got.code != 1 || got.stdout != "" || requests.Load() != 0 { + t.Fatalf("missing credentials reached HTTP or changed exit: code=%d requests=%d", got.code, requests.Load()) + } + payload := decodeMainStructuredError(t, "json", got.stderr) + if !strings.Contains(fmt.Sprint(payload["message"]), "OPENAI_API_KEY") || + !strings.Contains(fmt.Sprint(payload["message"]), "admin key cannot replace") { + t.Fatal("missing ordinary credentials lost actionable guidance") + } + if strings.Contains(got.stderr, "sk-fake-") || strings.Contains(got.stderr, "synthetic-first") { + t.Fatal("credential failure exposed a synthetic credential") + } + return + } + if got.code != 0 || got.stderr != "" || requests.Load() != 1 || !json.Valid([]byte(got.stdout)) { + t.Fatalf("read failed or followed an undocumented cursor: code=%d requests=%d stderr=%q", got.code, requests.Load(), got.stderr) + } + if gjson.Get(got.stdout, "data.0.future.exact").Raw != "9007199254740993" || + gjson.Get(got.stdout, "next_page").String() != "synthetic-unfollowed-cursor" { + t.Fatal("read lost unfamiliar response fields") + } + }) + } +} + +func TestMainFineTuningModelLimitsOutputModes(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, fineTuningModelLimitsResponse) + })) + defer server.Close() + env := fineTuningModelLimitsEnvironment(t, server.URL) + for _, format := range []string{"json", "jsonl", "pretty", "raw", "yaml"} { + t.Run(format, func(t *testing.T) { + before := requests.Load() + got := runMainDispatchWithEnv(t, "bash", env, "openai", "fine-tuning", "model-limits", "--format="+format) + if got.code != 0 || got.stderr != "" || requests.Load() != before+1 { + t.Fatalf("format failed: code=%d stderr=%q", got.code, got.stderr) + } + if format == "pretty" { + for _, value := range []string{"fine-tuning model-limits", "synthetic-model", "9007199254740993", "0.100000000000000001", "synthetic-unfollowed-cursor"} { + if !strings.Contains(got.stdout, value) { + t.Fatalf("pretty output omitted %q", value) + } + } + return + } + output := []byte(got.stdout) + if format == "yaml" { + if !strings.Contains(got.stdout, "decimal: 0.100000000000000001") { + t.Fatal("YAML output rounded an unfamiliar decimal") + } + var err error + output, err = yaml.YAMLToJSON(output) + if err != nil { + t.Fatal(err) + } + } + if !json.Valid(output) || gjson.GetBytes(output, "data.0.future.exact").Raw != "9007199254740993" || + gjson.GetBytes(output, "data.0.future.unset").Raw != "null" || + gjson.GetBytes(output, "data.0.future.text").String() != `quoted "value"` || + !gjson.GetBytes(output, "has_more").Bool() { + t.Fatal("output dropped or coerced unfamiliar response fields") + } + // YAML uses the shared converter. JSON modes must retain original decimal tokens. + if format != "yaml" && gjson.GetBytes(output, "data.0.future.decimal").Raw != "0.100000000000000001" { + t.Fatal("JSON output rounded an unfamiliar decimal") + } + if format == "raw" && strings.TrimSpace(got.stdout) != fineTuningModelLimitsResponse { + t.Fatal("raw output changed the response bytes") + } + }) + } + for _, tc := range []struct { + name string + args []string + want string + }{ + {"extraction", []string{"--transform=data.0.future.exact"}, "9007199254740993"}, + {"raw string", []string{"--transform=data.0.future.text", "--raw-output"}, `quoted "value"`}, + {"quiet", []string{"--quiet", "--format=json", "--transform=data.0.future.decimal"}, "0.100000000000000001"}, + } { + t.Run(tc.name, func(t *testing.T) { + args := append([]string{"openai", "fine-tuning", "model-limits"}, tc.args...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if got.code != 0 || got.stderr != "" || strings.TrimSpace(got.stdout) != tc.want { + t.Fatalf("extraction contract changed: %+v", got) + } + }) + } +} + +func TestMainFineTuningModelLimitsLargeResponse(t *testing.T) { + const size = 32<<20 + 1 + body := `{"future":{"payload":"` + strings.Repeat("x", size) + `","exact":9007199254740993},"data":[]}` + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + got := runMainDispatchWithEnv(t, "bash", fineTuningModelLimitsEnvironment(t, server.URL), + "openai", "fine-tuning", "model-limits", "--format=raw") + if got.code != 0 || got.stderr != "" || requests.Load() != 1 { + t.Fatalf("large read failed: code=%d requests=%d stderr=%q", got.code, requests.Load(), got.stderr) + } + if strings.TrimSpace(got.stdout) != body { + t.Fatal("large unfamiliar response was truncated or rewritten") + } +} + +func TestMainFineTuningModelLimitsResponseValidation(t *testing.T) { + for _, tc := range []struct { + name, body string + invalid bool + }{ + {"empty", "", true}, + {"malformed", "synthetic-private-response\x1b]52;c;private\a", true}, + {"array", `[{"unfamiliar":9007199254740993}]`, false}, + {"string", `"unfamiliar response"`, false}, + {"number", "1e10000", false}, + {"true", "true", false}, + {"false", "false", false}, + {"null", "null", false}, + } { + t.Run(tc.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, tc.body) + })) + defer server.Close() + got := runMainDispatchWithEnv(t, "bash", fineTuningModelLimitsEnvironment(t, server.URL), + "openai", "fine-tuning", "model-limits", "--format=json") + if requests.Load() != 1 { + t.Fatalf("read issued %d requests", requests.Load()) + } + if tc.invalid { + if got.code != 1 || got.stdout != "" { + t.Fatalf("invalid response reported success: code=%d", got.code) + } + payload := decodeMainStructuredError(t, "json", got.stderr) + if payload["message"] != "Fine-tuning model limits returned an invalid JSON response. Check --base-url, then retry this read." { + t.Fatal("invalid response lost safe recovery guidance") + } + if strings.Contains(got.stderr, "synthetic-private") || strings.ContainsAny(got.stderr, "\x1b\a") { + t.Fatal("invalid response exposed private contents") + } + return + } + var compact bytes.Buffer + if got.code != 0 || got.stderr != "" || json.Compact(&compact, []byte(got.stdout)) != nil || compact.String() != tc.body { + t.Fatalf("valid JSON shape changed: code=%d stderr=%q stdout=%q", got.code, got.stderr, got.stdout) + } + }) + } +} + +func TestMainFineTuningModelLimitsErrors(t *testing.T) { + const details = `{"message":"synthetic-private-response\u001b]52;c;private\u0007","type":"invalid_request_error","code":"synthetic_code","param":null,"future":{"exact":9007199254740993}}` + for _, tc := range []struct { + name, contentType, body string + status int + flags []string + }{ + {"ordinary authentication", "application/json", `{"error":` + details + `}`, 401, []string{"--format-error=text"}}, + {"unavailable route", "application/json", `{"error":` + details + `}`, 404, []string{"--format-error=text"}}, + {"JSON", "application/json", `{"error":` + details + `}`, 400, []string{"--format-error=json"}}, + {"YAML", "application/json", `{"error":` + details + `}`, 400, []string{"--format-error=yaml"}}, + {"extraction", "application/json", `{"error":` + details + `}`, 400, []string{"--format-error=json", "--transform-error=future.exact"}}, + {"empty server error", "text/html", "synthetic-private-response", 502, []string{"--format-error=json"}}, + } { + t.Run(tc.name, func(t *testing.T) { + got := runMainCommandAPIErrorResponse(t, tc.status, tc.contentType, tc.body, + []string{"fine-tuning", "model-limits"}, tc.flags...) + control := runMainAPIErrorResponse(t, tc.status, tc.contentType, tc.body, tc.flags...) + if got != control { + t.Fatalf("ordinary API error contract changed: got=%+v control=%+v", got, control) + } + if strings.HasSuffix(tc.name, "authentication") || tc.name == "unavailable route" { + if strings.Contains(got.stderr, "synthetic-private") || strings.Contains(got.stderr, "\x1b") || + strings.Contains(got.stderr, "setup admin") { + t.Fatal("human error exposed private data or requested admin authentication") + } + } + }) + } +} + +func TestMainFineTuningModelLimitsHelpAndValidation(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + fmt.Fprint(w, fineTuningModelLimitsResponse) + })) + defer server.Close() + env := fineTuningModelLimitsEnvironment(t, server.URL) + for _, argv := range [][]string{ + {"openai", "fine-tuning", "--help"}, + {"openai", "fine-tuning", "model-limits", "--help"}, + {"openai", "help", "fine-tuning", "model-limits"}, + } { + t.Run(strings.Join(argv[1:], " "), func(t *testing.T) { + got := runMainDispatchWithEnv(t, "bash", env, argv...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "model-limits") || requests.Load() != 0 { + t.Fatalf("help hid the command or made a request: %+v", got) + } + if len(argv) == 4 && argv[2] == "model-limits" || argv[1] == "help" { + if !strings.Contains(got.stdout, "openai fine-tuning model-limits") { + t.Fatal("command help omitted its invocation") + } + } + }) + } + for _, args := range [][]string{{"synthetic-private-operand"}, {"--unknown-synthetic=private"}, {"--max-items=1"}} { + t.Run(strings.Join(args, " "), func(t *testing.T) { + argv := append([]string{"openai", "--format-error=json", "fine-tuning", "model-limits"}, args...) + got := runMainDispatchWithEnv(t, "bash", env, argv...) + if got.code != 1 || got.stdout != "" || requests.Load() != 0 { + t.Fatalf("invalid arguments reached the endpoint: %+v", got) + } + decodeMainStructuredError(t, "json", got.stderr) + if strings.Contains(got.stderr, "synthetic-private-operand") || strings.Contains(got.stderr, "=private") { + t.Fatal("argument validation exposed a rejected value") + } + }) + } +} + +func TestFineTuningModelLimitsErrorCoverage(t *testing.T) { + env := errorCoverageEnvironment(t.TempDir(), "http://127.0.0.1:1") + manifest := readErrorCoverageManifest(t, env) + var found bool + for _, command := range manifest.Commands { + if strings.Join(command.Path, " ") == "fine-tuning model-limits" { + if command.HasChildren || command.Compatibility || command.Class != "public" { + t.Fatal("fine-tuning model limits is not an ordinary public leaf") + } + found = true + } + } + if !found { + t.Fatal("error inventory omitted fine-tuning model limits") + } + for _, probe := range declaredErrorCoverageCases(manifest) { + if probe.Name != "fine-tuning model-limits/unknown-option/" { + continue + } + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format-error=json"}, probe.Args...)...) + payload := decodeMainStructuredError(t, "json", got.stderr) + if got.code != probe.Code || got.stdout != "" || payload["message"] != probe.Message { + t.Fatal("the inventoried parser probe did not preserve its public diagnostic") + } + return + } + t.Fatal("error inventory omitted the new leaf's unknown-option probe") +} + +func TestMainFineTuningModelLimitsBaseURLSecurity(t *testing.T) { + var requests atomic.Int32 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusTeapot) + })) + defer proxy.Close() + env := adminCredentialsProxyEnv(proxy.URL, fineTuningModelLimitsEnvironment(t, + "http://synthetic-user:synthetic-password@remote.example/v1?token=synthetic-private")) + got := runMainDispatchWithEnv(t, "bash", env, "openai", "fine-tuning", "model-limits", "--format-error=json") + if got.code != 1 || got.stdout != "" || requests.Load() != 0 { + t.Fatal("insecure remote base URL reached HTTP") + } + payload := decodeMainStructuredError(t, "json", got.stderr) + if !strings.Contains(fmt.Sprint(payload["message"]), "HTTPS") { + t.Fatal("base URL rejection omitted recovery") + } + if bytes.ContainsAny([]byte(got.stderr), "\x1b\a") || strings.Contains(got.stderr, "synthetic-password") || + strings.Contains(got.stderr, "remote.example") { + t.Fatal("base URL rejection exposed private URL contents") + } +} diff --git a/cmd/openai/fine_tuning_model_limits_unix_test.go b/cmd/openai/fine_tuning_model_limits_unix_test.go new file mode 100644 index 00000000..912af995 --- /dev/null +++ b/cmd/openai/fine_tuning_model_limits_unix_test.go @@ -0,0 +1,111 @@ +//go:build !windows + +package main + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "os" + "os/exec" + "path/filepath" + "slices" + "strings" + "sync/atomic" + "testing" +) + +func TestMainFineTuningModelLimitsNativeHelpExamples(t *testing.T) { + work, binary := buildNativeShellCLI(t) + executable := filepath.Join(work, binary) + invocation := "'" + strings.ReplaceAll(executable, "'", "'\\''") + "'" + var requests, adminRequests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if strings.HasPrefix(r.URL.Path, "/organization/") || strings.HasPrefix(r.URL.Path, "/admin/") { + adminRequests.Add(1) + } + if r.Method != http.MethodGet || r.URL.RequestURI() != "/fine_tuning/model_limits" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL.RequestURI()) + } + if r.Header.Get("Authorization") != "Bearer fake-native-shell-key" { + t.Error("request omitted the ordinary synthetic credential") + } + body, err := io.ReadAll(r.Body) + if err != nil || len(body) != 0 { + t.Errorf("read request has a body or read error: bytes=%d error=%v", len(body), err) + } + w.Header().Set("Content-Type", "application/json") + if _, err := fmt.Fprint(w, fineTuningModelLimitsResponse); err != nil { + t.Errorf("write synthetic limits response: %v", err) + } + })) + defer server.Close() + required := strings.Split(os.Getenv("OPENAI_CLI_REQUIRE_NATIVE_SHELLS"), ",") + for _, shell := range []nativeShell{ + {"bash", "bash", "", "export OPENAI_API_KEY='fake-native-shell-key'; ", []string{"--noprofile", "--norc", "-c"}}, + {"zsh", "zsh", "", "export OPENAI_API_KEY='fake-native-shell-key'; ", []string{"-f", "-c"}}, + } { + t.Run(shell.name, func(t *testing.T) { + path, err := exec.LookPath(shell.executable) + if err != nil { + if slices.Contains(required, shell.name) { + t.Fatalf("required shell is unavailable: %v", err) + } + t.Skipf("%s is not installed", shell.name) + } + shell.executable = path + for _, lookup := range []string{"direct_path", "installed"} { + t.Run(lookup, func(t *testing.T) { + directory, home, searchPath := t.TempDir(), t.TempDir(), t.TempDir() + printedInvocation := invocation + if lookup == "direct_path" { + if err := os.WriteFile(filepath.Join(searchPath, "openai"), []byte("#!/bin/sh\nprintf 'wrong executable\\n'\nexit 99\n"), 0o700); err != nil { + t.Fatal(err) + } + } else { + searchPath, printedInvocation = work, "openai" + } + t.Setenv("PATH", searchPath) + for _, helpArgs := range []struct{ name, args string }{ + {"concise", " fine-tuning model-limits --help"}, + {"full", " help fine-tuning model-limits"}, + } { + t.Run(helpArgs.name, func(t *testing.T) { + requests.Store(0) + adminRequests.Store(0) + help := runNativeShell(t, shell, directory, home, server.URL, invocation+helpArgs.args) + if help.code != 0 || help.stderr != "" || requests.Load() != 0 { + t.Fatalf("help failed or made a request: %+v", help) + } + _, examples, found := strings.Cut(help.stdout, "EXAMPLES:\n") + if !found { + t.Fatal("help omitted executable examples") + } + examples, _, _ = strings.Cut(examples, "\nOPTIONS:") + var commands []string + for line := range strings.SplitSeq(examples, "\n") { + if strings.HasPrefix(strings.TrimSpace(line), printedInvocation+" fine-tuning model-limits") { + commands = append(commands, strings.TrimSpace(line)) + } + } + if len(commands) != 1 || commands[0] != printedInvocation+" fine-tuning model-limits" { + t.Fatalf("help changed the example or executable identity %q:\n%s", printedInvocation, help.stdout) + } + got := runNativeShell(t, shell, directory, home, server.URL, shell.setKey+commands[0]) + if got.code != 0 || got.stderr != "" || requests.Load() != 1 || adminRequests.Load() != 0 { + t.Fatalf("copied example failed: result=%+v requests=%d admin=%d", got, requests.Load(), adminRequests.Load()) + } + for _, value := range []string{"synthetic-model", "9007199254740993", "0.100000000000000001", "synthetic-unfollowed-cursor"} { + if strings.Count(got.stdout, value) != 1 { + t.Fatalf("example output lost or duplicated %q: %q", value, got.stdout) + } + } + }) + } + }) + } + }) + } +} diff --git a/cmd/openai/limits_workflow_test.go b/cmd/openai/limits_workflow_test.go new file mode 100644 index 00000000..49d743d2 --- /dev/null +++ b/cmd/openai/limits_workflow_test.go @@ -0,0 +1,612 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "sync" + "sync/atomic" + "testing" + + "github.com/tidwall/gjson" +) + +const limitsWorkflowProject = `{"id":"proj_synthetic","object":"organization.project","name":"Synthetic project","status":"active","future":0.100000000000000001}` +const limitsWorkflowRate = `{"id":"rl_synthetic","object":"project.rate_limit","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993,"max_audio_megabytes_per_1_minute":1.25,"future":{"exact":123456789012345678901234567890}}` + +func limitsWorkflowArgs(project bool, show bool) []string { + args := []string{"admin", "limits"} + if project { + args = []string{"projects", "limits", "--project-id", "proj_synthetic"} + } + if show { + args = append(args, "show") + } + return args +} + +func limitsWorkflowRecords(project bool) (limit, alert string) { + limit = spendThresholdBody(project, "spend-limit", `{"threshold_amount":9007199254740993,"currency":"USD","interval":"month","enforcement":{"status":"enforcing"},"future":{"decimal":0.100000000000000001}}`) + alert = spendThresholdBody(project, "spend-alerts", `{"id":"alert_synthetic","threshold_amount":10000,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["synthetic@example.invalid"],"subject_prefix":null},"future":{"exact":123456789012345678901234567890}}`) + return limit, alert +} + +func limitsWorkflowPage(items string, more bool, last string) string { + return fmt.Sprintf(`{"object":"list","data":[%s],"has_more":%t,"last_id":%q}`, items, more, last) +} + +func limitsWorkflowEnvelope(project bool, limit, alerts string, configured string, alertsComplete bool, rates, errors string) string { + fields := `"scope":"organization",` + if project { + fields = `"scope":"project","project_id":"proj_synthetic","project":` + limitsWorkflowProject + `,` + } + fields += `"spend_limit":` + limit + `,"spend_limit_configured":` + configured + `,"spend_alerts":[` + alerts + `],"spend_alerts_complete":` + fmt.Sprint(alertsComplete) + if project { + fields += `,"project_rate_limits":[` + rates + `],"project_rate_limits_complete":true` + } + return `{` + fields + `,"source_errors":[` + errors + `]}` +} + +type limitsWorkflowRequests struct { + mu sync.Mutex + paths []string +} + +func (r *limitsWorkflowRequests) count(path string) int { + r.mu.Lock() + defer r.mu.Unlock() + count := 0 + for _, request := range r.paths { + if strings.SplitN(request, "?", 2)[0] == path { + count++ + } + } + return count +} + +func newLimitsWorkflowServer(t *testing.T, respond func(*http.Request) (int, string)) (*httptest.Server, *limitsWorkflowRequests) { + t.Helper() + requests := &limitsWorkflowRequests{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.mu.Lock() + requests.paths = append(requests.paths, r.URL.RequestURI()) + requests.mu.Unlock() + if r.Method != http.MethodGet { + t.Errorf("read-only overview sent %s", r.Method) + } + if r.Header.Get("Authorization") != "Bearer sk-fake-spend-admin" { + t.Error("overview did not select the synthetic admin credential") + } + body, err := io.ReadAll(r.Body) + if err != nil || len(body) != 0 { + t.Errorf("read-only request has a body or read error: bytes=%d error=%v", len(body), err) + } + status, response := respond(r) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("x-should-retry", "false") + w.WriteHeader(status) + io.WriteString(w, response) + })) + t.Cleanup(server.Close) + return server, requests +} + +func limitsWorkflowSuccess(t *testing.T, project bool, r *http.Request) (int, string) { + t.Helper() + limit, alert := limitsWorkflowRecords(project) + switch r.URL.Path { + case "/organization/costs": + return http.StatusOK, `{"object":"page","data":[],"has_more":false,"next_page":null}` + case "/organization/projects/proj_synthetic": + return http.StatusOK, limitsWorkflowProject + case spendThresholdPath(project, "spend-limit"): + return http.StatusOK, limit + case spendThresholdPath(project, "spend-alerts"): + return http.StatusOK, limitsWorkflowPage(alert, false, "alert_synthetic") + case spendThresholdPath(true, "rate-limits"): + return http.StatusOK, limitsWorkflowPage(limitsWorkflowRate, false, "rl_synthetic") + default: + t.Errorf("unexpected overview route: %s", r.URL.Path) + return http.StatusNotFound, `{"error":{"message":"unexpected synthetic route"}}` + } +} + +func TestMainLimitsWorkflowBaseURLSecurity(t *testing.T) { + const remote = "http://fake-user:fake-password@remote.example/v1?token=fake-private" + for _, project := range []bool{false, true} { + t.Run(fmt.Sprintf("project=%t", project), func(t *testing.T) { + var proxyRequests atomic.Int32 + proxy := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + proxyRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusForbidden) + io.WriteString(w, `{"error":{"message":"synthetic proxy rejects all destinations"}}`) + })) + t.Cleanup(proxy.Close) + home := t.TempDir() + env := adminCredentialsProxyEnv(proxy.URL, []string{"HOME=" + home, "XDG_CONFIG_HOME=" + home, + "OPENAI_API_KEY=sk-fake-spend-project", "OPENAI_ADMIN_KEY=sk-fake-spend-admin"}) + for _, source := range []string{"flag", "environment"} { + for _, format := range []string{"text", "json"} { + t.Run(source+"/"+format, func(t *testing.T) { + args := []string{"openai", "--format-error", format} + childEnv := append([]string{}, env...) + wantSource := "OPENAI_BASE_URL" + if source == "flag" { + args = append(args, "--base-url", remote) + wantSource = "--base-url" + } else { + childEnv = append(childEnv, "OPENAI_BASE_URL="+remote) + } + got := runMainDispatchWithEnv(t, "bash", childEnv, append(args, limitsWorkflowArgs(project, true)...)...) + if got.code != 1 || got.stdout != "" || proxyRequests.Load() != 0 { + t.Fatalf("remote HTTP reached the overview: code=%d requests=%d", got.code, proxyRequests.Load()) + } + message := got.stderr + if format == "json" { + var value map[string]string + if err := json.Unmarshal([]byte(got.stderr), &value); err != nil || len(value) != 1 { + t.Fatal("remote HTTP rejection did not preserve a structured diagnostic") + } + message = value["message"] + } + if !strings.Contains(message, wantSource) || !strings.Contains(message, "HTTPS") { + t.Fatal("remote HTTP rejection omitted its setting or recovery") + } + for _, private := range []string{"fake-", "remote.example", "\x1b"} { + if strings.Contains(got.stderr, private) { + t.Fatal("remote HTTP diagnostic exposed URL or credential contents") + } + } + }) + } + } + t.Run("loopback flag overrides remote environment", func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, project, r) + }) + childEnv := append(append([]string{}, env...), "OPENAI_BASE_URL="+remote) + args := append([]string{"openai", "--format", "json", "--base-url", server.URL}, limitsWorkflowArgs(project, true)...) + got := runMainDispatchWithEnv(t, "bash", childEnv, args...) + if got.code != 0 || got.stderr != "" || !json.Valid([]byte(got.stdout)) || proxyRequests.Load() != 0 { + t.Fatalf("loopback override failed: code=%d proxy_requests=%d", got.code, proxyRequests.Load()) + } + paths := []string{spendThresholdPath(project, "spend-limit"), spendThresholdPath(project, "spend-alerts"), "/organization/costs"} + if project { + paths = append(paths, "/organization/projects/proj_synthetic", spendThresholdPath(true, "rate-limits")) + } + for _, path := range paths { + if requests.count(path) != 1 { + t.Errorf("loopback overview requested %s %d times", path, requests.count(path)) + } + } + }) + }) + } +} + +func TestMainLimitsWorkflowReadableAndFormats(t *testing.T) { + for _, project := range []bool{false, true} { + for _, mode := range []string{"bare", "show", "text", "JSON", "jsonl", "raw", "yaml", "pretty", "extraction", "raw-output"} { + t.Run(fmt.Sprintf("project=%t/%s", project, mode), func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, project, r) + }) + args := limitsWorkflowArgs(project, mode != "bare") + switch mode { + case "bare", "show": + case "extraction": + args = append([]string{"--transform", "spend_limit.threshold_amount"}, args...) + case "raw-output": + args = append([]string{"--raw-output"}, args...) + default: + args = append([]string{"--format", mode}, args...) + } + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("overview failed: %+v", got) + } + for _, resource := range []string{"spend-limit", "spend-alerts"} { + if n := requests.count(spendThresholdPath(project, resource)); n != 1 { + t.Errorf("%s requests=%d, want one", resource, n) + } + } + if project && (requests.count("/organization/projects/proj_synthetic") != 1 || requests.count(spendThresholdPath(true, "rate-limits")) != 1) { + t.Error("project overview must validate the project and fetch its rates") + } + limit, alert := limitsWorkflowRecords(project) + want := limitsWorkflowEnvelope(project, limit, alert, "true", true, limitsWorkflowRate, "") + switch mode { + case "bare", "show", "text": + for _, text := range []string{"USD 90071992547409.93 per month", "USD 100.00 per month", "Alerts notify; they are not spending caps.", "0.100000000000000001", "123456789012345678901234567890"} { + if !strings.Contains(got.stdout, text) { + t.Errorf("readable overview lost %q: %q", text, got.stdout) + } + } + if project { + for _, text := range []string{"Max requests per minute: 0", "Max tokens per minute: 9007199254740993", "Max audio megabytes per minute: 1.25"} { + if !strings.Contains(got.stdout, text) { + t.Errorf("rate units lost %q: %q", text, got.stdout) + } + } + } + case "yaml", "pretty": + for _, text := range []string{"threshold_amount: 9007199254740993", "decimal: 0.100000000000000001", "exact: 123456789012345678901234567890"} { + if !strings.Contains(got.stdout, text) { + t.Errorf("%s lost %q: %q", mode, text, got.stdout) + } + } + case "extraction": + if got.stdout != "9007199254740993\n" { + t.Errorf("overview extraction changed: %q", got.stdout) + } + default: + costs := gjson.Get(got.stdout, "costs") + if costs.Get("total").Str != "0.00" || costs.Get("currency").Str != "USD" || !costs.Get("complete").Bool() { + t.Fatalf("Costs summary missing: %s", costs.Raw) + } + want = strings.TrimSuffix(want, "}") + `,"costs":` + costs.Raw + "}" + assertSpendThresholdJSON(t, got.stdout, want) + } + if strings.Contains(got.stdout, "\x1b") { + t.Error("nonterminal overview emitted terminal control sequences") + } + }) + } + } +} + +func TestMainLimitsWorkflowMissingLimitAndPartialFailures(t *testing.T) { + for _, tc := range []struct { + name, failedResource string + project bool + status int + }{ + {"organization no limit", "spend-limit", false, http.StatusNotFound}, + {"validated project no limit", "spend-limit", true, http.StatusNotFound}, + {"limit forbidden", "spend-limit", true, http.StatusForbidden}, + {"missing project", "project", true, http.StatusNotFound}, + {"alerts forbidden", "spend-alerts", false, http.StatusForbidden}, + {"rates unavailable", "rate-limits", true, http.StatusBadGateway}, + } { + t.Run(tc.name, func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + failedPath := spendThresholdPath(tc.project, tc.failedResource) + if tc.failedResource == "project" { + failedPath = "/organization/projects/proj_synthetic" + } + if r.URL.Path == failedPath { + return tc.status, `{"error":{"type":"synthetic_error","message":"synthetic private limits response"}}` + } + return limitsWorkflowSuccess(t, tc.project, r) + }) + args := append([]string{"--format", "json"}, limitsWorkflowArgs(tc.project, true)...) + got := runSpendThresholdCommand(t, server, args...) + if !json.Valid([]byte(got.stdout)) { + t.Fatalf("partial overview is not complete JSON: %+v", got) + } + value := gjson.Parse(got.stdout) + { + if tc.failedResource == "spend-limit" && value.Get("spend_limit_configured").Type != gjson.Null { + t.Fatalf("failed lookup incorrectly established configuration: %s", got.stdout) + } + if got.code != 1 || strings.TrimSpace(got.stderr) == "" || value.Get("source_errors.#").Int() == 0 { + t.Fatalf("partial failure disappeared: %+v", got) + } + if !json.Valid([]byte(got.stderr)) { + t.Errorf("structured error stream is invalid: %q", got.stderr) + } + resource := strings.ReplaceAll(tc.failedResource, "-", "_") + if resource == "rate_limits" { + resource = "project_rate_limits" + } + matched := false + for _, failure := range value.Get("source_errors").Array() { + if failure.Get("resource").Str == resource { + matched = failure.Get("status").Str == "api_error" && failure.Get("status_code").Int() == int64(tc.status) + } + failure.ForEach(func(key, _ gjson.Result) bool { + if key.Str != "resource" && key.Str != "status" && key.Str != "status_code" { + t.Errorf("source failure includes undocumented field %q", key.Str) + } + return true + }) + } + if !matched { + t.Errorf("missing safe source/status record: %s", got.stdout) + } + if tc.failedResource == "project" { + if value.Get("spend_limit_configured").Raw != "null" || requests.count(spendThresholdPath(true, "spend-limit")) != 0 { + t.Errorf("unvalidated project inferred an absent limit: %s", got.stdout) + } + } else if tc.failedResource == "spend-limit" { + if value.Get("spend_limit_configured").Raw != "null" || value.Get("spend_limit").Raw != "null" || value.Get("spend_alerts.#").Int() != 1 { + t.Errorf("failed limit became unconfigured or lost independent alerts: %s", got.stdout) + } + } else if value.Get("spend_limit.threshold_amount").Raw != "9007199254740993" { + t.Errorf("partial failure lost successful spend data: %s", got.stdout) + } + } + for _, secret := range []string{"synthetic private limits response", "sk-fake-spend-admin", "sk-fake-spend-test"} { + if strings.Contains(got.stdout+got.stderr, secret) { + t.Errorf("overview diagnostics included private fixture data %q", secret) + } + } + }) + } +} + +func TestMainLimitsWorkflowAmbiguousAndOptionalRecords(t *testing.T) { + const limit = `{"object":"project.spend_limit","threshold_amount":10000,"threshold_amount":90000,"currency":"USD","interval":"month"}` + const alert = `{"object":"project.spend_alert","id":"alert_synthetic","threshold_amount":10000,"currency":"USD","currency":"EUR","interval":"month"}` + const rate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":7,"future":"first","future":"second"}` + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + switch r.URL.Path { + case spendThresholdPath(true, "spend-limit"): + return http.StatusOK, limit + case spendThresholdPath(true, "spend-alerts"): + return http.StatusOK, limitsWorkflowPage(alert, false, "alert_synthetic") + case spendThresholdPath(true, "rate-limits"): + return http.StatusOK, limitsWorkflowPage(rate, false, "rl_synthetic") + default: + return limitsWorkflowSuccess(t, true, r) + } + }) + args := limitsWorkflowArgs(true, true) + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("ambiguous records lost read-only access: %+v", got) + } + // Ambiguous rates keep source keys; numeric segments require quoted labels. + for _, text := range []string{ + "Threshold amount: 10000", "Threshold amount: 90000", "Currency: USD", "Currency: EUR", + `"max_requests_per_1_minute": 0`, `"max_tokens_per_1_minute": 7`, "Future: first", "Future: second", + } { + if !strings.Contains(got.stdout, text) { + t.Errorf("readable overview lost %q: %q", text, got.stdout) + } + } + for _, absent := range []string{"Spend threshold:", "Alerts notify;", "not reported in this response", "Max requests per minute:", "Max tokens per minute:", "Max images", "Max audio", "Max requests per day", "Max batch"} { + if strings.Contains(got.stdout, absent) { + t.Errorf("ambiguous overview invented %q: %q", absent, got.stdout) + } + } + got = runSpendThresholdCommand(t, server, append([]string{"--format", "raw"}, args...)...) + if got.code != 0 || got.stderr != "" || !json.Valid([]byte(got.stdout)) { + t.Fatalf("raw ambiguous overview failed: %+v", got) + } + for _, record := range []string{limit, alert, rate} { + if !strings.Contains(got.stdout, record) { + t.Errorf("raw overview discarded duplicate fields: %q", got.stdout) + } + } +} + +func TestMainLimitsWorkflowNoninteractiveGuards(t *testing.T) { + for _, tc := range []struct { + name string + flags, env []string + }{ + {"quiet", []string{"--quiet"}, nil}, + {"CI", nil, []string{"CI=1"}}, + {"dumb", nil, []string{"TERM=dumb"}}, + {"explicit auto", []string{"--format", "auto"}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, false, r) + }) + env := append([]string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_API_KEY=sk-fake-spend-test", "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "FORCE_COLOR=0"}, tc.env...) + args := append([]string{"openai"}, tc.flags...) + args = append(args, limitsWorkflowArgs(false, false)...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "USD 90071992547409.93 per month") || strings.Contains(got.stdout, "\x1b") { + t.Fatalf("noninteractive invocation entered a workspace or lost data: %+v", got) + } + }) + } +} + +func TestMainLimitsWorkflowEnvironmentHeadersAndDebug(t *testing.T) { + for _, debug := range []bool{false, true} { + t.Run(fmt.Sprintf("debug=%t", debug), func(t *testing.T) { + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.Header.Get("X-Limits-Env") != "synthetic-private-header" { + t.Error("overview lost configured environment headers") + } + return limitsWorkflowSuccess(t, false, r) + }) + env := []string{"OPENAI_BASE_URL=" + server.URL, "OPENAI_API_KEY=sk-fake-spend-test", "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "OPENAI_CUSTOM_HEADERS=X-Limits-Env: synthetic-private-header", "FORCE_COLOR=0"} + args := []string{"openai", "--format", "json"} + if debug { + args = append(args, "--debug") + } + args = append(args, limitsWorkflowArgs(false, true)...) + got := runMainDispatchWithEnv(t, "bash", env, args...) + if got.code != 0 || !json.Valid([]byte(got.stdout)) || (got.stderr != "") != debug { + t.Fatalf("debug or environment-header handling changed: %+v", got) + } + for _, secret := range []string{"synthetic-private-header", "sk-fake-spend-admin", "sk-fake-spend-test"} { + if strings.Contains(got.stdout+got.stderr, secret) { + t.Errorf("debug output exposed fixture credential/header %q", secret) + } + } + }) + } +} + +func TestMainLimitsWorkflowPaginationAndBounds(t *testing.T) { + for _, resource := range []string{"spend-alerts", "rate-limits"} { + for _, tc := range []struct { + name, max string + count, calls int + complete, cycle bool + }{ + {name: "unlimited", max: "-1", count: 3, calls: 2, complete: true}, + {name: "zero", max: "0", calls: 1}, + {name: "one", max: "1", count: 1, calls: 1}, + {name: "page boundary", max: "2", count: 2, calls: 1}, + {name: "cursor cycle", max: "-1", count: -1, calls: 2, cycle: true}, + } { + t.Run(resource+"/"+tc.name, func(t *testing.T) { + project := resource == "rate-limits" + path := spendThresholdPath(project, resource) + _, alert := limitsWorkflowRecords(project) + item := alert + field := "spend_alerts" + if project { + item, field = limitsWorkflowRate, "project_rate_limits" + } + originalID := gjson.Get(item, "id").Str + first := strings.Replace(item, originalID, "item_a", 1) + second := strings.Replace(item, originalID, "item_b", 1) + third := strings.Replace(item, originalID, "item_c", 1) + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.URL.Path != path { + return limitsWorkflowSuccess(t, project, r) + } + if tc.max == "0" && r.URL.Query().Get("limit") != "1" { + t.Errorf("zero bound requested limit=%q, want 1", r.URL.Query().Get("limit")) + } + switch r.URL.Query().Get("after") { + case "": + return http.StatusOK, limitsWorkflowPage(first+","+second, true, "item_b") + case "item_b": + if tc.cycle { + return http.StatusOK, limitsWorkflowPage(first+","+second, true, "item_b") + } + return http.StatusOK, limitsWorkflowPage(third, false, "item_c") + default: + t.Errorf("unexpected pagination cursor: %s", r.URL.Query().Get("after")) + return http.StatusBadRequest, `{"error":{"message":"unexpected cursor"}}` + } + }) + args := append([]string{"--format", "json"}, limitsWorkflowArgs(project, false)...) + args = append(args, "--max-items", tc.max, "show") + got := runSpendThresholdCommand(t, server, args...) + if !json.Valid([]byte(got.stdout)) || requests.count(path) != tc.calls { + t.Fatalf("pagination failed: %+v; calls=%d", got, requests.count(path)) + } + value := gjson.Parse(got.stdout) + if tc.cycle { + if got.code != 1 || value.Get("source_errors.#").Int() == 0 || value.Get(field+".#").Int() == 0 { + t.Errorf("cursor cycle lost failure or partial data: %+v", got) + } + } else if got.code != 0 || got.stderr != "" || value.Get(field+".#").Int() != int64(tc.count) { + t.Errorf("bounded list changed: %+v", got) + } + if value.Get(field+"_complete").Bool() != tc.complete { + t.Errorf("list completeness changed: %s", got.stdout) + } + }) + } + } +} + +func TestMainLimitsWorkflowRootFlagsAndHelp(t *testing.T) { + for _, trailing := range []bool{false, true} { + t.Run(fmt.Sprintf("trailing=%t", trailing), func(t *testing.T) { + server, _ := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.Header.Get("OpenAI-Organization") != "org_header" || r.Header.Get("OpenAI-Project") != "proj_header" || r.Header.Get("X-Limits-Test") != "synthetic" { + t.Error("root context or custom headers changed") + } + return limitsWorkflowSuccess(t, true, r) + }) + flags := []string{"--format", "json", "--organization", "org_header", "--project", "proj_header", "-H", "X-Limits-Test: synthetic"} + args := limitsWorkflowArgs(true, true) + if trailing { + args = append(args, flags...) + } else { + args = append(flags, args...) + } + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" || gjson.Get(got.stdout, "project_id").Str != "proj_synthetic" { + t.Fatalf("root flags changed project route or output: %+v", got) + } + }) + } + for _, args := range [][]string{ + {"admin", "limits", "--help"}, + {"admin", "limits", "show", "--help"}, + {"projects", "limits", "--help"}, + {"projects", "limits", "show", "--help"}, + } { + t.Run(strings.Join(args, "/"), func(t *testing.T) { + got := runMainDispatch(t, "bash", append([]string{"openai"}, args...)...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "limits") || !strings.Contains(got.stdout, "--max-items") { + t.Fatalf("limits help unavailable: %+v", got) + } + }) + } +} + +func TestMainLimitsWorkflowRequiresAdminCredentials(t *testing.T) { + for _, project := range []bool{false, true} { + t.Run(fmt.Sprintf("project=%t", project), func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusForbidden) + })) + t.Cleanup(server.Close) + args := append([]string{"openai", "--base-url", "https://api.openai.com/v1"}, limitsWorkflowArgs(project, true)...) + got := runMainDispatchWithEnv(t, "bash", adminCredentialsProxyEnv(server.URL, []string{"OPENAI_API_KEY=synthetic-project-key"}), args...) + if got.code != 1 || got.stdout != "" || strings.TrimSpace(got.stderr) != missingAdminCredentialsMessage || requests.Load() != 0 { + t.Fatalf("workspace bypassed admin credential validation: result=%+v requests=%d", got, requests.Load()) + } + }) + } +} + +func TestMainLimitsWorkflowCostsPrecisionAndPartialFailure(t *testing.T) { + for _, failNext := range []bool{false, true} { + t.Run(fmt.Sprintf("partial=%t", failNext), func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if r.URL.Path != "/organization/costs" { + return limitsWorkflowSuccess(t, true, r) + } + query := r.URL.Query() + if query.Get("project_ids[]") != "proj_synthetic" { + t.Errorf("Costs lost the selected project: %s", r.URL.RawQuery) + } + if query.Get("page") == "next_costs" { + if failNext { + return http.StatusServiceUnavailable, `{"error":{"message":"private synthetic costs failure"}}` + } + return http.StatusOK, `{"object":"page","data":[],"has_more":false,"next_page":null}` + } + start, err := strconv.ParseInt(query.Get("start_time"), 10, 64) + end, endErr := strconv.ParseInt(query.Get("end_time"), 10, 64) + if err != nil || endErr != nil || start%86400 != 0 || end%86400 != 0 || end <= start { + t.Fatalf("Costs period is not explicit UTC days: %s", r.URL.RawQuery) + } + return http.StatusOK, fmt.Sprintf(`{"object":"page","data":[{"object":"bucket","start_time":%d,"end_time":%d,"results":[{"object":"organization.costs.result","amount":{"value":0.100000000000000001,"currency":"usd"},"project_id":"proj_synthetic","future":9007199254740993},{"object":"organization.costs.result","amount":{"value":0.2,"currency":"usd"},"project_id":"proj_synthetic"}]}],"has_more":true,"next_page":"next_costs"}`, start, start+86400) + }) + got := runSpendThresholdCommand(t, server, "projects", "limits", "show", "--project-id", "proj_synthetic", "--format", "json") + value := gjson.Parse(got.stdout) + if !json.Valid([]byte(got.stdout)) || requests.count("/organization/costs") != 2 { + t.Fatalf("Costs pagination failed: %+v", got) + } + if value.Get("costs.buckets.0.results.0.amount.value").Raw != "0.100000000000000001" || value.Get("costs.buckets.0.results.0.future").Raw != "9007199254740993" { + t.Fatalf("Costs raw data lost precision: %s", got.stdout) + } + if failNext { + if got.code != 1 || value.Get("costs.complete").Bool() || value.Get("costs.total").Str != "0.300000000000000001" || value.Get("source_errors.#").Int() != 1 { + t.Fatalf("Partial Costs became a complete total: %+v", got) + } + if strings.Contains(got.stderr, "private synthetic") || !json.Valid([]byte(got.stderr)) { + t.Fatalf("Costs recovery exposed source data or broke JSON: %s", got.stderr) + } + } else if got.code != 0 || got.stderr != "" || !value.Get("costs.complete").Bool() || value.Get("costs.total").Str != "0.300000000000000001" { + t.Fatalf("Costs total is not exact: %+v", got) + } + if !value.Get("spend_limit").IsObject() || value.Get("spend_alerts.#").Int() != 1 { + t.Fatalf("Costs handling lost independent sources: %s", got.stdout) + } + }) + } +} diff --git a/cmd/openai/limits_workflow_unix_test.go b/cmd/openai/limits_workflow_unix_test.go new file mode 100644 index 00000000..4e03e358 --- /dev/null +++ b/cmd/openai/limits_workflow_unix_test.go @@ -0,0 +1,274 @@ +//go:build !windows + +package main + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "os/exec" + "slices" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "github.com/openai/openai-cli/generated/commands" + "github.com/openai/openai-cli/internal/app" + "github.com/openai/openai-cli/internal/terminal" +) + +type limitsBorrowedOutputProbe struct { + fail bool + writes atomic.Int32 + ready atomic.Bool + failed atomic.Bool +} + +func (w *limitsBorrowedOutputProbe) Write(data []byte) (int, error) { + w.writes.Add(1) + if w.fail && w.ready.Load() && !w.failed.Swap(true) { + return 0, errors.New("synthetic-private-borrowed-output") + } + n, err := os.Stdout.Write(data) + if err == nil && n == len(data) && bytes.Contains(data, []byte("Website controls and source errors: ?")) { + w.ready.Store(true) + } + return n, err +} + +// Exercise production startup with the supported embedding writer installed. +func TestMainLimitsBorrowedWriterProcess(t *testing.T) { + mode := os.Getenv("OPENAI_CLI_LIMITS_BORROWED_PROCESS") + if mode == "" { + return + } + separator := slices.Index(os.Args, "--") + if separator < 0 { + t.Fatal("missing subprocess argument separator") + } + sink := &limitsBorrowedOutputProbe{fail: mode == "write-failure"} + borrowed := terminal.BorrowedWriter{Writer: sink, File: os.Stdout} + commands.Command.Writer, commands.Command.Reader = borrowed, os.Stdin + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + code := app.Run(ctx, os.Args[separator+1:]) + if err := ctx.Err(); err != nil { + t.Fatal("workspace required fallback cancellation", err) + } + if sink.writes.Load() == 0 || sink.fail && !sink.failed.Load() { + t.Fatal("workspace bypassed the borrowed output sink") + } + if _, err := os.Stdout.Stat(); err != nil { + t.Fatal("workspace closed the caller's terminal", err) + } + if _, err := fmt.Fprintln(os.Stderr, "borrowed-terminal-verified"); err != nil { + t.Fatal(err) + } + os.Exit(code) +} + +func TestMainLimitsWorkflowBorrowedTerminalWriter(t *testing.T) { + python, err := exec.LookPath("python3") + if err != nil { + t.Skip("Python 3 is required for native Unix borrowed-terminal checks") + } + binary, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, test := range []struct { + name, mode string + project bool + exit int + }{ + {"organization", "quit", false, 0}, + {"project", "quit", true, 0}, + {"interrupt", "interrupt", false, 130}, + {"output failure", "write-failure", false, 1}, + } { + t.Run(test.name, func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + return limitsWorkflowSuccess(t, test.project, r) + }) + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + args := []string{"-I", "-B", "-c", limitsBorrowedTerminalPTY, binary, + "-test.run=^TestMainLimitsBorrowedWriterProcess$", "--", "openai"} + args = append(args, limitsWorkflowArgs(test.project, false)...) + child := exec.CommandContext(ctx, python, args...) + child.Env = spendRecoverySetEnvironment(spendRecoveryEnvironment(t, server.URL), "CI", "") + child.Env = append(child.Env, "PATH=/usr/bin:/bin", "TERM=xterm-256color", "GOMAXPROCS=2", + "OPENAI_CLI_LIMITS_BORROWED_PROCESS="+test.mode) + child.Cancel = func() error { return child.Process.Signal(syscall.SIGTERM) } + child.WaitDelay = 2 * time.Second + output, err := child.CombinedOutput() + if err != nil || ctx.Err() != nil { + t.Fatalf("borrowed terminal harness failed: error=%v context=%v output=%s", err, ctx.Err(), output) + } + var got struct { + Code int + Stdout, Stderr string + } + if err := json.Unmarshal(output, &got); err != nil { + t.Fatalf("invalid borrowed terminal capture: %v; %s", err, output) + } + if got.Code != test.exit || !strings.Contains(got.Stderr, "borrowed-terminal-verified\n") { + t.Fatalf("borrowed terminal changed exit or ownership: %+v", got) + } + enter, leave := strings.Index(got.Stdout, "\x1b[?1049h"), strings.LastIndex(got.Stdout, "\x1b[?1049l") + if enter < 0 || leave <= enter || strings.Contains(got.Stdout+got.Stderr, "panic:") || strings.Contains(got.Stderr, "synthetic-private") { + t.Fatalf("workspace did not restore safe terminal output: %+v", got) + } + if test.exit == 0 && (!strings.Contains(got.Stdout[leave:], "Limits workspace closed.") || got.Stderr != "borrowed-terminal-verified\n") { + t.Fatalf("workspace did not close normally after restoration: %+v", got) + } + if test.mode == "write-failure" && got.Stderr != "Could not write command output. Check your terminal, output file, or receiving command.\nborrowed-terminal-verified\n" { + t.Fatalf("workspace output failure omitted safe recovery: %q", got.Stderr) + } + for _, path := range []string{spendThresholdPath(test.project, "spend-limit"), spendThresholdPath(test.project, "spend-alerts"), "/organization/costs"} { + if requests.count(path) != 1 { + t.Errorf("workspace did not load %s exactly once", path) + } + } + }) + } +} + +// The private PTY follows the existing process-test cleanup pattern. Only the +// ready workspace receives a key; output failure ends after a redraw signal. +const limitsBorrowedTerminalPTY = ` +import errno, fcntl, json, os, pty, selectors, signal, struct, subprocess, sys, termios, time +def interrupted(signum, frame): + raise TimeoutError("borrowed terminal harness interrupted") +signal.signal(signal.SIGTERM, interrupted) +master, slave = pty.openpty() +fcntl.ioctl(slave, termios.TIOCSWINSZ, struct.pack("HHHH", 24, 80, 0, 0)) +def session(): + os.setsid() + fcntl.ioctl(slave, termios.TIOCSCTTY, 0) +selector = selectors.DefaultSelector() +process = None +completed = False +output = [bytearray(), bytearray()] +try: + process = subprocess.Popen(sys.argv[1:], stdin=slave, stdout=slave, stderr=subprocess.PIPE, preexec_fn=session) + os.close(slave) + slave = -1 + for descriptor, index in [(master, 0), (process.stderr.fileno(), 1)]: + os.set_blocking(descriptor, False) + selector.register(descriptor, selectors.EVENT_READ, index) + sent = False + mode = os.environ["OPENAI_CLI_LIMITS_BORROWED_PROCESS"] + deadline = time.monotonic() + 10 + while selector.get_map() or process.poll() is None: + if time.monotonic() >= deadline: + raise TimeoutError("borrowed terminal command exceeded ten seconds") + for key, _ in selector.select(.1): + try: + data = os.read(key.fd, 65536) + except OSError as error: + if error.errno == errno.EAGAIN: + continue + if error.errno != errno.EIO: + raise + data = b"" + if not data: + selector.unregister(key.fd) + continue + output[key.data].extend(data) + if len(output[key.data]) > 2 << 20: + raise RuntimeError("borrowed terminal capture exceeded its allowance") + if not sent and mode in ("quit", "interrupt") and b"Website controls and source errors: ?" in output[0]: + key = b"\x03" if mode == "interrupt" else b"q" + os.write(master, key) + sent = True + elif not sent and mode == "write-failure" and b"Website controls and source errors: ?" in output[0]: + os.kill(process.pid, signal.SIGWINCH) + sent = True + code = process.wait(timeout=2) + completed = True +finally: + if process is not None and not completed: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait(timeout=2) + if process is not None: + process.stderr.close() + selector.close() + os.close(master) + if slave >= 0: + os.close(slave) +print(json.dumps({"Code": code, "Stdout": output[0].decode(), "Stderr": output[1].decode()})) +` + +func TestMainLimitsWorkflowPartialFailureSurvivesClosedPipe(t *testing.T) { + for _, test := range []struct { + name string + sourceFailure bool + }{{"complete sources", false}, {"partial source failure", true}} { + t.Run(test.name, func(t *testing.T) { + server, requests := newLimitsWorkflowServer(t, func(r *http.Request) (int, string) { + if test.sourceFailure && r.URL.Path == "/organization/spend_alerts" { + return http.StatusForbidden, `{"error":{"message":"private synthetic source failure"}}` + } + status, body := limitsWorkflowSuccess(t, false, r) + if r.URL.Path == "/organization/spend_limit" { + body = strings.TrimSuffix(body, "}") + `,"future":"` + strings.Repeat("synthetic", 2<<20) + `"}` + } + return status, body + }) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + child := batchesWorkflowProcess(t, ctx, server, "admin", "limits", "show", "--format", "json") + child.Env = append(child.Env, "OPENAI_ADMIN_KEY=sk-fake-spend-admin") + stdout, err := child.StdoutPipe() + if err != nil { + t.Fatal(err) + } + defer stdout.Close() + var stderr bytes.Buffer + child.Stderr = &stderr + if err := child.Start(); err != nil { + t.Fatal(err) + } + defer func() { + if child.ProcessState == nil { + _ = child.Process.Kill() + _ = child.Wait() + } + }() + if _, err := io.ReadFull(stdout, make([]byte, 1)); err != nil { + t.Fatal(err) + } + if err := stdout.Close(); err != nil { + t.Fatal(err) + } + err = child.Wait() + if ctx.Err() != nil || requests.count("/organization/spend_alerts") != 1 { + t.Fatalf("closed-pipe command timed out or repeated a source: context=%v stderr=%q", ctx.Err(), stderr.String()) + } + if !test.sourceFailure { + if err != nil || stderr.Len() != 0 { + t.Fatalf("lone output EPIPE changed success: error=%v stderr=%q", err, stderr.String()) + } + return + } + exit, ok := err.(*exec.ExitError) + if !ok || exit.ExitCode() != 1 { + t.Fatalf("source failure disappeared after EPIPE: error=%v stderr=%q", err, stderr.String()) + } + if !json.Valid(stderr.Bytes()) || !strings.Contains(stderr.String(), "overview is incomplete") || strings.Contains(stderr.String(), "private synthetic") { + t.Fatalf("safe structured recovery missing: %q", stderr.String()) + } + }) + } +} diff --git a/cmd/openai/spend_error_recovery_test.go b/cmd/openai/spend_error_recovery_test.go new file mode 100644 index 00000000..f2f0ec1a --- /dev/null +++ b/cmd/openai/spend_error_recovery_test.go @@ -0,0 +1,523 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "slices" + "strings" + "sync/atomic" + "testing" +) + +func spendRecoveryEnvironment(t *testing.T, endpoint string) []string { + t.Helper() + home := t.TempDir() + return []string{ + "HOME=" + home, "USERPROFILE=" + home, "XDG_CONFIG_HOME=" + home, + "OPENAI_BASE_URL=" + endpoint, "OPENAI_API_KEY=sk-fake-spend-project", + "OPENAI_ADMIN_KEY=sk-fake-spend-admin", "OPENAI_ORG_ID=org_synthetic_recovery", + "OPENAI_PROJECT_ID=proj_synthetic_context", "OPENAI_CUSTOM_HEADERS=X-Spend-Recovery: retained", + "NO_COLOR=1", "FORCE_COLOR=0", "CI=1", + } +} + +func spendRecoverySetEnvironment(env []string, key, value string) []string { + result := slices.Clone(env) + for i, entry := range result { + if strings.HasPrefix(entry, key+"=") { + result[i] = key + "=" + value + return result + } + } + return append(result, key+"="+value) +} + +func checkSpendRecoveryContext(t *testing.T, r *http.Request, adminKey string) { + t.Helper() + if r.Header.Get("Authorization") != "Bearer "+adminKey || + r.Header.Get("OpenAI-Organization") != "org_synthetic_recovery" || + r.Header.Get("OpenAI-Project") != "proj_synthetic_context" || + r.Header.Get("X-Spend-Recovery") != "retained" { + t.Error("recovery changed the original synthetic request context") + } +} + +func writeSpendRecoveryFailure(w http.ResponseWriter, status int, code string) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("X-Should-Retry", "false") + w.Header().Set("X-Request-Id", "req_synthetic_recovery") + w.WriteHeader(status) + fmt.Fprintf(w, `{"error":%s}`, spendRecoveryDetails(code)) +} + +func spendRecoveryDetails(code string) string { + return `{"message":"synthetic-private-response\u001b]52;c;synthetic-private\u0007","type":"invalid_request_error","code":"` + code + `","param":null,"future":{"sequence":9007199254740993}}` +} + +func checkSpendRecoveryFailure(t *testing.T, got mainDispatchResult, wants ...string) { + t.Helper() + if got.code != 1 || got.stdout != "" { + t.Fatalf("failure changed exit or output routing: %+v", got) + } + for _, want := range wants { + if !strings.Contains(got.stderr, want) { + t.Errorf("missing recovery instruction %q: %q", want, got.stderr) + } + } + for _, private := range []string{"synthetic-private", "sk-fake-", "\x1b", "\a"} { + if strings.Contains(got.stderr, private) { + t.Errorf("readable recovery exposed synthetic private data: %q", got.stderr) + } + } +} + +// These fixtures invoke the public entrypoint with the literal openai identity. +// Follow its displayed help, rather than maintaining a second recovery command. +func followSpendRecoveryHelp(t *testing.T, env []string, diagnostic, label string) mainDispatchResult { + t.Helper() + for _, line := range strings.Split(diagnostic, "\n") { + command, ok := strings.CutPrefix(line, label+": ") + if !ok { + continue + } + if !strings.HasPrefix(command, "openai help ") { + t.Fatalf("unexpected recovery command: %q", command) + } + got := runMainDispatchWithEnv(t, "bash", env, strings.Fields(command)...) + if got.code != 0 || got.stderr != "" || got.stdout == "" { + t.Fatalf("displayed recovery help failed: %+v", got) + } + return got + } + t.Fatalf("missing %s recovery line: %q", label, diagnostic) + return mainDispatchResult{} +} + +func TestMainSpendErrorRecoveryResponseConversion(t *testing.T) { + for _, project := range []bool{false, true} { + for _, operation := range []struct{ resource, action string }{ + {"spend-limit", "update"}, {"spend-limit", "delete"}, + {"spend-alerts", "create"}, {"spend-alerts", "update"}, {"spend-alerts", "delete"}, + {"rate-limits", "update-rate-limit"}, + } { + if !project && operation.resource == "rate-limits" { + continue + } + t.Run(fmt.Sprintf("project=%t/%s/%s", project, operation.resource, operation.action), func(t *testing.T) { + var requests atomic.Int32 + path := spendThresholdPath(project, operation.resource) + if operation.resource == "spend-alerts" && operation.action != "create" { + path += "/alert_synthetic" + } else if operation.resource == "rate-limits" { + path += "/rl_synthetic" + } + method := http.MethodPost + if operation.action == "delete" { + method = http.MethodDelete + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + if r.Method != method || r.URL.Path != path { + t.Errorf("unexpected mutation request: %s %s", r.Method, r.URL.Path) + } + body, err := io.ReadAll(r.Body) + if err != nil || method == http.MethodDelete && len(body) != 0 { + t.Errorf("unexpected mutation body: %s; %v", body, err) + } + if method != http.MethodDelete { + var fields map[string]any + if err := json.Unmarshal(body, &fields); err != nil { + t.Errorf("invalid mutation body: %s; %v", body, err) + } else if operation.resource == "rate-limits" { + if len(fields) != 1 || fields["max_requests_per_1_minute"] != float64(300) { + t.Errorf("rate-limit fields changed: %s", body) + } + } else if fields["threshold_amount"] != float64(10000) || fields["currency"] != "USD" || fields["interval"] != "month" { + t.Errorf("mutation fields changed: %s", body) + } + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"synthetic-private-response": broken}`) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + argv := append([]string{"openai"}, spendThresholdArgs(project, operation.resource, operation.action)...) + if operation.resource == "rate-limits" { + argv = append(argv, "--rate-limit-id", "rl_synthetic", "--max-requests-per-1-minute", "300") + } else if operation.action != "delete" { + argv = append(argv, "--currency", "USD", "--interval", "month", "--threshold-amount", "10000") + if operation.resource == "spend-alerts" { + argv = append(argv, "--notification-channel", `{"type":"email","recipients":["synthetic@example.invalid"]}`) + } + } + argv = append(argv, "--format", "yaml") + got := runMainDispatchWithEnv(t, "bash", env, append(slices.Clone(argv), "--format-error", "text")...) + checkSpendRecoveryFailure(t, got, "The API may have received the change.", "Check its current state before repeating", "Keep the original API address") + if operation.resource == "rate-limits" { + checkSpendRecoveryFailure(t, got, "rate-limit change", "help admin organization projects rate-limits list-rate-limits") + } + followSpendRecoveryHelp(t, env, got.stderr, "Read options") + if requests.Load() != 1 { + t.Fatal("response conversion or read help repeated a mutation", requests.Load()) + } + if operation.resource != "rate-limits" && (project || operation.resource != "spend-alerts" || operation.action != "create") { + return + } + const baseline = "Could not decode JSON: invalid JSON syntax." + for _, format := range []string{"json", "jsonl", "text"} { + args := append(slices.Clone(argv), "--format-error", format) + if format == "text" { + args = append(args, "--transform-error", "message") + } + before := requests.Load() + machine := runMainDispatchWithEnv(t, "bash", env, args...) + checkSpendRecoveryFailure(t, machine, baseline) + if requests.Load() != before+1 { + t.Fatal("machine error formatting retried the mutation") + } + if format == "text" { + if machine.stderr != baseline+"\n" { + t.Fatal("error extraction changed", machine.stderr) + } + } else { + var value map[string]any + if json.Unmarshal([]byte(machine.stderr), &value) != nil || len(value) != 1 || value["message"] != baseline { + t.Fatal("machine error contract changed", machine.stderr) + } + } + } + }) + } + } +} + +func TestMainSpendErrorRecoveryInputDecodeDoesNotClaimMutation(t *testing.T) { + for _, test := range []struct{ name, input string }{ + {"syntax", `[synthetic-private-input not closed`}, + {"type", `[{"synthetic-private-input":"value"}]`}, + } { + t.Run(test.name, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + argv := spendThresholdArgs(false, "spend-alerts", "create") + argv = append(argv, "--currency", "USD", "--interval", "month", "--threshold-amount", "10000", + "--notification-channel", test.input) + human := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format", "yaml", "--format-error", "text"}, argv...)...) + checkSpendRecoveryFailure(t, human) + if human.stderr == "" || strings.Contains(human.stderr, "may have received") || strings.Contains(human.stderr, "before repeating") { + t.Fatal("input decoding incorrectly claimed an attempted mutation", human.stderr) + } + machine := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format", "yaml", "--format-error", "json"}, argv...)...) + checkSpendRecoveryFailure(t, machine) + var value map[string]any + if json.Unmarshal([]byte(machine.stderr), &value) != nil || len(value) != 1 { + t.Fatal("invalid machine input error", machine.stderr) + } + message, ok := value["message"].(string) + if !ok || message == "" { + t.Fatal("machine input error omitted its message", machine.stderr) + } + extracted := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format", "yaml", "--format-error", "text", "--transform-error", "message"}, argv...)...) + checkSpendRecoveryFailure(t, extracted) + if extracted.stderr != message+"\n" || requests.Load() != 0 { + t.Fatal("input error changed extraction or sent an HTTP request", extracted.stderr, requests.Load()) + } + }) + } +} + +func TestMainSpendErrorRecoveryAdminSetup(t *testing.T) { + for _, args := range [][]string{ + {"admin", "organization", "spend-limit", "retrieve"}, + {"admin", "spend-limit", "retrieve"}, + {"admin:organization:spend-limit", "retrieve"}, + {"projects", "spend-limit", "retrieve", "--project-id", "proj_synthetic"}, + } { + t.Run(strings.Join(args, "/"), func(t *testing.T) { + var requests atomic.Int32 + project := args[0] == "projects" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + attempt := requests.Add(1) + if r.Method != http.MethodGet || r.URL.Path != spendThresholdPath(project, "spend-limit") { + t.Error("unexpected authentication recovery request") + } + if attempt == 1 { + checkSpendRecoveryContext(t, r, "sk-fake-rejected-admin") + writeSpendRecoveryFailure(w, http.StatusUnauthorized, "invalid_api_key") + return + } + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(project, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", "sk-fake-rejected-admin") + argv := append([]string{"openai"}, args...) + got := runMainDispatchWithEnv(t, "bash", env, argv...) + checkSpendRecoveryFailure(t, got, "HTTP 401", "OPENAI_ADMIN_KEY", "help setup admin") + help := followSpendRecoveryHelp(t, env, got.stderr, "Key setup") + if !strings.Contains(help.stdout, "organization/admin-keys") || !strings.Contains(help.stdout, "OPENAI_ADMIN_KEY") || requests.Load() != 1 { + t.Fatal("admin setup recovery omitted admin instructions or performed a request") + } + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", "sk-fake-spend-admin") + recovered := runMainDispatchWithEnv(t, "bash", env, argv...) + if recovered.code != 0 || recovered.stderr != "" || requests.Load() != 2 { + t.Fatalf("following admin setup did not recover the fixture: %+v", recovered) + } + }) + } +} + +func TestMainSpendErrorRecoveryNotFound(t *testing.T) { + for _, test := range []struct { + name, resource string + project, missingProject bool + }{ + {"organization limit", "spend-limit", false, false}, + {"project missing", "spend-limit", true, true}, + {"project limit missing", "spend-limit", true, false}, + {"organization alert", "spend-alerts", false, false}, + {"project alert", "spend-alerts", true, false}, + } { + t.Run(test.name, func(t *testing.T) { + var requests, writes atomic.Int32 + var configured atomic.Bool + path := spendThresholdPath(test.project, test.resource) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodPost && r.URL.Path == path && test.name == "organization limit": + writes.Add(1) + configured.Store(true) + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + case r.Method == http.MethodGet && r.URL.Path == path && test.resource == "spend-alerts": + alert := spendThresholdBody(test.project, "spend-alerts", `{"id":"alert_recovered","threshold_amount":10000,"currency":"USD","interval":"month"}`) + fmt.Fprintf(w, `{"object":"list","data":[%s],"has_more":false}`, alert) + case r.Method == http.MethodGet && r.URL.Path == "/organization/projects/proj_synthetic": + if test.missingProject { + writeSpendRecoveryFailure(w, 404, "not_found") + } else { + io.WriteString(w, `{"id":"proj_synthetic","object":"organization.project","name":"Synthetic project","status":"active"}`) + } + case r.Method == http.MethodGet && r.URL.Path == path && configured.Load(): + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + default: + writeSpendRecoveryFailure(w, 404, "not_found") + } + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + args := spendThresholdArgs(test.project, test.resource, "retrieve") + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai"}, args...)...) + checkSpendRecoveryFailure(t, got, "HTTP 404", "Keep the original API address") + label := "Limit options" + if test.project && test.resource == "spend-limit" { + label = "Project lookup" + if !strings.Contains(got.stderr, "If the project exists") { + t.Error("project404 lost missing-project versus missing-limit uncertainty") + } + } else if test.resource == "spend-alerts" { + label = "List options" + } else if !strings.Contains(got.stderr, "may not be configured") { + t.Error("organization404 falsely classified an unknown server state") + } + followSpendRecoveryHelp(t, env, got.stderr, label) + if requests.Load() != 1 || writes.Load() != 0 { + t.Fatal("recovery help performed a request") + } + var read []string + if test.resource == "spend-alerts" { + read = spendThresholdArgs(test.project, test.resource, "list") + } else if test.project { + read = []string{"admin", "organization", "projects", "retrieve", "--project-id", "proj_synthetic"} + } else { + update := []string{"openai", "admin", "organization", "spend-limit", "update", "--currency", "USD", "--interval", "month", "--threshold-amount", "10000"} + if saved := runMainDispatchWithEnv(t, "bash", env, update...); saved.code != 0 { + t.Fatalf("documented limit configuration failed: %+v", saved) + } + read = args + } + recovered := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai"}, read...)...) + wantExit := 0 + if test.missingProject { + wantExit = 1 + } + if recovered.code != wantExit { + t.Fatalf("documented recovery did not identify the fixture state: %+v", recovered) + } + if test.resource == "spend-alerts" && !strings.Contains(recovered.stdout, "alert_recovered") { + t.Error("alert recovery did not expose the available alert ID") + } + }) + } +} + +func TestMainSpendErrorRecoveryOrganizationCap(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if r.Method != http.MethodGet { + t.Error("cap recovery attempted a mutation") + } + if r.URL.Path == "/models/synthetic-model" { + checkSpendRecoveryContext(t, r, "sk-fake-spend-project") + writeSpendRecoveryFailure(w, 429, "organization_spend_limit_exceeded") + return + } + if r.URL.Path != "/organization/spend_limit" { + t.Error("cap recovery selected the wrong scope") + } + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + })) + defer server.Close() + env := spendRecoverySetEnvironment(spendRecoveryEnvironment(t, server.URL), "OPENAI_ADMIN_KEY", "") + got := runMainDispatchWithEnv(t, "bash", env, "openai", "models", "retrieve", "--model", "synthetic-model") + checkSpendRecoveryFailure(t, got, "HTTP 429", "organization spend limit", "admin API key", "Keep the original API address") + if strings.Contains(got.stderr, "project's billing") || strings.Contains(got.stderr, "Wait before trying") { + t.Error("organization cap used project-billing or rate-throttling guidance") + } + followSpendRecoveryHelp(t, env, got.stderr, "Limit lookup") + followSpendRecoveryHelp(t, env, got.stderr, "Admin key setup") + if requests.Load() != 1 { + t.Fatal("cap recovery help performed a request") + } + env = spendRecoverySetEnvironment(env, "OPENAI_ADMIN_KEY", "sk-fake-spend-admin") + read := runMainDispatchWithEnv(t, "bash", env, "openai", "admin", "organization", "spend-limit", "retrieve") + if read.code != 0 || read.stderr != "" || requests.Load() != 2 { + t.Fatalf("cap recovery failed with retained context and an admin key: %+v", read) + } +} + +func TestMainSpendErrorRecoveryMachineContracts(t *testing.T) { + for _, test := range []struct { + status int + code string + }{ + {401, "invalid_api_key"}, {404, "not_found"}, {429, "organization_spend_limit_exceeded"}, + } { + t.Run(fmt.Sprint(test.status), func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + writeSpendRecoveryFailure(w, test.status, test.code) + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + args := []string{"admin", "organization", "spend-limit", "retrieve"} + for _, flag := range []string{"--format", "--format-error"} { + for _, format := range []string{"json", "jsonl", "raw", "yaml"} { + argv := append([]string{"openai", flag, format}, args...) + got := runMainDispatchWithEnv(t, "bash", env, argv...) + if got.code != 1 || got.stdout != "" { + t.Fatalf("machine error changed exit or output routing: %+v", got) + } + if format == "yaml" { + want := "message: \"synthetic-private-response\\x1B]52;c;synthetic-private\\x07\"\n" + + "type: invalid_request_error\ncode: " + test.code + "\nparam: null\nfuture:\n sequence: 9007199254740993\n" + if got.stderr != want { + t.Errorf("YAML error bytes changed: %q", got.stderr) + } + } else { + assertSpendThresholdJSON(t, got.stderr, spendRecoveryDetails(test.code)) + if (format == "raw" || format == "jsonl") && got.stderr != spendRecoveryDetails(test.code)+"\n" { + t.Errorf("%s error bytes changed", format) + } + } + } + } + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai", "--format-error", "text", "--transform-error", "future.sequence"}, args...)...) + if got.code != 1 || got.stdout != "" || got.stderr != "9007199254740993\n" { + t.Fatalf("error extraction changed: %+v", got) + } + }) + } +} + +func TestMainSpendErrorRecoveryLostAndTruncatedResponse(t *testing.T) { + for _, truncated := range []bool{false, true} { + t.Run(fmt.Sprintf("truncated=%t", truncated), func(t *testing.T) { + var writes, reads atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + checkSpendRecoveryContext(t, r, "sk-fake-spend-admin") + if r.URL.Path != "/organization/spend_limit" { + t.Error("unexpected mutation recovery route") + } + if r.Method == http.MethodGet { + reads.Add(1) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + return + } + if r.Method != http.MethodPost { + t.Error("unexpected mutation method") + } + var body map[string]any + if err := json.NewDecoder(r.Body).Decode(&body); err != nil || body["threshold_amount"] != float64(10000) { + t.Error("mutation body changed") + } + writes.Add(1) + connection, buffer, err := w.(http.Hijacker).Hijack() + if err != nil { + t.Error(err) + return + } + defer connection.Close() + if truncated { + io.WriteString(buffer, "HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: 1000\r\nConnection: close\r\n\r\n{\"object\":\"organization.spend_limit\"") + buffer.Flush() + } + })) + defer server.Close() + env := spendRecoveryEnvironment(t, server.URL) + mutation := []string{"admin", "organization", "spend-limit", "update", "--currency", "USD", "--interval", "month", "--threshold-amount", "10000"} + got := runMainDispatchWithEnv(t, "bash", env, append([]string{"openai"}, mutation...)...) + checkSpendRecoveryFailure(t, got, "The API may have received the change", "before repeating the command") + wantAttempts := int32(3) + if truncated { + wantAttempts = 1 + } + if writes.Load() != wantAttempts { + t.Errorf("request retry behavior changed: got %d attempts, want %d", writes.Load(), wantAttempts) + } + followSpendRecoveryHelp(t, env, got.stderr, "Read options") + if writes.Load() != wantAttempts || reads.Load() != 0 { + t.Fatal("response-loss recovery help performed a request") + } + read := runMainDispatchWithEnv(t, "bash", env, "openai", "admin", "organization", "spend-limit", "retrieve") + if read.code != 0 || read.stderr != "" || writes.Load() != wantAttempts || reads.Load() != 1 { + t.Fatalf("recovery did not inspect state without another mutation: %+v", read) + } + baseline := "Could not connect to the API. Check your connection, proxy, and --base-url setting." + if truncated { + baseline = "The command could not be completed. Check your arguments with --help." + } + for _, flags := range [][]string{{"--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { + argv := append(append([]string{"openai"}, flags...), mutation...) + machine := runMainDispatchWithEnv(t, "bash", env, argv...) + if machine.code != 1 || machine.stdout != "" { + t.Fatalf("machine transport error changed exit or output routing: %+v", machine) + } + if flags[1] == "json" { + var value map[string]string + if err := json.Unmarshal([]byte(machine.stderr), &value); err != nil || len(value) != 1 || value["message"] != baseline { + t.Errorf("machine transport fallback changed: %q", machine.stderr) + } + } else if machine.stderr != baseline+"\n" { + t.Errorf("transport extraction changed: %q", machine.stderr) + } + } + }) + } +} diff --git a/cmd/openai/spend_thresholds_test.go b/cmd/openai/spend_thresholds_test.go new file mode 100644 index 00000000..0a260c06 --- /dev/null +++ b/cmd/openai/spend_thresholds_test.go @@ -0,0 +1,653 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" +) + +func runSpendThresholdCommand(t *testing.T, server *httptest.Server, args ...string) mainDispatchResult { + t.Helper() + return runMainDispatchWithEnv(t, "bash", []string{ + "OPENAI_BASE_URL=" + server.URL, + "OPENAI_API_KEY=sk-fake-spend-test", + "OPENAI_ADMIN_KEY=sk-fake-spend-admin", + "FORCE_COLOR=0", "NO_COLOR=1", "COLUMNS=28", + }, append([]string{"openai"}, args...)...) +} + +func spendThresholdArgs(project bool, resource, operation string) []string { + args := []string{"admin", "organization"} + if project { + args = append(args, "projects") + } + args = append(args, resource, operation) + if project { + args = append(args, "--project-id", "proj_synthetic") + } + if resource == "spend-alerts" && operation != "create" && operation != "list" { + args = append(args, "--alert-id", "alert_synthetic") + } + return args +} + +func spendThresholdPath(project bool, resource string) string { + path := "/organization" + if project { + path += "/projects/proj_synthetic" + } + return path + "/" + strings.ReplaceAll(resource, "-", "_") +} + +func spendThresholdBody(project bool, resource, fields string) string { + scope := "organization" + if project { + scope = "project" + } + object := scope + "." + strings.TrimSuffix(strings.ReplaceAll(resource, "-", "_"), "s") + return `{"object":"` + object + `",` + strings.TrimPrefix(fields, "{") +} + +func assertSpendThresholdJSON(t *testing.T, output string, records ...string) { + t.Helper() + decoder := json.NewDecoder(strings.NewReader(output)) + decoder.UseNumber() + for _, record := range records { + original := json.NewDecoder(strings.NewReader(record)) + original.UseNumber() + var actual, want any + if err := original.Decode(&want); err != nil { + t.Fatal(err) + } + if err := decoder.Decode(&actual); err != nil || !reflect.DeepEqual(actual, want) { + t.Fatalf("spend response changed: actual=%v want=%v error=%v; output=%q", actual, want, err, output) + } + } + if err := decoder.Decode(new(any)); err != io.EOF { + t.Fatalf("unexpected trailing spend response: %v; output=%q", err, output) + } +} + +func TestMainSpendThresholdRoutesAndRequests(t *testing.T) { + for _, project := range []bool{false, true} { + for _, operation := range []struct{ resource, method string }{ + {"spend-limit", "retrieve"}, {"spend-limit", "update"}, + {"spend-alerts", "create"}, {"spend-alerts", "retrieve"}, {"spend-alerts", "update"}, + } { + for _, format := range []string{"", "text"} { + t.Run(fmt.Sprintf("project=%t/%s/%s/%s", project, operation.resource, operation.method, format), func(t *testing.T) { + path := spendThresholdPath(project, operation.resource) + if operation.resource == "spend-alerts" && operation.method != "create" { + path += "/alert_synthetic" + } + mutation := operation.method != "retrieve" + bodies := make(chan string, 8) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + method := http.MethodGet + if mutation { + method = http.MethodPost + } + if r.Method != method || r.URL.Path != path { + t.Errorf("unexpected request: %s %s, want %s %s", r.Method, r.URL.Path, method, path) + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + bodies <- string(body) + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(project, operation.resource, `{"id":"alert_synthetic","currency":"USD","interval":"month","threshold_amount":10000,"future":{"note":"preserved"}}`)) + })) + defer server.Close() + args := spendThresholdArgs(project, operation.resource, operation.method) + if mutation { + args = append(args, "--currency", "USD", "--interval", "month", "--threshold-amount", "9007199254740993") + if operation.resource == "spend-alerts" { + args = append(args, "--notification-channel", `{"type":"email","recipients":["synthetic@example.invalid"],"subject_prefix":null}`) + } + } + if format != "" { + args = append([]string{"--format", format}, args...) + } + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("spend command failed: %+v", got) + } + if len(bodies) != 1 { + t.Fatalf("received %d requests, want one", len(bodies)) + } + body := <-bodies + if mutation { + want := `{"currency":"USD","interval":"month","threshold_amount":9007199254740993}` + if operation.resource == "spend-alerts" { + want = `{"currency":"USD","interval":"month","threshold_amount":9007199254740993,"notification_channel":{"type":"email","recipients":["synthetic@example.invalid"],"subject_prefix":null}}` + } + assertSpendThresholdJSON(t, body, want) + } else if body != "" { + t.Errorf("retrieve sent a request body: %q", body) + } + for _, want := range []string{"Spend threshold: USD 100.00 per month", "Currency: USD", "Interval: month", "Note: preserved"} { + if !strings.Contains(got.stdout, want) { + t.Errorf("missing %q in %q", want, got.stdout) + } + } + want := "Enforcement: not reported in this response" + if operation.resource == "spend-alerts" { + want = "Alert behavior: Alerts notify; they are not spending caps." + } + if !strings.Contains(got.stdout, want) || strings.Contains(got.stdout, "Threshold amount:") { + t.Errorf("unclear spend threshold presentation: %q; want %q", got.stdout, want) + } + }) + } + } + } +} + +func TestMainSpendThresholdPrecisionAndUnknownFields(t *testing.T) { + for _, tc := range []struct { + name, body string + want, absent []string + }{ + {"zero", `{"threshold_amount":0,"currency":"USD","interval":"month"}`, []string{"Spend threshold: USD 0.00 per month"}, nil}, + {"negative", `{"threshold_amount":-1,"currency":"USD","interval":"month"}`, []string{"Spend threshold: USD -0.01 per month"}, nil}, + {"large", `{"threshold_amount":9223372036854775807,"currency":"USD","interval":"month"}`, []string{"Spend threshold: USD 92233720368547758.07 per month"}, nil}, + {"decimal", `{"threshold_amount":100.005,"currency":"USD","interval":"month"}`, []string{"100.005", "cents"}, []string{"USD 1.00"}}, + {"exponent", `{"threshold_amount":1e4,"currency":"USD","interval":"month"}`, []string{"1e4", "cents"}, []string{"USD 100.00"}}, + {"unknown units", `{"threshold_amount":123,"currency":"ZZZ","interval":"quarter"}`, []string{"123 cents", "ZZZ", "(interval: quarter)"}, []string{"USD", "per month"}}, + {"missing units", `{"threshold_amount":123}`, []string{"123 cents"}, []string{"USD", "per month"}}, + {"missing interval", `{"threshold_amount":123,"currency":"USD"}`, []string{"USD 1.23"}, []string{"per month"}}, + {"null enforcement", `{"threshold_amount":123,"currency":"USD","interval":"month","enforcement":null}`, []string{"Enforcement: not reported in this response"}, nil}, + {"reported enforcement", `{"threshold_amount":123,"currency":"USD","interval":"month","enforcement":{"mode":"reported_mode","future":9007199254740993}}`, []string{"Mode: reported_mode", "Future: 9007199254740993"}, []string{"not reported in this response"}}, + {"null threshold", `{"threshold_amount":null,"future":"preserved"}`, []string{"Threshold amount: (null)", "Future: preserved"}, []string{"Spend threshold:", "USD", "per month"}}, + } { + t.Run(tc.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", tc.body)) + })) + defer server.Close() + got := runSpendThresholdCommand(t, server, spendThresholdArgs(false, "spend-limit", "retrieve")...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("threshold response failed: %+v", got) + } + for _, want := range tc.want { + if !strings.Contains(got.stdout, want) { + t.Errorf("missing %q in %q", want, got.stdout) + } + } + for _, absent := range tc.absent { + if strings.Contains(got.stdout, absent) { + t.Errorf("invented or changed %q in %q", absent, got.stdout) + } + } + }) + } +} + +func TestMainSpendThresholdShortAlias(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/organization/spend_limit" { + t.Errorf("unexpected alias request: %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month"}`)) + })) + defer server.Close() + for _, args := range [][]string{ + {"admin", "spend-limit", "retrieve"}, + {"admin:organization:spend-limit", "retrieve"}, + {"--format", "TEXT", "admin", "spend-limit", "retrieve"}, + } { + t.Run(strings.Join(args, "/"), func(t *testing.T) { + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "Spend threshold: USD 100.00 per month") || !strings.Contains(got.stdout, "Enforcement: not reported in this response") { + t.Fatalf("spend-limit alias failed: %+v", got) + } + }) + } +} + +func TestMainSpendThresholdReportedEnforcementStatus(t *testing.T) { + for _, status := range []string{"inactive", "enforcing", "future_status"} { + t.Run(status, func(t *testing.T) { + body := spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"`+status+`","future":9007199254740993}}`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + args := spendThresholdArgs(false, "spend-limit", "retrieve") + got := runSpendThresholdCommand(t, server, args...) + want := "Object: organization.spend_limit\nSpend threshold: USD 100.00 per month\nCurrency: USD\nInterval: month\nEnforcement:\n Status: " + status + "\n Future: 9007199254740993\n" + if got.code != 0 || got.stderr != "" || got.stdout != want { + t.Fatalf("reported enforcement status changed: %+v; want %q", got, want) + } + got = runSpendThresholdCommand(t, server, append([]string{"--transform", "enforcement"}, args...)...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("enforcement extraction failed: %+v", got) + } + assertSpendThresholdJSON(t, got.stdout, `{"status":"`+status+`","future":9007199254740993}`) + }) + } +} + +func TestMainSpendThresholdMachineFormatsAndExtraction(t *testing.T) { + for _, resource := range []string{"spend-limit", "spend-alerts"} { + body := spendThresholdBody(true, resource, `{"id":"alert_synthetic","currency":"USD","interval":"month","threshold_amount":9007199254740993,"future":{"exact":123456789012345678901234567890,"decimal":0.100000000000000001},"enforcement":null}`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + for _, format := range []string{"json", "JSON", "jsonl", "raw", "yaml", "pretty", "explore"} { + t.Run(resource+"/"+format, func(t *testing.T) { + args := append([]string{"--format", format}, spendThresholdArgs(true, resource, "retrieve")...) + got := runSpendThresholdCommand(t, server, args...) + wantStderr := "" + if format == "explore" { + wantStderr = "Warning: Output format 'explore' not supported for non-terminal output; falling back to 'json'\n" + } + if got.code != 0 || got.stderr != wantStderr { + t.Fatalf("explicit format failed: %+v", got) + } + if format == "yaml" || format == "pretty" { + for _, value := range []string{"threshold_amount: 9007199254740993", "exact: 123456789012345678901234567890", "decimal: 0.100000000000000001", "enforcement: null"} { + if !strings.Contains(got.stdout, value) { + t.Errorf("%s lost %q: %q", format, value, got.stdout) + } + } + } else { + assertSpendThresholdJSON(t, got.stdout, body) + } + if format == "raw" && got.stdout != body+"\n" { + t.Errorf("raw response bytes changed: %q", got.stdout) + } + if strings.Contains(got.stdout, "spend_threshold") || strings.Contains(got.stdout, "alert_behavior") { + t.Errorf("projection entered explicit output: %q", got.stdout) + } + }) + } + for _, tc := range []struct { + name string + flags []string + want string + }{ + {"raw-output", []string{"--raw-output"}, body}, + {"number extraction", []string{"--transform", "threshold_amount"}, "9007199254740993"}, + {"text extraction", []string{"--format", "text", "--transform", "threshold_amount"}, "9007199254740993"}, + {"raw extraction", []string{"--transform", "future.decimal", "--raw-output"}, "0.100000000000000001"}, + } { + t.Run(resource+"/"+tc.name, func(t *testing.T) { + got := runSpendThresholdCommand(t, server, append(tc.flags, spendThresholdArgs(true, resource, "retrieve")...)...) + if got.code != 0 || got.stderr != "" { + t.Fatalf("explicit extraction failed: %+v", got) + } + assertSpendThresholdJSON(t, got.stdout, tc.want) + }) + } + } +} + +func TestMainSpendAlertsPagination(t *testing.T) { + for _, project := range []bool{false, true} { + first := spendThresholdBody(project, "spend-alerts", `{"id":"alert_first","currency":"USD","interval":"month","threshold_amount":10000,"future":9007199254740993}`) + second := spendThresholdBody(project, "spend-alerts", `{"id":"alert_second","currency":"USD","interval":"month","threshold_amount":0}`) + page := `{"object":"list","data":[` + first + `],"has_more":true,"first_id":"alert_first","last_id":"alert_first"}` + for _, tc := range []struct { + name string + flags []string + max string + count, calls int + failure, raw bool + }{ + {name: "default", count: 2, calls: 2}, + {name: "unlimited", max: "-1", count: 2, calls: 2}, + {name: "zero", max: "0", calls: 1}, + {name: "one", max: "1", count: 1, calls: 1}, + {name: "raw page", flags: []string{"--format", "raw"}, count: 1, calls: 1, raw: true}, + {name: "jsonl", flags: []string{"--format", "jsonl"}, count: 2, calls: 2}, + {name: "partial failure", count: 1, calls: 2, failure: true}, + {name: "structured partial failure", flags: []string{"--format", "jsonl"}, count: 1, calls: 2, failure: true}, + } { + t.Run(fmt.Sprintf("project=%t/%s", project, tc.name), func(t *testing.T) { + var mu sync.Mutex + var cursors []string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cursor := r.URL.Query().Get("after") + mu.Lock() + cursors = append(cursors, cursor) + mu.Unlock() + if r.Method != http.MethodGet || r.URL.Path != spendThresholdPath(project, "spend-alerts") || r.URL.Query().Get("limit") != "1" { + t.Errorf("unexpected list request: %s %s", r.Method, r.URL) + } + w.Header().Set("Content-Type", "application/json") + switch cursor { + case "": + io.WriteString(w, page) + case "alert_first": + if tc.failure { + w.Header().Set("x-should-retry", "false") + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, `{"error":{"type":"invalid_request_error","message":"Synthetic page failure"}}`) + } else { + io.WriteString(w, `{"object":"list","data":[`+second+`],"has_more":false,"first_id":"alert_second","last_id":"alert_second"}`) + } + default: + http.Error(w, "unexpected cursor", http.StatusBadRequest) + } + })) + defer server.Close() + args := append(tc.flags, spendThresholdArgs(project, "spend-alerts", "list")...) + args = append(args, "--limit", "1") + if tc.max != "" { + args = append(args, "--max-items", tc.max) + } + got := runSpendThresholdCommand(t, server, args...) + mu.Lock() + actualCursors := slices.Clone(cursors) + mu.Unlock() + if want := []string{"", "alert_first"}[:tc.calls]; !slices.Equal(actualCursors, want) { + t.Fatalf("cursors=%q, want %q", actualCursors, want) + } + if tc.failure { + if got.code != 1 { + t.Fatalf("partial API failure disappeared: %+v", got) + } + if slices.Contains(tc.flags, "jsonl") { + payload := decodeMainStructuredError(t, "jsonl", got.stderr) + if payload["message"] != "Synthetic page failure" { + t.Errorf("structured page failure changed: %#v", payload) + } + } else if !strings.Contains(got.stderr, "HTTP 400: Bad Request.") || strings.Contains(got.stderr, "Synthetic page failure") { + t.Errorf("readable page failure framing changed: %q", got.stderr) + } + } else if got.code != 0 || got.stderr != "" { + t.Fatalf("spend alerts list failed: %+v", got) + } + if tc.raw { + if got.stdout != page+"\n" { + t.Fatalf("raw page changed: %q", got.stdout) + } + } else if slices.Contains(tc.flags, "jsonl") { + assertSpendThresholdJSON(t, got.stdout, []string{first, second}[:tc.count]...) + } else { + if strings.Count(got.stdout, "Spend threshold:") != tc.count || strings.Count(got.stdout, "Alerts notify; they are not spending caps.") != tc.count { + t.Fatalf("list lost threshold or alert clarification: %+v", got) + } + if tc.count > 0 && (!strings.Contains(got.stdout, "ID: alert_first") || !strings.Contains(got.stdout, "9007199254740993")) { + t.Errorf("partial result lost original metadata: %q", got.stdout) + } + if tc.count == 0 && got.stdout != "" { + t.Errorf("zero item list emitted output: %q", got.stdout) + } + } + }) + } + } +} + +func TestMainSpendAlertsJSONPagination(t *testing.T) { + for _, project := range []bool{false, true} { + first := spendThresholdBody(project, "spend-alerts", `{"id":"alert_first","currency":"USD","interval":"month","threshold_amount":9007199254740993,"future":{"decimal":0.100000000000000001}}`) + second := spendThresholdBody(project, "spend-alerts", `{"id":"alert_second","currency":"USD","interval":"month","threshold_amount":0}`) + for _, tc := range []struct { + name string + flags []string + count, calls int + empty, failure bool + }{ + {name: "two pages", count: 2, calls: 2}, + {name: "unlimited", flags: []string{"--max-items", "-1"}, count: 2, calls: 2}, + {name: "zero", flags: []string{"--max-items", "0"}, calls: 1}, + {name: "one", flags: []string{"--max-items", "1"}, count: 1, calls: 1}, + {name: "empty", empty: true, calls: 1}, + {name: "later failure", count: 1, calls: 2, failure: true}, + {name: "extraction", flags: []string{"--transform", "threshold_amount"}, count: 2, calls: 2}, + {name: "raw output", flags: []string{"--raw-output"}, count: 2, calls: 2}, + } { + t.Run(fmt.Sprintf("project=%t/%s", project, tc.name), func(t *testing.T) { + var calls atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + call := calls.Add(1) + if r.Method != http.MethodGet || r.URL.Path != spendThresholdPath(project, "spend-alerts") || r.URL.Query().Get("limit") != "1" { + t.Errorf("unexpected request: %s %s", r.Method, r.URL) + } + w.Header().Set("Content-Type", "application/json") + if call == 1 { + if r.URL.Query().Get("after") != "" { + t.Errorf("unexpected first cursor: %s", r.URL) + } + if tc.empty { + io.WriteString(w, `{"object":"list","data":[],"has_more":false}`) + } else { + fmt.Fprintf(w, `{"object":"list","data":[%s],"has_more":true,"last_id":"alert_first"}`, first) + } + return + } + if call != 2 || r.URL.Query().Get("after") != "alert_first" { + t.Errorf("unexpected later cursor: %s", r.URL) + } + if tc.failure { + w.Header().Set("x-should-retry", "false") + w.WriteHeader(http.StatusBadRequest) + io.WriteString(w, `{"error":{"message":"Synthetic JSON page failure","type":"invalid_request_error"}}`) + return + } + fmt.Fprintf(w, `{"object":"list","data":[%s],"has_more":false,"last_id":"alert_second"}`, second) + })) + defer server.Close() + args := append([]string{"--format", "json"}, spendThresholdArgs(project, "spend-alerts", "list")...) + args = append(args, "--limit", "1") + got := runSpendThresholdCommand(t, server, append(args, tc.flags...)...) + if int(calls.Load()) != tc.calls { + t.Fatalf("requests=%d; want %d", calls.Load(), tc.calls) + } + if tc.failure { + if got.code != 1 || decodeMainStructuredError(t, "json", got.stderr)["message"] != "Synthetic JSON page failure" { + t.Fatalf("later JSON failure disappeared: %+v", got) + } + assertFiniteJSONListPartial(t, got.stdout, "alert_first") + decoder := json.NewDecoder(strings.NewReader(got.stdout)) + if _, err := decoder.Token(); err != nil { + t.Fatal(err) + } + var record json.RawMessage + if err := decoder.Decode(&record); err != nil { + t.Fatal(err) + } + assertSpendThresholdJSON(t, string(record), first) + } else if got.code != 0 || got.stderr != "" { + t.Fatalf("JSON alert list failed: %+v", got) + } else if slices.Contains(tc.flags, "--transform") { + assertSpendThresholdJSON(t, got.stdout, "9007199254740993", "0") + } else if slices.Contains(tc.flags, "--raw-output") { + assertSpendThresholdJSON(t, got.stdout, first, second) + } else { + assertSpendThresholdJSON(t, got.stdout, "["+strings.Join([]string{first, second}[:tc.count], ",")+"]") + } + }) + } + } +} + +func TestMainSpendThresholdMalformedAPIErrors(t *testing.T) { + for _, tc := range []struct{ name, body string }{ + {"empty", ""}, + {"malformed", `{"error":{"message":"synthetic private spend detail"`}, + {"empty envelope", `{}`}, + {"empty error", `{"error":{}}`}, + } { + for _, format := range []string{"text", "json"} { + t.Run(tc.name+"/"+format, func(t *testing.T) { + var requests atomic.Int32 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + w.Header().Set("x-should-retry", "false") + w.WriteHeader(http.StatusBadGateway) + io.WriteString(w, tc.body) + })) + defer server.Close() + // Preserve the shared error contract, including an empty error object. + reference := runSpendThresholdCommand(t, server, "--format", format, "models", "retrieve", "model_synthetic") + args := append([]string{"--format", format}, spendThresholdArgs(false, "spend-limit", "retrieve")...) + got := runSpendThresholdCommand(t, server, args...) + if got != reference || got.code != 1 || got.stdout != "" || requests.Load() != 2 { + t.Fatalf("spend error framing changed: got=%+v reference=%+v requests=%d", got, reference, requests.Load()) + } + if format == "json" { + payload := decodeMainErrorObject(t, "json", got.stderr) + if payload["status_code"] != json.Number("502") || payload["message"] != "HTTP 502 Bad Gateway: the server returned no usable JSON error details." { + t.Errorf("fallback status or message changed: %#v", payload) + } + } else if !strings.Contains(got.stderr, "HTTP 502: Bad Gateway.") || !strings.Contains(got.stderr, "API error details: --format-error json.") { + t.Errorf("readable error lacks status or guidance: %q", got.stderr) + } + for _, absent := range []string{"synthetic private spend detail", "sk-fake-spend", "Spend threshold:", "spend_threshold"} { + if strings.Contains(got.stderr, absent) { + t.Errorf("error included %q: %q", absent, got.stderr) + } + } + }) + } + } +} + +func TestMainSpendThresholdControlsAndDeleteIdentity(t *testing.T) { + const body = `{"object":"organization.spend_limit","threshold_amount":123,"currency":"ZZZ\u001b[31m","interval":"quarter\r\u001b[2J","future":"keep\u0007value"}` + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + got := runSpendThresholdCommand(t, server, spendThresholdArgs(false, "spend-limit", "retrieve")...) + if got.code != 0 || got.stderr != "" || strings.ContainsAny(got.stdout, "\x1b\r\a") || !strings.Contains(got.stdout, "123 cents") || !strings.Contains(got.stdout, "keep") { + t.Fatalf("readable spend threshold emitted controls or lost content: %+v", got) + } + for _, resource := range []string{"spend-limit", "spend-alerts"} { + got := runSpendThresholdCommand(t, server, spendThresholdArgs(false, resource, "delete")...) + if got.code != 0 || got.stderr != "" || strings.Contains(got.stdout, "Spend threshold:") || strings.Contains(got.stdout, "Alert behavior:") || strings.Contains(got.stdout, "Enforcement:") || !strings.Contains(got.stdout, "Threshold amount: 123") { + t.Fatalf("delete response incorrectly projected: %+v", got) + } + } +} + +func TestMainSpendThresholdLargeUnknownField(t *testing.T) { + // Keep a complete unknown field larger than common line and buffer limits. + value := strings.Repeat("synthetic", 1<<20) + body := spendThresholdBody(false, "spend-limit", `{"threshold_amount":10000,"currency":"USD","interval":"month","future":"`+value+`"}`) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + args := spendThresholdArgs(false, "spend-limit", "retrieve") + got := runSpendThresholdCommand(t, server, args...) + if got.code != 0 || got.stderr != "" || !strings.Contains(got.stdout, "Spend threshold: USD 100.00 per month") || !strings.Contains(got.stdout, "Future: "+value+"\n") { + t.Fatalf("large readable response lost data: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } + got = runSpendThresholdCommand(t, server, append([]string{"--format", "raw"}, args...)...) + if got.code != 0 || got.stderr != "" || got.stdout != body+"\n" { + t.Fatalf("large raw response changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } + got = runSpendThresholdCommand(t, server, append([]string{"--transform", "future", "--raw-output"}, args...)...) + if got.code != 0 || got.stderr != "" || got.stdout != value+"\n" { + t.Fatalf("large extracted field changed: code=%d stderr=%q output bytes=%d", got.code, got.stderr, len(got.stdout)) + } +} + +func TestMainSpendThresholdDuplicateKeys(t *testing.T) { + for _, tc := range []struct { + name, resource string + project bool + fields, text string + }{ + { + name: "threshold amount", resource: "spend-limit", + fields: `{"threshold_amount":10000,"threshold_amount":90000,"currency":"USD","interval":"month"}`, + text: "Threshold amount: 10000\nThreshold amount: 90000\nCurrency: USD\nInterval: month\n", + }, + { + name: "currency", resource: "spend-limit", project: true, + fields: `{"threshold_amount":10000,"currency":"USD","currency":"EUR","interval":"month"}`, + text: "Threshold amount: 10000\nCurrency: USD\nCurrency: EUR\nInterval: month\n", + }, + { + name: "interval", resource: "spend-alerts", + fields: `{"threshold_amount":10000,"currency":"USD","interval":"month","interval":"year"}`, + text: "Threshold amount: 10000\nCurrency: USD\nInterval: month\nInterval: year\n", + }, + { + name: "enforcement", resource: "spend-limit", project: true, + fields: `{"threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"inactive"},"enforcement":{"status":"enforcing"}}`, + text: "Threshold amount: 10000\nCurrency: USD\nInterval: month\nEnforcement:\n Status: inactive\nEnforcement:\n Status: enforcing\n", + }, + { + name: "object", resource: "spend-alerts", project: true, + fields: `{"object":"future.spend_alert","threshold_amount":10000,"currency":"USD","interval":"month"}`, + text: "Object: future.spend_alert\nThreshold amount: 10000\nCurrency: USD\nInterval: month\n", + }, + { + name: "escaped threshold key", resource: "spend-alerts", project: true, + fields: `{"threshold_amount":10000,"threshold_\u0061mount":90000,"currency":"USD","interval":"month"}`, + text: "Threshold amount: 10000\nThreshold amount: 90000\nCurrency: USD\nInterval: month\n", + }, + { + name: "unknown field", resource: "spend-alerts", + fields: `{"threshold_amount":10000,"currency":"USD","interval":"month","future":"first","future":"second"}`, + text: "Threshold amount: 10000\nCurrency: USD\nInterval: month\nFuture: first\nFuture: second\n", + }, + } { + t.Run(tc.name, func(t *testing.T) { + body := spendThresholdBody(tc.project, tc.resource, tc.fields) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + path := spendThresholdPath(tc.project, tc.resource) + if tc.resource == "spend-alerts" { + path += "/alert_synthetic" + } + if r.Method != http.MethodGet || r.URL.Path != path { + t.Errorf("unexpected duplicate-key request: %s %s", r.Method, r.URL.Path) + } + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, body) + })) + defer server.Close() + scope := "organization" + if tc.project { + scope = "project" + } + object := scope + "." + strings.TrimSuffix(strings.ReplaceAll(tc.resource, "-", "_"), "s") + for _, format := range []string{"", "text", "raw"} { + t.Run("format="+format, func(t *testing.T) { + args := spendThresholdArgs(tc.project, tc.resource, "retrieve") + if format != "" { + args = append([]string{"--format", format}, args...) + } + got := runSpendThresholdCommand(t, server, args...) + // Compare text and bytes directly: decoding into maps discards duplicates. + want := "Object: " + object + "\n" + tc.text + if format == "raw" { + want = body + "\n" + } + if got.code != 0 || got.stderr != "" || got.stdout != want { + t.Fatalf("duplicate fields changed: got=%+v; want stdout=%q", got, want) + } + for _, synthesized := range []string{"Spend threshold:", "spend_threshold", "not reported in this response", "Alert behavior:", "alert_behavior"} { + if strings.Contains(got.stdout, synthesized) { + t.Errorf("ambiguous response synthesized %q: %q", synthesized, got.stdout) + } + } + }) + } + }) + } +} diff --git a/docs/limits-workspace.md b/docs/limits-workspace.md new file mode 100644 index 00000000..eddd89bb --- /dev/null +++ b/docs/limits-workspace.md @@ -0,0 +1,160 @@ +# Limits in the terminal + +The limits workspace manages the controls exposed by the public administration API. +It requires an admin API key through the existing CLI authentication settings. + +```sh +openai admin limits +openai projects limits --project-id proj_example +``` + +An interactive terminal opens the workspace. +Pipes, CI, quiet mode, explicit output formats, and extraction print a read-only overview. +The `show` command always prints the overview. + +```sh +openai admin limits show +openai projects limits show --project-id proj_example --format json +openai admin limits show --max-items -1 +openai admin limits show --format json --transform spend_alerts +``` + +`--max-items` defaults to 100 records per list. +Use `-1` for all records or `0` to validate the list request without retaining records. +The interactive workspace puts each list's Complete or Incomplete status before its loaded count. +Failed reads preserve available data and return a nonzero exit status. +Cancellation can stop output before the CLI prints those records. +The `source_errors` field identifies failed sources without exposing response error bodies. +Machine formats and extraction retain original resource fields and numeric values. + +## Current spend + +The overview includes reported current-month Costs with UTC daily boundaries. +Today’s costs remain partial and can arrive later. +Costs amounts use currency units; spend-limit amounts use integer cents. +The CLI preserves exact decimal totals and original buckets. +Incomplete Costs totals cover only returned buckets. +Check `costs.complete` before treating a total as the complete reported period. +Ambiguous Costs data produces no total. +Costs data cannot establish authoritative enforcement state. +The derived summary supports up to 10,000 decimal digits. +Values beyond that precision remain available in `costs.buckets`; the summary reports its limitation. + +## Workspace controls + +| Control | Action | +| --- | --- | +| Left / Right | Select Spend, Alerts, or project Rates | +| Up / Down | Select a record or scroll a review | +| Enter | Open the selected action | +| Tab / Shift+Tab | Move through form fields or confirmation actions | +| r | Refresh the overview or a rejected draft | +| i | Inspect the selected original API record | +| ? | Show workspace help and API boundaries | +| F1 | Show help, including complete form diagnostics | +| d | Delete the selected alert | +| Escape | Return without saving | +| q | Close the overview | +| Ctrl+P | Print the reviewed command and exit without saving | +| Ctrl+C | Interrupt from any state | + +The workspace needs at least 40 columns and 12 rows. +Smaller terminals retain cancellation and block submission. +The workspace restores the terminal screen when it exits. +Escape closes help and restores the active form or review. +Forms retain their fields, focus, and subject-prefix mode. +Repeated F1 presses preserve that return point. +Escape from a review returns to the form, like Back. + +Every change opens a before-and-after review. +Spend and alert reviews show exact monthly USD amounts alongside the original API fields. +Scroll to the end before selecting Apply. +Back is the initial selection, including after resizing. +Returning from help also selects Back. +Resizing during help requires rereading the review before selecting Apply. +The review shows a shell command when the CLI can reproduce the request context safely. +Wrapped command rows are for inspection only. +Press Ctrl+P to print the complete command after the terminal screen restores. +Copy the printed command when you want to run it later. +Printing sends no change and preserves any earlier confirmed receipt. +The workspace omits command previews that require private options or unsupported shells. +Existing administration commands remain available for scripts. + +```sh +# Existing scriptable commands use integer cents. +openai admin spend-limit update --currency USD --interval month --threshold-amount 12550 +openai admin spend-alerts list --max-items -1 +openai projects rate-limits list-rate-limits --project-id proj_example +openai admin usage costs --help +``` + +## Supported changes + +Spend supports setting, replacing, and removing hard monthly USD limits. +Enter decimal dollars, such as `125.50`. +The CLI converts dollars to integer cents without floating-point arithmetic. +A hard limit must be at least USD 0.01. +Enforcement can be delayed after the threshold is reached. +Removing a hard limit removes that limit’s enforcement. +Parent limits can still apply. +Use the Platform website to inspect or edit any retained soft budget. + +Alerts support creation, editing, and deletion. +Alerts notify recipients; they do not enforce a cap. +Enter recipients as comma-separated email addresses. +Subject prefixes can preserve the original value, replace it, or clear it to null. +An empty replacement remains an empty string. +The workspace does not require a hard limit before creating an alert. +The API still validates account-specific constraints. + +Project Rates supports all six public rate fields. +The form sends only changed fields. +Blank fields remain unchanged. +Zero has no assumed unlimited meaning. +The API decides which integer values the project permits. +Rate updates require a complete preflight list; use `--max-items -1` when necessary. + +## Concurrent changes and recovery + +The CLI checks the selected record again before writing. +This check is not atomic and cannot prevent every concurrent change. +A changed baseline stops the write. +A 404 response leaves the current limit unconfirmed. +Setting a limit requires reviewing that uncertainty and explicitly approving creation or replacement. +Refresh the retained draft before reviewing it again. +Ambiguous or unfamiliar replacement configuration remains available for inspection. +The form refuses edits that could discard that configuration. + +The workspace never automatically retries a mutation. +Scriptable API commands retain their existing retry behavior. +A command preview does not reproduce workspace preflight checks or retry suppression. +An uncertain save locks further writes in that workspace session. +Inspect the resource before reopening the workspace or repeating the action. +A refresh alone cannot prove that an uncertain alert creation failed. +Cancellation does not prove that the server canceled a dispatched write. + +## Fine-tuning model limits + +The new `openai fine-tuning model-limits` command reads the documented fine-tuning model limits. +Use an ordinary `OPENAI_API_KEY`; this command does not use admin-key fallback. +Root organization, project, connection, and output options remain available. +The command reads one endpoint and preserves all response fields, including unknown fields. +It adds no pagination or mutation flags. + +```sh +openai fine-tuning model-limits --format json +``` + +See the [fine-tuning rate-limits documentation](https://developers.openai.com/api/docs/guides/rate-limits#fine-tuning-rate-limits). + +## Platform controls without matching contracts + +The [Platform limits page](https://platform.openai.com/settings/organization/limits) includes controls beyond these public operations. +The inspected public contract does not expose the website's soft-budget and enforcement-toggle combination. +It also lacks the organization tier catalog, account upgrade operations, and approved usage-limit controls. +The workspace does not emulate these controls through undocumented requests. + +Project rate records do not describe every organization-wide model limit or shared model-family limit. +Historical costs are not a live allowance or enforcement counter. +See the [spend-limits guide](https://developers.openai.com/api/docs/guides/spend-limits) +and [rate-limits guide](https://developers.openai.com/api/docs/guides/rate-limits) for API behavior. diff --git a/docs/spend-thresholds.md b/docs/spend-thresholds.md new file mode 100644 index 00000000..86571941 --- /dev/null +++ b/docs/spend-thresholds.md @@ -0,0 +1,77 @@ +# Spend thresholds + +Spend-limit and spend-alert amounts use cents. Readable output shows integer USD amounts as dollars and cents. + +```sh +# Read the organization threshold. +openai admin organization spend-limit retrieve + +# Read a project threshold. +openai admin organization projects spend-limit retrieve --project-id proj_example + +# List notification thresholds. +openai admin organization spend-alerts list +``` + +A response containing `threshold_amount: 10000`, `currency: "USD"`, and `interval: "month"` displays: + +```text +Spend threshold: USD 100.00 per month +``` + +The CLI preserves reported enforcement fields. When enforcement is absent or null, it displays: + +```text +Enforcement: not reported in this response +``` + +Spend alerts notify recipients. They do not enforce spending caps or stop API requests. +Each readable alert includes this distinction, including alerts returned across multiple pages. +Notification channels, recipients, identifiers, and unfamiliar fields remain visible. + +## Preserved data and inputs + +The projection applies to default readable output and `--format text`. +Organization and project responses share these rules. +Successful create/update responses use the same presentation as retrieve/list responses. +Delete confirmations remain unchanged. + +```sh +# Preserve the original numeric cents and response fields. +openai admin organization spend-limit retrieve --format json + +# Extract the original amount. +openai admin organization spend-limit retrieve --transform threshold_amount --raw-output + +# Preserve existing pagination controls. +openai admin organization spend-alerts list --max-items -1 + +# Read alert records as one JSON array. +openai admin organization spend-alerts list --format json +``` + +Explicit JSON, JSONL, YAML, raw, pretty, and explore formats retain their existing data behavior. +JSON alert lists use one array across pages. JSONL emits separate records; raw format preserves the response page. +Extraction and `--raw-output` bypass the projection. +The CLI does not change errors, exit status, or pagination. + +Unknown currencies retain the original amount in cents without assuming a currency conversion. +Unknown intervals remain literal values. Missing currency or interval receives a `not reported` label. +The CLI preserves exact large integers without floating-point conversion. +Unexpected decimal or exponent values retain their original numeric spelling in cents. +Responses with duplicate top-level keys retain their original readable fields instead of a summary. +This avoids assigning ambiguous currency, interval, or enforcement information to a threshold. + +Existing update/create flags still accept cents. This change does not convert inputs or change validation. +Limit inputs require at least one cent in the API schema; alert inputs allow zero. +Response edge-case handling does not establish new accepted mutation inputs. + +## Scope + +The bundled API contract establishes USD, monthly intervals, and reported limit enforcement states. +The CLI does not infer an enforcement state from the threshold amount. +This presentation change adds no billing action, policy API, usage-tier discovery, or Costs API conversion. +Costs API amounts follow a separate contract. + +See the [organization spend-limit reference](https://developers.openai.com/api/reference/resources/admin/subresources/organization/subresources/spend_limit/methods/retrieve) +and [spend-alert guidance](https://developers.openai.com/api/docs/guides/terraform/rate-limits-and-spend). diff --git a/internal/output/transform/project_limits_overview.go b/internal/output/transform/project_limits_overview.go new file mode 100644 index 00000000..1d0ee0be --- /dev/null +++ b/internal/output/transform/project_limits_overview.go @@ -0,0 +1,166 @@ +package transform + +import ( + "context" + "strings" + + "github.com/tidwall/gjson" +) + +// ProjectLimitsOverview adds units to recognized records in a limits snapshot. +// The caller selects readable output; machine formats keep the original snapshot. +func ProjectLimitsOverview(ctx context.Context, value gjson.Result) (gjson.Result, error) { + unique, err := limitsOverviewUniqueObject(ctx, value) + if err != nil || !unique { + return value, err + } + scope := value.Get("scope") + if scope.Type != gjson.String || scope.Str != "organization" && scope.Str != "project" { + return value, nil + } + var out strings.Builder + out.WriteByte('{') + first := true + value.ForEach(func(key, field gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + projected := field + switch key.Str { + case "spend_limit": + projected, err = projectSpendThreshold(ctx, field, scope.Str+".spend_limit", false) + case "spend_alerts": + projected, err = projectLimitsOverviewArray(ctx, field, func(item gjson.Result) (gjson.Result, error) { + return projectSpendThreshold(ctx, item, scope.Str+".spend_alert", true) + }) + case "project_rate_limits": + if scope.Str == "project" { + projected, err = projectLimitsOverviewArray(ctx, field, func(item gjson.Result) (gjson.Result, error) { + return projectLimitsOverviewRate(ctx, item) + }) + } + } + if err != nil { + return false + } + if !first { + out.WriteByte(',') + } + first = false + out.WriteString(key.Raw) + out.WriteByte(':') + out.WriteString(projected.Raw) + return true + }) + if err != nil { + return gjson.Result{}, err + } + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + out.WriteByte('}') + return gjson.Parse(out.String()), nil +} + +func projectLimitsOverviewArray(ctx context.Context, value gjson.Result, project func(gjson.Result) (gjson.Result, error)) (gjson.Result, error) { + if !value.IsArray() { + return value, ctx.Err() + } + var out strings.Builder + out.WriteByte('[') + first := true + var err error + value.ForEach(func(_, item gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + var projected gjson.Result + projected, err = project(item) + if err != nil { + return false + } + if !first { + out.WriteByte(',') + } + first = false + out.WriteString(projected.Raw) + return true + }) + if err != nil { + return gjson.Result{}, err + } + out.WriteByte(']') + return gjson.Parse(out.String()), ctx.Err() +} + +var limitsOverviewRateLabels = map[string]string{ + "max_requests_per_1_minute": "max_requests_per_minute", + "max_tokens_per_1_minute": "max_tokens_per_minute", + "max_images_per_1_minute": "max_images_per_minute", + "max_audio_megabytes_per_1_minute": "max_audio_megabytes_per_minute", + "max_requests_per_1_day": "max_requests_per_day", + "batch_1_day_max_input_tokens": "max_batch_input_tokens_per_day", +} + +func projectLimitsOverviewRate(ctx context.Context, value gjson.Result) (gjson.Result, error) { + unique, err := limitsOverviewUniqueObject(ctx, value) + if err != nil || !unique || value.Get("object").Str != "project.rate_limit" { + return value, err + } + // A future response field must never collide with a derived readable label. + for source, label := range limitsOverviewRateLabels { + if value.Get(source).Type == gjson.Number && value.Get(label).Exists() { + return value, nil + } + } + var out strings.Builder + out.WriteByte('{') + first := true + value.ForEach(func(key, field gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + if !first { + out.WriteByte(',') + } + first = false + if label, ok := limitsOverviewRateLabels[key.Str]; ok && field.Type == gjson.Number { + out.WriteByte('"') + out.WriteString(label) + out.WriteByte('"') + } else { + out.WriteString(key.Raw) + } + out.WriteByte(':') + out.WriteString(field.Raw) + return true + }) + if err != nil { + return gjson.Result{}, err + } + out.WriteByte('}') + return gjson.Parse(out.String()), ctx.Err() +} + +func limitsOverviewUniqueObject(ctx context.Context, value gjson.Result) (bool, error) { + if err := ctx.Err(); err != nil { + return false, err + } + if !value.IsObject() { + return false, nil + } + seen := make(map[string]bool) + unique := true + value.ForEach(func(key, _ gjson.Result) bool { + if ctx.Err() != nil { + return false + } + if seen[key.Str] { + unique = false + return false + } + seen[key.Str] = true + return true + }) + return unique, ctx.Err() +} diff --git a/internal/output/transform/project_limits_overview_test.go b/internal/output/transform/project_limits_overview_test.go new file mode 100644 index 00000000..cadc2d95 --- /dev/null +++ b/internal/output/transform/project_limits_overview_test.go @@ -0,0 +1,127 @@ +package transform + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/tidwall/gjson" +) + +func TestProjectLimitsOverviewSpendScopes(t *testing.T) { + for _, scope := range []string{"organization", "project"} { + t.Run(scope, func(t *testing.T) { + raw := `{"scope":"` + scope + `","spend_limit":{"object":"` + scope + `.spend_limit","threshold_amount":9007199254740993,"currency":"USD","interval":"month","enforcement":{"status":"future_status"},"unknown":0.100000000000000001},"spend_limit_configured":true,"spend_alerts":[{"object":"` + scope + `.spend_alert","id":"alert_synthetic","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"recipients":["synthetic@example.invalid"],"subject_prefix":null}}],"spend_alerts_complete":false,"source_errors":[],"future":{"value":9007199254740993123456789}}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || !gjson.Valid(value.Raw) { + t.Fatalf("invalid overview: %s; %v", value.Raw, err) + } + if value.Get("spend_limit.spend_threshold").Str != "USD 90071992547409.93 per month" || value.Get("spend_alerts.0.spend_threshold").Str != "USD 0.00 per month" { + t.Fatalf("incorrect spend units: %s", value.Raw) + } + if value.Get("spend_alerts.0.alert_behavior").Str != "Alerts notify; they are not spending caps." { + t.Fatalf("missing alert behavior: %s", value.Raw) + } + for _, field := range []string{"scope", "spend_limit_configured", "spend_alerts_complete", "source_errors", "future", "spend_limit.enforcement", "spend_limit.unknown", "spend_alerts.0.notification_channel"} { + if got, want := value.Get(field).Raw, gjson.Get(raw, field).Raw; got != want { + t.Errorf("%s changed: %q; want %q", field, got, want) + } + } + }) + } +} + +func TestProjectLimitsOverviewExactRateUnits(t *testing.T) { + const rate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":0,"max_tokens_per_1_minute":9007199254740993123456789,"max_images_per_1_minute":-1,"max_audio_megabytes_per_1_minute":0.1234567890123456789,"max_requests_per_1_day":1e9,"batch_1_day_max_input_tokens":9223372036854775807,"future":{"count":9007199254740993}}` + raw := `{"scope":"project","project_id":"proj_synthetic","project":{"id":"proj_synthetic","future":1e100},"spend_limit":null,"spend_limit_configured":false,"spend_alerts":[],"spend_alerts_complete":true,"project_rate_limits":[` + rate + `],"project_rate_limits_complete":true,"source_errors":[]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil { + t.Fatal(err) + } + projected := value.Get("project_rate_limits.0") + for label, want := range map[string]string{ + "max_requests_per_minute": "0", "max_tokens_per_minute": "9007199254740993123456789", + "max_images_per_minute": "-1", "max_audio_megabytes_per_minute": "0.1234567890123456789", + "max_requests_per_day": "1e9", "max_batch_input_tokens_per_day": "9223372036854775807", + } { + if projected.Get(label).Raw != want { + t.Errorf("rate %s changed units or precision: %s", label, projected.Raw) + } + } + for _, field := range []string{"object", "id", "model", "future"} { + if projected.Get(field).Raw != gjson.Get(rate, field).Raw { + t.Errorf("rate field %s changed: %s", field, projected.Raw) + } + } + for _, field := range []string{"project_id", "project", "spend_limit", "spend_limit_configured", "spend_alerts", "spend_alerts_complete", "project_rate_limits_complete", "source_errors"} { + if value.Get(field).Raw != gjson.Get(raw, field).Raw { + t.Errorf("overview field %s changed: %s", field, value.Raw) + } + } +} + +func TestProjectLimitsOverviewRatePresenceAndAmbiguity(t *testing.T) { + for _, raw := range []string{ + `{"object":"project.rate_limit","id":"rl_synthetic"}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":null,"max_tokens_per_1_minute":"100"}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_minute":null}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_1_minute":900}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"max_requests_per_1_\u006dinute":900}`, + `{"object":"project.rate_limit","max_requests_per_1_minute":100,"future":1,"future":2}`, + `{"object":"future.rate_limit","max_requests_per_1_minute":100}`, + `{"max_requests_per_1_minute":100}`, + `null`, `[]`, `"future"`, + } { + t.Run(raw, func(t *testing.T) { + value, err := projectLimitsOverviewRate(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("ambiguous or unknown rate changed: %s; %v", value.Raw, err) + } + }) + } +} + +func TestProjectLimitsOverviewUnrecognizedEnvelope(t *testing.T) { + for _, raw := range []string{ + `null`, `[]`, `{"scope":"future","spend_limit":{}}`, + `{"scope":"project","scope":"organization","spend_limit":{}}`, + `{"scope":"project","future":1,"future":2}`, + `{"scope":"project","spend_alerts":[],"spend_\u0061lerts":null}`, + `{"scope":null}`, `{"scope":{}}`, + } { + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("unrecognized envelope changed: %s; %v", value.Raw, err) + } + } + const raw = `{"scope":"organization","spend_limit":null,"spend_limit_configured":null,"spend_alerts":null,"project_rate_limits":[{"object":"project.rate_limit","max_requests_per_1_minute":7}],"source_errors":[{"resource":"spend_limit","status":403}]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("absent data or project-only rate changed: %s; %v", value.Raw, err) + } +} + +func TestProjectLimitsOverviewKeepsAmbiguousSpendRecords(t *testing.T) { + const limit = `{"object":"project.spend_limit","threshold_amount":10000,"threshold_amount":90000,"currency":"USD","interval":"month"}` + const alert = `{"object":"project.spend_alert","threshold_amount":10000,"currency":"USD","currency":"EUR","interval":"month"}` + raw := `{"scope":"project","spend_limit":` + limit + `,"spend_alerts":[` + alert + `,null,{"object":"future.spend_alert","threshold_amount":10000}]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Raw != raw { + t.Fatalf("ambiguous spend or unknown alert changed: %s; %v", value.Raw, err) + } +} + +func TestProjectLimitsOverviewCancellationAndLargeUnknownField(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := ProjectLimitsOverview(ctx, gjson.Parse(`{"scope":"project"}`)); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + unknown := strings.Repeat("x", 1<<20) + raw := `{"scope":"project","project_rate_limits":[{"object":"project.rate_limit","max_requests_per_1_minute":1,"future":"` + unknown + `"}]}` + value, err := ProjectLimitsOverview(t.Context(), gjson.Parse(raw)) + if err != nil || value.Get("project_rate_limits.0.future").Str != unknown || value.Get("project_rate_limits.0.max_requests_per_minute").Raw != "1" { + t.Fatalf("large unknown field lost: length=%d, error=%v", len(value.Raw), err) + } +} diff --git a/internal/output/transform/project_spend_threshold.go b/internal/output/transform/project_spend_threshold.go new file mode 100644 index 00000000..e64b608b --- /dev/null +++ b/internal/output/transform/project_spend_threshold.go @@ -0,0 +1,183 @@ +package transform + +import ( + "context" + "encoding/json" + "strings" + + "github.com/tidwall/gjson" +) + +// Spend thresholds use cents. Costs API amounts have a separate contract. +func selectSpendThresholdTransformer(route Route) Transformer { + var object string + alert := false + switch route { + case Route{"(resource) admin.organization.spend_limit > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.spend_limit > (method) update", OutputResponse}: + object = "organization.spend_limit" + case Route{"(resource) admin.organization.projects.spend_limit > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_limit > (method) update", OutputResponse}: + object = "project.spend_limit" + case Route{"(resource) admin.organization.spend_alerts > (method) create", OutputResponse}, + Route{"(resource) admin.organization.spend_alerts > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.spend_alerts > (method) update", OutputResponse}, + Route{"(resource) admin.organization.spend_alerts > (method) list", OutputPageItem}: + object, alert = "organization.spend_alert", true + case Route{"(resource) admin.organization.projects.spend_alerts > (method) create", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_alerts > (method) retrieve", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_alerts > (method) update", OutputResponse}, + Route{"(resource) admin.organization.projects.spend_alerts > (method) list", OutputPageItem}: + object, alert = "project.spend_alert", true + default: + return nil + } + return func(ctx context.Context, value gjson.Result) (gjson.Result, error) { + return projectSpendThreshold(ctx, value, object, alert) + } +} + +func projectSpendThreshold(ctx context.Context, value gjson.Result, object string, alert bool) (gjson.Result, error) { + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + if !value.IsObject() || value.Get("object").String() != object { + return value, nil + } + // Get selects the first occurrence, while ForEach presents every occurrence. + // Preserve ambiguous responses instead of attaching one field's units/value + // to another field with the same name, including escaped key spellings. + seen := make(map[string]bool) + duplicate := false + value.ForEach(func(key, _ gjson.Result) bool { + if ctx.Err() != nil { + return false + } + duplicate = seen[key.Str] + seen[key.Str] = true + return !duplicate + }) + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + if duplicate { + return value, nil + } + amount := value.Get("threshold_amount") + summary := "" + var summaryJSON []byte + // Do not overwrite an unfamiliar response field with a derived field. + if amount.Type == gjson.Number && !value.Get("spend_threshold").Exists() { + var err error + summary, err = spendThresholdText(ctx, amount.Raw, value.Get("currency"), value.Get("interval")) + if err != nil { + return gjson.Result{}, err + } + summaryJSON, err = json.Marshal(summary) + if err != nil { + return gjson.Result{}, err + } + } + var out strings.Builder + out.WriteByte('{') + first := true + write := func(key, raw string) { + if !first { + out.WriteByte(',') + } + first = false + out.WriteString(key) + out.WriteByte(':') + out.WriteString(raw) + } + value.ForEach(func(key, field gjson.Result) bool { + if ctx.Err() != nil { + return false + } + switch { + case key.Str == "threshold_amount" && summary != "": + write(`"spend_threshold"`, string(summaryJSON)) + case key.Str == "enforcement" && !alert && field.Type == gjson.Null: + write(key.Raw, `"not reported in this response"`) + default: + // Preserve unknown fields and number spellings without decoding floats. + write(key.Raw, field.Raw) + } + return true + }) + if alert { + if !value.Get("alert_behavior").Exists() { + write(`"alert_behavior"`, `"Alerts notify; they are not spending caps."`) + } + } else if !value.Get("enforcement").Exists() { + write(`"enforcement"`, `"not reported in this response"`) + } + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + out.WriteByte('}') + return gjson.Parse(out.String()), nil +} + +func spendThresholdText(ctx context.Context, amount string, currency, interval gjson.Result) (string, error) { + text := amount + " cents" + if currency.Type == gjson.String && currency.Str == "USD" { + decimal, err := spendCentsDecimal(ctx, amount) + if err != nil { + return "", err + } + if decimal != "" { + text = "USD " + decimal + } else { + text += " (currency: USD)" + } + } else { + text += " (currency: " + spendReportedUnit(currency) + ")" + } + if interval.Type == gjson.String && interval.Str == "month" { + text += " per month" + } else { + text += " (interval: " + spendReportedUnit(interval) + ")" + } + return text, ctx.Err() +} + +func spendReportedUnit(value gjson.Result) string { + if value.Type == gjson.Null || value.Type == gjson.String && value.Str == "" { + return "not reported" + } + if value.Type == gjson.String { + return value.Str + } + return "see response field" +} + +// Insert a decimal point in integer cents without float conversion or bounds. +// Unexpected fractions and exponents retain their original spelling in cents. +func spendCentsDecimal(ctx context.Context, amount string) (string, error) { + sign := "" + if strings.HasPrefix(amount, "-") { + sign, amount = "-", amount[1:] + } + if amount == "" { + return "", nil + } + for i := 0; i < len(amount); i++ { + if i%32768 == 0 { + if err := ctx.Err(); err != nil { + return "", err + } + } + if amount[i] < '0' || amount[i] > '9' { + return "", nil + } + } + switch len(amount) { + case 1: + return sign + "0.0" + amount, nil + case 2: + return sign + "0." + amount, nil + default: + return sign + amount[:len(amount)-2] + "." + amount[len(amount)-2:], nil + } +} diff --git a/internal/output/transform/project_spend_threshold_test.go b/internal/output/transform/project_spend_threshold_test.go new file mode 100644 index 00000000..db7b5cb5 --- /dev/null +++ b/internal/output/transform/project_spend_threshold_test.go @@ -0,0 +1,190 @@ +package transform + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/tidwall/gjson" +) + +func TestSpendThresholdExactUnits(t *testing.T) { + for _, tc := range []struct{ amount, currency, interval, want string }{ + {"10000", `"USD"`, `"month"`, "USD 100.00 per month"}, + {"0", `"USD"`, `"month"`, "USD 0.00 per month"}, + {"1", `"USD"`, `"month"`, "USD 0.01 per month"}, + {"99", `"USD"`, `"month"`, "USD 0.99 per month"}, + {"-1", `"USD"`, `"month"`, "USD -0.01 per month"}, + {"-10000", `"USD"`, `"month"`, "USD -100.00 per month"}, + {"900719925474099312345678901234567890", `"USD"`, `"month"`, "USD 9007199254740993123456789012345678.90 per month"}, + {"10000", `"EUR"`, `"month"`, "10000 cents (currency: EUR) per month"}, + {"10000", `"USD"`, `"fortnight"`, "USD 100.00 (interval: fortnight)"}, + {"10000", `null`, `null`, "10000 cents (currency: not reported) (interval: not reported)"}, + {"10000", `""`, `""`, "10000 cents (currency: not reported) (interval: not reported)"}, + {"10000", `{}`, `false`, "10000 cents (currency: see response field) (interval: see response field)"}, + {"0.1234567890123456789", `"USD"`, `"month"`, "0.1234567890123456789 cents (currency: USD) per month"}, + {"1e100", `"USD"`, `"month"`, "1e100 cents (currency: USD) per month"}, + } { + t.Run(tc.amount+tc.currency+tc.interval, func(t *testing.T) { + got, err := spendThresholdText(t.Context(), tc.amount, gjson.Parse(tc.currency), gjson.Parse(tc.interval)) + if err != nil || got != tc.want { + t.Fatalf("got %q, %v; want %q", got, err, tc.want) + } + }) + } +} + +func TestSpendThresholdPreservesResponseFields(t *testing.T) { + const raw = `{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"future","extra":1.234567890123456789},"future":{"integer":9007199254740993,"decimal":0.000000000000000001}}` + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil { + t.Fatal(err) + } + if value.Get("spend_threshold").Str != "USD 100.00 per month" || value.Get("threshold_amount").Exists() { + t.Fatalf("threshold projection: %s", value.Raw) + } + for _, field := range []string{"object", "currency", "interval", "enforcement", "future"} { + if got, want := value.Get(field).Raw, gjson.Get(raw, field).Raw; got != want { + t.Errorf("%s changed: %s; want %s", field, got, want) + } + } +} + +func TestSpendThresholdSummaryUsesJSONEscapes(t *testing.T) { + const raw = `{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD\u001b[31m","interval":"\u0007\u000b\u0000"}` + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil || !gjson.Valid(value.Raw) { + t.Fatalf("control characters broke JSON projection: %q, %v", value.Raw, err) + } + want := "10000 cents (currency: USD\x1b[31m) (interval: \a\v\x00)" + if value.Get("spend_threshold").Str != want { + t.Fatalf("controls changed before renderer: %q", value.Get("spend_threshold").Str) + } +} + +func TestSpendThresholdMissingFieldsAndCollisions(t *testing.T) { + for _, raw := range []string{ + `{"object":"organization.spend_limit"}`, + `{"object":"organization.spend_limit","threshold_amount":null,"enforcement":null}`, + `{"object":"organization.spend_limit","threshold_amount":"10000"}`, + `{"object":"organization.spend_limit","threshold_amount":10000,"spend_threshold":"future field"}`, + } { + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil || !gjson.Valid(value.Raw) { + t.Fatalf("invalid projection: %s, %v", value.Raw, err) + } + if value.Get("enforcement").Str != "not reported in this response" { + t.Fatalf("missing enforcement: %s", value.Raw) + } + if value.Get("threshold_amount").Raw != gjson.Get(raw, "threshold_amount").Raw { + t.Fatalf("unexpected amount rewritten: %s", value.Raw) + } + if value.Get("spend_threshold").Raw != gjson.Get(raw, "spend_threshold").Raw { + t.Fatalf("collision overwritten: %s", value.Raw) + } + } + const alert = `{"object":"organization.spend_alert","threshold_amount":0,"alert_behavior":{"future":"kept"}}` + value, err := projectSpendThreshold(t.Context(), gjson.Parse(alert), "organization.spend_alert", true) + if err != nil || value.Get("alert_behavior").Raw != `{"future":"kept"}` || value.Get("enforcement").Exists() { + t.Fatalf("alert collision or invented enforcement: %s, %v", value.Raw, err) + } +} + +func TestSpendThresholdRouteBoundaries(t *testing.T) { + for _, resource := range []string{"admin.organization", "admin.organization.projects"} { + for _, tc := range []struct { + resource, method string + kind OutputKind + match bool + }{ + {"spend_limit", "retrieve", OutputResponse, true}, + {"spend_limit", "update", OutputResponse, true}, + {"spend_limit", "delete", OutputResponse, false}, + {"spend_alerts", "create", OutputResponse, true}, + {"spend_alerts", "retrieve", OutputResponse, true}, + {"spend_alerts", "update", OutputResponse, true}, + {"spend_alerts", "list", OutputPageItem, true}, + {"spend_alerts", "list", OutputResponse, false}, + {"spend_alerts", "list", OutputStreamEvent, false}, + {"spend_alerts", "delete", OutputResponse, false}, + {"spend_limit", "retrieve", OutputUnspecified, false}, + {"costs", "list", OutputPageItem, false}, + } { + route := Route{"(resource) " + resource + "." + tc.resource + " > (method) " + tc.method, tc.kind} + if got := selectSpendThresholdTransformer(route); (got != nil) != tc.match { + t.Errorf("route %+v selected=%t, want %t", route, got != nil, tc.match) + } + } + } + for _, raw := range []string{`null`, `[]`, `{"object":"project.spend_limit","threshold_amount":10000}`, `{"threshold_amount":10000}`} { + value, err := projectSpendThreshold(t.Context(), gjson.Parse(raw), "organization.spend_limit", false) + if err != nil || value.Raw != raw { + t.Fatalf("unrecognized object changed: %s, %v", value.Raw, err) + } + } +} + +func TestSpendThresholdCancellationAndLargeValues(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := projectSpendThreshold(ctx, gjson.Parse(`{}`), "organization.spend_limit", false); !errors.Is(err, context.Canceled) { + t.Fatalf("cancellation lost: %v", err) + } + if _, err := spendCentsDecimal(ctx, strings.Repeat("9", 100000)); !errors.Is(err, context.Canceled) { + t.Fatalf("amount cancellation lost: %v", err) + } + // This exceeds machine integers without adding a presentation rejection limit. + amount := strings.Repeat("9", 100000) + got, err := spendCentsDecimal(t.Context(), amount) + if err != nil || got != amount[:len(amount)-2]+".99" { + t.Fatalf("large exact amount changed: length=%d, error=%v", len(got), err) + } +} + +func TestSpendThresholdCancelsDuringScan(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + // Entry and duplicate-key checks consume six polls before amount scanning. + controlled := &cancelSummaryContext{Context: ctx, cancel: cancel, after: 9} + raw := `{"object":"organization.spend_limit","threshold_amount":` + strings.Repeat("9", 200000) + `,"currency":"USD","interval":"month"}` + transform := Select(Route{"(resource) admin.organization.spend_limit > (method) retrieve", OutputResponse}) + got, err := transform(controlled, gjson.Parse(raw)) + if !errors.Is(err, context.Canceled) || got.Raw != "" { + t.Fatalf("cancellation during scan returned partial output: bytes=%d error=%v", len(got.Raw), err) + } +} + +func TestSpendThresholdPreservesDuplicateKeys(t *testing.T) { + for _, fields := range []string{ + `"threshold_amount":10000,"threshold_amount":90000`, + `"threshold_amount":10000,"threshold\u005famount":90000`, + `"threshold_amount":10000,"currency":"USD","currency":"EUR"`, + `"threshold_amount":10000,"interval":"month","interval":"year"`, + `"threshold_amount":10000,"enforcement":{"status":"inactive"},"enforcement":{"status":"enforcing"}`, + `"threshold_amount":10000,"object":"future.spend_limit"`, + `"threshold_amount":10000,"future":9007199254740993,"future":0.1234567890123456789`, + `"threshold_amount":10000,"spend_threshold":null,"spend_threshold":"future"`, + `"threshold_amount":10000,"alert_behavior":null,"alert_behavior":"future"`, + } { + for _, object := range []string{"organization.spend_limit", "project.spend_alert"} { + raw := ` { "object":"` + object + `",` + fields + ` } ` + value := gjson.Parse(raw) + got, err := projectSpendThreshold(t.Context(), value, object, strings.HasSuffix(object, "spend_alert")) + if err != nil || got.Raw != value.Raw { + t.Fatalf("ambiguous object changed: got=%s want=%s error=%v", got.Raw, value.Raw, err) + } + } + } +} + +func TestSpendThresholdCancelsDuringDuplicateScan(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + controlled := &cancelSummaryContext{Context: ctx, cancel: cancel, after: 4} + value := gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month"}`) + got, err := projectSpendThreshold(controlled, value, "organization.spend_limit", false) + if !errors.Is(err, context.Canceled) || got.Raw != "" { + t.Fatalf("cancellation during duplicate scan returned partial output: %s error=%v", got.Raw, err) + } +} diff --git a/internal/output/transform/select_transformer.go b/internal/output/transform/select_transformer.go index c956e78d..2184c48b 100644 --- a/internal/output/transform/select_transformer.go +++ b/internal/output/transform/select_transformer.go @@ -36,5 +36,8 @@ func Identity(_ context.Context, value gjson.Result) (gjson.Result, error) { // Select is the SDK-owned hook for command-specific default transformations. // Only known API fields are summarized; unfamiliar routes preserve their values. func Select(route Route) Transformer { + if transform := selectSpendThresholdTransformer(route); transform != nil { + return transform + } return selectReadableTransformer(route) } diff --git a/internal/terminal/shell_command.go b/internal/terminal/shell_command.go index ee4a3119..b8c19ff5 100644 --- a/internal/terminal/shell_command.go +++ b/internal/terminal/shell_command.go @@ -26,8 +26,7 @@ func FormatCommand(args []string, shell string) string { if quote == nil { return "" } - words := make([]string, 0, len(args)+1) - words = append(words, "openai") + words := []string{"openai"} for _, arg := range args { words = append(words, quote(arg)) } diff --git a/pkg/custom/cmdutil.go b/pkg/custom/cmdutil.go index 496c79f3..86ccd47c 100644 --- a/pkg/custom/cmdutil.go +++ b/pkg/custom/cmdutil.go @@ -386,7 +386,7 @@ func showJSON(res gjson.Result, opts ShowJSONOpts, selectTransformer transformer opts.Transform = "" formatted, err := formatJSON(res, opts) if err != nil { - return err + return spendMutationResponseFailure(err, opts) } _, err = (clioutput.Writer{Context: opts.Context, Sink: opts.Stdout}).Write(formatted) return err diff --git a/pkg/custom/command.go b/pkg/custom/command.go index 1f47f483..b3f39bb3 100644 --- a/pkg/custom/command.go +++ b/pkg/custom/command.go @@ -46,6 +46,8 @@ func ConfigureCommand(root *cli.Command) { registerAdminSetup(root) admin.ConfigureCredentials(root) localcommands.ConfigureSubgroups(root) + registerFineTuningModelLimits(root) + registerLimitsCommands(root) localcommands.ConfigureTasks(root) registerAudioCommands(root) filecommands.Configure(root) diff --git a/pkg/custom/fine_tuning_model_limits.go b/pkg/custom/fine_tuning_model_limits.go new file mode 100644 index 00000000..bfe763e0 --- /dev/null +++ b/pkg/custom/fine_tuning_model_limits.go @@ -0,0 +1,99 @@ +package custom + +import ( + "context" + "os" + "strings" + + "github.com/openai/openai-cli/internal/apiquery" + "github.com/openai/openai-cli/internal/clihelp" + localcommands "github.com/openai/openai-cli/internal/commands" + requestconfig "github.com/openai/openai-cli/internal/request" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +func registerFineTuningModelLimits(root *cli.Command) { + group := root.Command("fine-tuning") + if group == nil || group.Command("model-limits") != nil { + return + } + group.Commands = append(group.Commands, &cli.Command{ + Name: "model-limits", Usage: "Read organization fine-tuning rate limits.", + Description: "Uses an ordinary API key. Supports full JSON output and standard output options.", + Suggest: true, HideHelpCommand: true, + Metadata: map[string]any{"help-content": clihelp.Content{ + Examples: []clihelp.Example{{Description: "Read fine-tuning rate limits:", Command: "fine-tuning model-limits"}}, + }}, + Action: handleFineTuningModelLimits, + }) +} + +func handleFineTuningModelLimits(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Present() { + return &localcommands.LocalError{Message: "Fine-tuning model limits takes no positional arguments. Use --help to inspect its options."} + } + options, err := FlagOptions(cmd, apiquery.NestedQueryFormatBrackets, apiquery.ArrayQueryFormatBrackets, EmptyBody, false) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if err := checkFineTuningModelLimitsCredentials(cmd); err != nil { + return err + } + client := openai.NewClient(GetDefaultRequestOptions(cmd)...) + var response []byte + // Client.Get permits admin fallback. Clear it before explicit request-header overrides. + options = append([]option.RequestOption{option.WithAdminAPIKey("")}, options...) + options = append(options, option.WithResponseBodyInto(&response)) + if err := client.Get(ctx, "fine_tuning/model_limits", nil, nil, options...); err != nil { + return err + } + if !gjson.ValidBytes(response) { + return &localcommands.LocalError{Message: "Fine-tuning model limits returned an invalid JSON response. Check --base-url, then retry this read."} + } + root := cmd.Root() + return ShowJSON(gjson.ParseBytes(response), ShowJSONOpts{ + Context: ctx, Operation: "(resource) fine-tuning > (method) model-limits", OutputKind: OutputResponse, + ExplicitFormat: root.IsSet("format"), Format: root.String("format"), + RawOutput: root.Bool("raw-output"), Transform: root.String("transform"), + Title: "fine-tuning model-limits", Stdout: root.Writer, Stderr: root.ErrWriter, + }) +} + +// Match shared request-header precedence without imposing an admin credential requirement. +func checkFineTuningModelLimitsCredentials(cmd *cli.Command) error { + missing := &localcommands.LocalError{Message: "API key required for fine-tuning model limits. Set OPENAI_API_KEY or an explicit Authorization header. An admin key cannot replace an API key."} + headers, err := requestconfig.Headers(cmd) + if err != nil { + return err + } + if values, ok := headers["Authorization"]; ok { + if values[0] != "" { + return nil + } + return missing + } + key := os.Getenv("OPENAI_API_KEY") + if cmd.IsSet("api-key") { + key = cmd.String("api-key") + } + if key != "" { + return nil + } + var authorization string + for line := range strings.SplitSeq(os.Getenv("OPENAI_CUSTOM_HEADERS"), "\n") { + name, value, ok := strings.Cut(line, ":") + if ok && strings.EqualFold(strings.TrimSpace(name), "Authorization") { + authorization = strings.TrimSpace(value) + } + } + if authorization != "" { + return nil + } + return missing +} diff --git a/pkg/custom/limits_commands.go b/pkg/custom/limits_commands.go new file mode 100644 index 00000000..7ee09617 --- /dev/null +++ b/pkg/custom/limits_commands.go @@ -0,0 +1,130 @@ +package custom + +import ( + "context" + "errors" + "os" + "strings" + + "github.com/charmbracelet/x/term" + "github.com/openai/openai-cli/internal/clihelp" + localcommands "github.com/openai/openai-cli/internal/commands" + "github.com/openai/openai-cli/internal/commands/admin" + clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-cli/internal/output/transform" + "github.com/urfave/cli/v3" +) + +// Aggregate workflows retain their causes without presenting private source errors. +type limitsWorkflowFailure struct { + message string + // Only the text error presenter includes this additional recovery guidance. + humanDetail string + cause error +} + +func (failure *limitsWorkflowFailure) Error() string { return failure.message } +func (failure *limitsWorkflowFailure) Unwrap() error { return failure.cause } + +// Register before task promotion so the canonical and task routes share behavior. +func registerLimitsCommands(root *cli.Command) { + admin := root.Command("admin") + if admin == nil { + return + } + organization := admin.Command("organization") + if organization == nil { + return + } + organization.Commands = append(organization.Commands, newLimitsCommand(false)) + if projects := organization.Command("projects"); projects != nil { + projects.Commands = append(projects.Commands, newLimitsCommand(true)) + } +} + +func newLimitsCommand(project bool) *cli.Command { + example := "admin limits" + if project { + example = "projects limits --project-id proj_example" + } + flags := func() []cli.Flag { + result := []cli.Flag{&cli.Int64Flag{Name: "max-items", Value: 100, Usage: "Maximum records per list; -1 fetches all records."}} + if project { + result = append(result, &cli.StringFlag{Name: "project-id", Required: true, Usage: "Project to inspect or manage."}) + } + return result + } + return &cli.Command{ + Name: "limits", Usage: "Inspect and manage hard spend limits and spend alerts.", + Metadata: map[string]any{admin.CredentialsRequiredMetadata: true, "help-content": clihelp.Content{Examples: []clihelp.Example{{Description: "Open the limits workspace:", Command: example}}}}, + Description: "Open the limits workspace in an interactive terminal. Every change requires review.\n" + + "Pipes, CI, explicit output formats, and show print a read-only overview.\n" + + "Project workspaces also manage project rate limits. Requires an admin key.\n" + + "Soft budgets, usage tiers, and organization rate catalogs require the Platform website.", + Flags: flags(), Action: limitsCommandAction(project, false), + Commands: []*cli.Command{{Name: "show", Usage: "Print a read-only limits overview.", Metadata: map[string]any{admin.CredentialsRequiredMetadata: true}, Action: limitsCommandAction(project, true)}}, + } +} + +func limitsCommandAction(project, show bool) cli.ActionFunc { + return func(ctx context.Context, cmd *cli.Command) error { + if cmd.Args().Present() { + return &localcommands.LocalError{Message: "Limits does not accept positional arguments. Use --help to inspect its options."} + } + backend, err := newLimitsBackend(cmd, project) + if err != nil { + return err + } + root := cmd.Root() + opts := ShowJSONOpts{Context: ctx, Format: root.String("format"), ExplicitFormat: root.IsSet("format"), + RawOutput: root.Bool("raw-output"), Transform: root.String("transform"), Stdout: root.Writer, Stderr: root.ErrWriter} + opts.setDefaults() + if !show && limitsInteractive(cmd, opts) { + return runLimitsWorkspace(ctx, cmd, backend, opts) + } + snapshot, loadErr := backend.Load(ctx) + var sourceErr error + if loadErr != nil || len(snapshot.Errors) != 0 { + causes := []error{loadErr} + for _, failure := range snapshot.Errors { + causes = append(causes, failure.Err) + } + route := "admin limits show" + if project { + route = "projects limits show" + } + message := "The limits overview is incomplete.\nThe source_errors field identifies failed sections when output is available.\n" + + "Overview options: " + clihelp.DisplayInvocation(root) + " help " + route + "\n" + + "Keep the original connection, organization, and project settings when refreshing." + causes = append(causes, &localcommands.LocalError{Message: message}) + sourceErr = &limitsWorkflowFailure{message: message, cause: errors.Join(causes...)} + } + value, err := snapshot.JSON() + if err != nil { + return errors.Join(sourceErr, ctx.Err(), err) + } + if resolvedOutputFormat(opts) == "text" && opts.Transform == "" && !opts.RawOutput { + value, err = transform.ProjectLimitsOverview(ctx, value) + if err != nil { + return errors.Join(sourceErr, ctx.Err(), err) + } + } + write := func() error { + outputErr := ShowJSON(value, opts) + return errors.Join(sourceErr, ctx.Err(), outputErr) + } + if file, ok := opts.Stdout.(*os.File); ok && file == os.Stdout { + // Join source failures before classifying a closed stdout pipe. + return clioutput.StreamToStdout(func(*os.File) error { return write() }) + } + return write() + } +} + +func limitsInteractive(cmd *cli.Command, opts ShowJSONOpts) bool { + format := strings.ToLower(opts.Format) + input, ok := cmd.Root().Reader.(*os.File) + return ok && term.IsTerminal(input.Fd()) && isTerminal(opts.Stdout) && + (format == "" || format == "auto") && !opts.ExplicitFormat && !opts.RawOutput && opts.Transform == "" && + !cmd.Root().Bool("quiet") && os.Getenv("CI") == "" && os.Getenv("TERM") != "dumb" +} diff --git a/pkg/custom/limits_draft.go b/pkg/custom/limits_draft.go new file mode 100644 index 00000000..ba9855df --- /dev/null +++ b/pkg/custom/limits_draft.go @@ -0,0 +1,335 @@ +package custom + +import ( + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" + + "charm.land/bubbles/v2/textinput" + "github.com/openai/openai-cli/internal/readable" + "github.com/openai/openai-cli/internal/terminal" + "github.com/tidwall/gjson" +) + +var limitsRateInputs = []struct{ key, label string }{ + {"max_requests_per_1_minute", "Requests / minute"}, + {"max_tokens_per_1_minute", "Tokens / minute"}, + {"max_requests_per_1_day", "Requests / day"}, + {"max_images_per_1_minute", "Images / minute"}, + {"max_audio_megabytes_per_1_minute", "Audio megabytes / minute"}, + {"batch_1_day_max_input_tokens", "Batch input tokens / day"}, +} + +type limitsDraft struct { + change limitsChange + labels []string + inputs []textinput.Model + initial []string + prefixMode int // Preserve, replace, or null. Empty replacement stays an empty string. +} + +func newLimitsDraft(change limitsChange) (limitsDraft, error) { + d := limitsDraft{change: change} + if change.Before.Exists() { + if err := limitsEditableBaseline(change.Before, change.Kind); err != nil { + return d, err + } + } + add := func(label, value string) { + input := textinput.New() + input.Prompt = "" + input.CharLimit = 0 + input.SetValue(value) + d.labels = append(d.labels, label) + d.inputs = append(d.inputs, input) + d.initial = append(d.initial, value) + } + switch change.Kind { + case "limit_set", "alert_create", "alert_update": + amount := "" + if change.Before.Get("threshold_amount").Exists() { + amount = formatLimitsDollars(change.Before.Get("threshold_amount").Int()) + } + add("Monthly USD", amount) + if change.Kind != "limit_set" { + var recipients []string + for _, value := range change.Before.Get("notification_channel.recipients").Array() { + recipients = append(recipients, value.String()) + } + add("Email recipients (comma separated)", strings.Join(recipients, ", ")) + add("Subject prefix", change.Before.Get("notification_channel.subject_prefix").String()) + } + case "rate_update": + for _, field := range limitsRateInputs { + value := change.Before.Get(field.key) + text := "" + if value.Type == gjson.Number { + text = value.Raw + } + add(field.label+" (blank: unchanged)", text) + } + } + for i, input := range d.inputs { + if input.Value() != d.initial[i] { + return d, errors.New("This record contains text that the form cannot preserve. Inspect the original API record.") + } + } + return d, nil +} + +func (d limitsDraft) validate() (limitsChange, error) { + c := d.change + switch c.Kind { + case "limit_set", "alert_create", "alert_update": + minimum := int64(0) + if c.Kind == "limit_set" { + minimum = 1 + } + amount, err := parseLimitsDollars(d.inputs[0].Value(), minimum) + if err != nil { + return c, err + } + c.Amount = amount + if c.Kind != "limit_set" { + c.Recipients, err = parseLimitsRecipients(d.inputs[1].Value()) + if err != nil { + return c, err + } + value := d.inputs[2].Value() + switch { + case d.prefixMode == 2: + c.SubjectPrefixSet, c.SubjectPrefix = true, nil + case d.prefixMode == 1 || value != d.initial[2]: + c.SubjectPrefixSet, c.SubjectPrefix = true, &value + default: + prefix := c.Before.Get("notification_channel.subject_prefix") + c.SubjectPrefixSet = prefix.Exists() + if prefix.Type == gjson.String { + s := prefix.String() + c.SubjectPrefix = &s + } + } + } + case "rate_update": + c.RateValues = map[string]int64{} + for i, field := range limitsRateInputs { + value := d.inputs[i].Value() + if value == "" || value == d.initial[i] { + continue + } + digits := strings.TrimPrefix(value, "-") + if digits == "" { + return c, fmt.Errorf("%s needs an integer.", field.label) + } + for _, ch := range digits { + if ch < '0' || ch > '9' { + return c, fmt.Errorf("%s needs an integer.", field.label) + } + } + number, err := strconv.ParseInt(value, 10, 64) + if err != nil { + return c, fmt.Errorf("%s exceeds the supported integer range.", field.label) + } + c.RateValues[field.key] = number + } + if len(c.RateValues) == 0 { + return c, errors.New("Change at least one rate value.") + } + } + return c, nil +} + +func limitsChangeTitle(kind string) string { + switch kind { + case "limit_set": + return "Set hard spend limit" + case "limit_delete": + return "Remove hard spend limit" + case "alert_create": + return "Create spend alert" + case "alert_update": + return "Edit spend alert" + case "alert_delete": + return "Delete spend alert" + case "rate_update": + return "Update project rate limits" + } + return "Review change" +} + +func limitsChangeBody(c limitsChange) map[string]any { + body := map[string]any{} + if c.Kind == "limit_set" || c.Kind == "alert_create" || c.Kind == "alert_update" { + body["currency"], body["interval"], body["threshold_amount"] = "USD", "month", c.Amount + } + if c.Kind == "alert_create" || c.Kind == "alert_update" { + channel := map[string]any{"type": "email", "recipients": c.Recipients} + if c.SubjectPrefixSet { + channel["subject_prefix"] = c.SubjectPrefix + } + body["notification_channel"] = channel + } + for name, value := range c.RateValues { + body[name] = value + } + return body +} + +func limitsReview(c limitsChange, projectID string, requestArgs []string, omit bool, shell string, executable ...string) string { + scope := "Organization" + if projectID != "" { + scope = "Project " + projectID + } + before := "Current limit not confirmed" + if c.Before.Exists() { + before = c.Before.Raw + } + if c.Kind == "alert_create" { + before = "New alert" + } + after, _ := json.MarshalIndent(limitsChangeBody(c), "", " ") + result := scope + "\n" + limitsChangeTitle(c.Kind) + "\n" + if c.ID != "" { + result += "ID: " + c.ID + "\n" + } + if c.Kind == "limit_set" || c.Kind == "limit_delete" || strings.HasPrefix(c.Kind, "alert_") { + label, previous := "Hard spend limit", "Not confirmed" + if strings.HasPrefix(c.Kind, "alert_") { + label = "Alert threshold" + } + if c.Before.Exists() { + previous = limitsThresholdLabel(c.Before) + } else if c.Kind == "alert_create" { + previous = "New alert" + } + next := "USD " + formatLimitsDollars(c.Amount) + " / month" + if strings.HasSuffix(c.Kind, "_delete") { + next = "Remove this resource" + } + result += "\n" + label + "\nBefore: " + previous + "\nAfter: " + next + "\n" + } + result += "\nBefore\n" + before + "\n\nAfter\n" + if strings.HasSuffix(c.Kind, "_delete") { + result += "Remove this resource.\n" + } else { + result += string(after) + "\n" + } + switch c.Kind { + case "limit_set": + result += "\nHard limit: USD " + formatLimitsDollars(c.Amount) + " / month. Enforcement can be delayed.\n" + if c.BeforeMissing { + result += "The current limit could not be confirmed. This action creates or replaces the hard limit.\n" + } + case "limit_delete": + result += "\nThis removes the hard limit. The public API does not guarantee soft-budget behavior.\nInspect soft budgets: https://platform.openai.com/settings/organization/limits\n" + case "alert_create", "alert_update": + result += "\nAlerts notify recipients. Alerts do not enforce a spending cap.\n" + case "rate_update": + result += "\nOnly changed rate fields are sent. Zero has no assumed unlimited meaning.\n" + } + result += "\nThe CLI checks the current record before saving. This check is not atomic.\n" + result += "The workspace sends this change once, without automatic retries.\n" + result = readable.Text(result) + command, reason := limitsCommand(c, projectID, requestArgs, omit, shell, executable...) + if command == "" { + return result + "\n" + reason + "\n" + } + return result + "\nCommand preview (inspection only)\n" + command + "\n\nCtrl+P prints the complete command and exits.\nThis command skips the workspace preflight and can retry automatically.\nRecheck current settings before running it.\n" +} + +// Keep executable bytes separate from prose and viewport wrapping. +func limitsCommand(c limitsChange, projectID string, requestArgs []string, omit bool, shell string, executable ...string) (string, string) { + if omit { + return "", "Command preview omitted: request context cannot be reproduced safely." + } + args := append([]string{}, requestArgs...) + args = append(args, "admin", "organization") + if projectID != "" { + args = append(args, "projects") + } + switch c.Kind { + case "limit_set": + args = append(args, "spend-limit", "update") + case "limit_delete": + args = append(args, "spend-limit", "delete") + case "alert_create": + args = append(args, "spend-alerts", "create") + case "alert_update": + args = append(args, "spend-alerts", "update") + case "alert_delete": + args = append(args, "spend-alerts", "delete") + case "rate_update": + args = append(args, "rate-limits", "update-rate-limit") + } + if projectID != "" { + args = append(args, "--project-id", projectID) + } + if c.ID != "" { + flag := "--alert-id" + if c.Kind == "rate_update" { + flag = "--rate-limit-id" + } + args = append(args, flag, c.ID) + } + if c.Kind == "limit_set" || c.Kind == "alert_create" || c.Kind == "alert_update" { + args = append(args, "--currency", "USD", "--interval", "month", "--threshold-amount", strconv.FormatInt(c.Amount, 10)) + } + if c.Kind == "alert_create" || c.Kind == "alert_update" { + channelValues := limitsChangeBody(c)["notification_channel"].(map[string]any) + if c.SubjectPrefixSet && c.SubjectPrefix != nil { + prefix := *c.SubjectPrefix + if strings.HasPrefix(prefix, "\\@") { + return "", "Command preview omitted: the request parser cannot preserve this literal prefix." + } + if strings.HasPrefix(prefix, "@") { + channelValues["subject_prefix"] = "\\" + prefix + } + } + channel, _ := json.Marshal(channelValues) + args = append(args, "--notification-channel", string(channel)) + } + for _, field := range limitsRateInputs { + if value, ok := c.RateValues[field.key]; ok { + args = append(args, "--"+strings.ReplaceAll(field.key, "_", "-"), strconv.FormatInt(value, 10)) + } + } + command := terminal.FormatCommand(args, shell) + if command == "" { + return "", "Command printing needs Bash, zsh, fish or PowerShell 7." + } + if len(executable) > 0 && executable[0] != "" { + command = terminal.ShellQuoter(shell)(executable[0]) + strings.TrimPrefix(command, "openai") + if shell == "pwsh" { + command = "& " + command + } + } + return command, "" +} + +// Use the generated operation identity so successful receipts retain projections. +func limitsChangeOperation(kind string, project bool) string { + resource := "admin.organization." + if project { + resource += "projects." + } + method := "" + switch kind { + case "limit_set": + resource, method = resource+"spend_limit", "update" + case "limit_delete": + resource, method = resource+"spend_limit", "delete" + case "alert_create": + resource, method = resource+"spend_alerts", "create" + case "alert_update": + resource, method = resource+"spend_alerts", "update" + case "alert_delete": + resource, method = resource+"spend_alerts", "delete" + case "rate_update": + resource, method = resource+"rate_limits", "update_rate_limit" + default: + return "" + } + return "(resource) " + resource + " > (method) " + method +} diff --git a/pkg/custom/limits_service.go b/pkg/custom/limits_service.go new file mode 100644 index 00000000..472f73a7 --- /dev/null +++ b/pkg/custom/limits_service.go @@ -0,0 +1,733 @@ +package custom + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "sync" + "time" + + "github.com/openai/openai-cli/internal/apiquery" + localcommands "github.com/openai/openai-cli/internal/commands" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +type limitsBackend interface { + Load(context.Context) (limitsSnapshot, error) + Mutate(context.Context, limitsChange) (gjson.Result, error) +} + +type limitsSourceFailure struct { + Resource string + Err error +} + +type limitsCostSummaryError struct{ Reason string } + +func (*limitsCostSummaryError) Error() string { + return "Costs summary unavailable. Inspect costs.buckets or use the usage costs command." +} + +type limitsMutationFailure struct { + Err error + Attempted bool +} + +func (failure *limitsMutationFailure) Error() string { return failure.Err.Error() } +func (failure *limitsMutationFailure) Unwrap() error { return failure.Err } + +type limitsSnapshot struct { + ProjectID string + Project gjson.Result + Limit gjson.Result + LimitAbsent bool + LimitUnconfirmed bool + Alerts []gjson.Result + Rates []gjson.Result + AlertsComplete bool + RatesComplete bool + Costs []gjson.Result + CostsComplete bool + CostsTotal string + CostsStart int64 + CostsEnd int64 + Errors []limitsSourceFailure +} + +type limitsChange struct { + Kind string + ID string + Before gjson.Result + BeforeMissing bool + Amount int64 + Recipients []string + SubjectPrefix *string + SubjectPrefixSet bool + RateValues map[string]int64 +} + +type limitsAPIBackend struct { + client openai.Client + projectID string + maxItems int64 + mutationMu sync.Mutex + now func() time.Time +} + +func newLimitsBackend(cmd *cli.Command, project bool) (limitsBackend, error) { + maximum := cmd.Int64("max-items") + if maximum < -1 { + return nil, &localcommands.LocalError{Message: "--max-items must be -1, zero, or a positive integer."} + } + projectID := "" + if project { + projectID = cmd.String("project-id") + if projectID == "" { + return nil, &localcommands.LocalError{Message: "Set --project-id to inspect project limits."} + } + } + options, err := FlagOptions(cmd, apiquery.NestedQueryFormatBrackets, apiquery.ArrayQueryFormatBrackets, EmptyBody, true) + if err != nil { + return nil, err + } + options = append(GetDefaultRequestOptions(cmd), options...) + return &limitsAPIBackend{client: openai.NewClient(options...), projectID: projectID, maxItems: maximum}, nil +} + +func (snapshot limitsSnapshot) JSON() (gjson.Result, error) { + raw := func(value gjson.Result) json.RawMessage { + if !value.Exists() { + return json.RawMessage("null") + } + return json.RawMessage(value.Raw) + } + array := func(items []gjson.Result) []json.RawMessage { + result := make([]json.RawMessage, 0, len(items)) + for _, item := range items { + result = append(result, raw(item)) + } + return result + } + sourceErrors := make([]map[string]any, 0, len(snapshot.Errors)) + for _, failure := range snapshot.Errors { + source := map[string]any{"resource": failure.Resource, "status": "request_failed"} + var apiErr *openai.Error + var summaryErr *limitsCostSummaryError + if errors.Is(failure.Err, context.Canceled) || errors.Is(failure.Err, context.DeadlineExceeded) { + source["status"] = "canceled" + } else if errors.As(failure.Err, &apiErr) { + source["status"] = "api_error" + if apiErr.StatusCode > 0 { + source["status_code"] = apiErr.StatusCode + } + } else if errors.As(failure.Err, &summaryErr) { + source["status"] = "summary_unavailable" + } + if errors.As(failure.Err, &summaryErr) { + source["summary_reason"] = summaryErr.Reason + } + sourceErrors = append(sourceErrors, source) + } + var configured any + limitFailed := false + for _, failure := range snapshot.Errors { + limitFailed = limitFailed || failure.Resource == "spend_limit" || failure.Resource == "project" + } + if !limitFailed && snapshot.LimitAbsent { + configured = false + } else if !limitFailed && snapshot.Limit.Exists() { + configured = true + } + view := map[string]any{ + "scope": "organization", "spend_limit": raw(snapshot.Limit), "spend_limit_configured": configured, + "spend_alerts": array(snapshot.Alerts), "spend_alerts_complete": snapshot.AlertsComplete, + "source_errors": sourceErrors, + } + var costTotal any + if snapshot.CostsTotal != "" { + costTotal = snapshot.CostsTotal + } + view["costs"] = map[string]any{ + "start_time": snapshot.CostsStart, "end_time": snapshot.CostsEnd, "timezone": "UTC", + "currency": "USD", "total": costTotal, "complete": snapshot.CostsComplete, "buckets": array(snapshot.Costs), + } + if snapshot.ProjectID != "" { + view["scope"], view["project_id"], view["project"] = "project", snapshot.ProjectID, raw(snapshot.Project) + view["project_rate_limits"], view["project_rate_limits_complete"] = array(snapshot.Rates), snapshot.RatesComplete + } + encoded, err := json.Marshal(view) + if err != nil { + return gjson.Result{}, err + } + return gjson.ParseBytes(encoded), nil +} + +func (backend *limitsAPIBackend) Load(ctx context.Context) (limitsSnapshot, error) { + snapshot := limitsSnapshot{ProjectID: backend.projectID} + var failures []error + record := func(resource string, err error) { + if err != nil { + snapshot.Errors = append(snapshot.Errors, limitsSourceFailure{Resource: resource, Err: err}) + failures = append(failures, fmt.Errorf("%s: %w", resource, err)) + } + } + projectVerified := backend.projectID == "" + var err error + if backend.projectID != "" { + snapshot.Project, err = backend.get(ctx, "project", "") + record("project", err) + projectVerified = err == nil + } + if projectVerified { + snapshot.Limit, err = backend.get(ctx, "spend_limit", "") + if limitsHTTPStatus(err, http.StatusNotFound) { + // HTTP 404 does not distinguish absent configuration from endpoint or access errors. + snapshot.LimitUnconfirmed = true + } + record("spend_limit", err) + } + snapshot.Alerts, snapshot.AlertsComplete, err = backend.list(ctx, false) + record("spend_alerts", err) + if backend.projectID != "" { + snapshot.Rates, snapshot.RatesComplete, err = backend.list(ctx, true) + record("project_rate_limits", err) + } + now := time.Now().UTC() + if backend.now != nil { + now = backend.now().UTC() + } + snapshot.CostsStart = time.Date(now.Year(), now.Month(), 1, 0, 0, 0, 0, time.UTC).Unix() + snapshot.CostsEnd = time.Date(now.Year(), now.Month(), now.Day()+1, 0, 0, 0, 0, time.UTC).Unix() + snapshot.Costs, snapshot.CostsComplete, err = backend.costs(ctx, snapshot.CostsStart, snapshot.CostsEnd) + if len(snapshot.Costs) != 0 || snapshot.CostsComplete { + var totalErr error + snapshot.CostsTotal, totalErr = limitsCostTotal(ctx, snapshot.Costs, snapshot.CostsStart, snapshot.CostsEnd, backend.projectID) + err = errors.Join(err, totalErr) + if totalErr != nil { + snapshot.CostsComplete = false + } + } + record("costs", err) + return snapshot, errors.Join(failures...) +} + +// Costs returns raw daily buckets. The independent summary never changes those records. +func (backend *limitsAPIBackend) costs(ctx context.Context, start, end int64) ([]gjson.Result, bool, error) { + items := []gjson.Result{} + cursor := "" + seen := map[string]bool{} + for { + if err := ctx.Err(); err != nil { + return items, false, err + } + pageSize := int64(100) + if backend.maxItems >= 0 { + pageSize = max(1, min(pageSize, backend.maxItems-int64(len(items)))) + } + params := openai.AdminOrganizationUsageCostsParams{ + StartTime: start, EndTime: openai.Int(end), Limit: openai.Int(pageSize), BucketWidth: "1d", + } + if backend.projectID != "" { + params.ProjectIDs = []string{backend.projectID} + } + if cursor != "" { + params.Page = openai.String(cursor) + } + var body []byte + _, err := backend.client.Admin.Organization.Usage.Costs(ctx, params, option.WithResponseBodyInto(&body)) + page, err := limitsResponse(body, err) + if err != nil { + return items, false, err + } + if err := limitsUniqueObjectContext(ctx, page); err != nil { + return items, false, err + } + data, more := page.Get("data"), page.Get("has_more") + if page.Get("object").Str != "page" || !data.IsArray() || more.Type != gjson.True && more.Type != gjson.False { + return items, false, errors.New("The Costs response is missing its buckets or pagination state.") + } + truncated := false + var readErr error + count := 0 + data.ForEach(func(_, item gjson.Result) bool { + count++ + if readErr = ctx.Err(); readErr != nil { + return false + } + if backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems { + truncated = true + return false + } + items = append(items, item) + return true + }) + if readErr != nil { + return items, false, readErr + } + if truncated || backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems && more.Bool() { + return items, false, nil + } + if !more.Bool() { + return items, true, nil + } + next := page.Get("next_page") + if count == 0 || next.Type != gjson.String || next.Str == "" || seen[next.Str] || next.Str == cursor { + return items, false, errors.New("The Costs pagination cursor is missing, repeated, or cannot advance.") + } + cursor, seen[next.Str] = next.Str, true + } +} + +func limitsHTTPStatus(err error, status int) bool { + var apiErr *openai.Error + return errors.As(err, &apiErr) && apiErr.StatusCode == status +} + +func limitsResponse(raw []byte, err error) (gjson.Result, error) { + if err != nil { + return gjson.Result{}, err + } + if !gjson.ValidBytes(raw) { + return gjson.Result{}, errors.New("The limits response contains invalid JSON.") + } + value := gjson.ParseBytes(raw) + if !value.IsObject() { + return gjson.Result{}, errors.New("The limits response must contain an object.") + } + return value, nil +} + +func (backend *limitsAPIBackend) get(ctx context.Context, resource, id string) (gjson.Result, error) { + if err := ctx.Err(); err != nil { + return gjson.Result{}, err + } + var body []byte + options := []option.RequestOption{option.WithResponseBodyInto(&body)} + var err error + switch resource { + case "project": + _, err = backend.client.Admin.Organization.Projects.Get(ctx, backend.projectID, options...) + case "spend_limit": + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendLimit.Get(ctx, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendLimit.Get(ctx, backend.projectID, options...) + } + case "spend_alert": + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendAlerts.Get(ctx, id, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.Get(ctx, backend.projectID, id, options...) + } + default: + return gjson.Result{}, errors.New("Unknown limits resource.") + } + value, err := limitsResponse(body, err) + if err == nil { + expectedObject := "organization." + resource + if backend.projectID != "" && resource != "project" { + expectedObject = "project." + resource + } + if value.Get("object").Str != expectedObject { + err = errors.New("The limits response has an unexpected object type.") + } else if resource == "project" { + err = limitsUniqueObjectContext(ctx, value) + if err == nil && value.Get("id").Str != backend.projectID { + err = errors.New("The project response does not identify the requested project.") + } + } else if resource == "spend_alert" && value.Get("id").Str != id { + err = errors.New("The alert response does not identify the requested alert.") + } + } + return value, err +} + +func (backend *limitsAPIBackend) list(ctx context.Context, rates bool) ([]gjson.Result, bool, error) { + items := []gjson.Result{} + after := "" + seen := map[string]bool{} + for { + if err := ctx.Err(); err != nil { + return items, false, err + } + pageSize := int64(100) + if backend.maxItems >= 0 { + pageSize = max(1, min(pageSize, backend.maxItems-int64(len(items)))) + } + var body []byte + options := []option.RequestOption{option.WithResponseBodyInto(&body)} + var err error + if rates { + params := openai.AdminOrganizationProjectRateLimitListRateLimitsParams{Limit: openai.Int(pageSize)} + if after != "" { + params.After = openai.String(after) + } + _, err = backend.client.Admin.Organization.Projects.RateLimits.ListRateLimits(ctx, backend.projectID, params, options...) + } else if backend.projectID == "" { + params := openai.AdminOrganizationSpendAlertListParams{Limit: openai.Int(pageSize)} + if after != "" { + params.After = openai.String(after) + } + _, err = backend.client.Admin.Organization.SpendAlerts.List(ctx, params, options...) + } else { + params := openai.AdminOrganizationProjectSpendAlertListParams{Limit: openai.Int(pageSize)} + if after != "" { + params.After = openai.String(after) + } + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.List(ctx, backend.projectID, params, options...) + } + page, err := limitsResponse(body, err) + if err != nil { + return items, false, err + } + if err := limitsUniqueObjectContext(ctx, page); err != nil { + return items, false, err + } + data, more := page.Get("data"), page.Get("has_more") + if !data.IsArray() || more.Type != gjson.True && more.Type != gjson.False { + return items, false, errors.New("The limits page is missing its records or pagination state.") + } + truncated := false + var recordErr error + data.ForEach(func(_, item gjson.Result) bool { + if err := ctx.Err(); err != nil { + recordErr = err + return false + } + if backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems { + truncated = true + return false + } + if !item.IsObject() { + recordErr = errors.New("A limits page contains a non-object record.") + return false + } + items = append(items, item) + return true + }) + if recordErr != nil { + return items, false, recordErr + } + if truncated || backend.maxItems >= 0 && int64(len(items)) >= backend.maxItems && more.Bool() { + return items, false, nil + } + if !more.Bool() { + return items, true, nil + } + cursor := page.Get("last_id") + if cursor.Type != gjson.String || cursor.Str == "" || cursor.Str == after || seen[cursor.Str] { + return items, false, errors.New("The limits pagination cursor is missing or repeated.") + } + if len(data.Array()) == 0 { + return items, false, errors.New("The limits page cannot advance without records.") + } + after, seen[cursor.Str] = cursor.Str, true + } +} + +func (backend *limitsAPIBackend) Mutate(ctx context.Context, change limitsChange) (result gjson.Result, resultErr error) { + attempted := false + defer func() { + if resultErr != nil { + resultErr = &limitsMutationFailure{Err: resultErr, Attempted: attempted} + } + }() + if !backend.mutationMu.TryLock() { + return gjson.Result{}, errors.New("A limits change is already in progress.") + } + defer backend.mutationMu.Unlock() + if err := backend.preflight(ctx, change); err != nil { + return gjson.Result{}, err + } + var body []byte + options := []option.RequestOption{option.WithMaxRetries(0), option.WithResponseBodyInto(&body)} + var err error + switch change.Kind { + case "limit_set": + attempted = true + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendLimit.Update(ctx, openai.AdminOrganizationSpendLimitUpdateParams{ + Currency: "USD", Interval: "month", ThresholdAmount: change.Amount, + }, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendLimit.Update(ctx, backend.projectID, openai.AdminOrganizationProjectSpendLimitUpdateParams{ + Currency: "USD", Interval: "month", ThresholdAmount: change.Amount, + }, options...) + } + case "limit_delete": + attempted = true + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendLimit.Delete(ctx, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendLimit.Delete(ctx, backend.projectID, options...) + } + case "alert_create", "alert_update": + payload, payloadErr := limitsAlertPayload(change) + if payloadErr != nil { + return gjson.Result{}, payloadErr + } + if backend.projectID == "" && change.Kind == "alert_create" { + var params openai.AdminOrganizationSpendAlertNewParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.SpendAlerts.New(ctx, params, options...) + } + } else if backend.projectID == "" { + var params openai.AdminOrganizationSpendAlertUpdateParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.SpendAlerts.Update(ctx, change.ID, params, options...) + } + } else if change.Kind == "alert_create" { + var params openai.AdminOrganizationProjectSpendAlertNewParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.New(ctx, backend.projectID, params, options...) + } + } else { + var params openai.AdminOrganizationProjectSpendAlertUpdateParams + if err = json.Unmarshal(payload, ¶ms); err == nil { + attempted = true + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.Update(ctx, backend.projectID, change.ID, params, options...) + } + } + case "alert_delete": + attempted = true + if backend.projectID == "" { + _, err = backend.client.Admin.Organization.SpendAlerts.Delete(ctx, change.ID, options...) + } else { + _, err = backend.client.Admin.Organization.Projects.SpendAlerts.Delete(ctx, backend.projectID, change.ID, options...) + } + case "rate_update": + params := openai.AdminOrganizationProjectRateLimitUpdateRateLimitParams{} + for field, value := range change.RateValues { + switch field { + case "max_requests_per_1_minute": + params.MaxRequestsPer1Minute = openai.Int(value) + case "max_tokens_per_1_minute": + params.MaxTokensPer1Minute = openai.Int(value) + case "max_requests_per_1_day": + params.MaxRequestsPer1Day = openai.Int(value) + case "max_images_per_1_minute": + params.MaxImagesPer1Minute = openai.Int(value) + case "max_audio_megabytes_per_1_minute": + params.MaxAudioMegabytesPer1Minute = openai.Int(value) + case "batch_1_day_max_input_tokens": + params.Batch1DayMaxInputTokens = openai.Int(value) + } + } + attempted = true + _, err = backend.client.Admin.Organization.Projects.RateLimits.UpdateRateLimit(ctx, backend.projectID, change.ID, params, options...) + } + value, responseErr := limitsResponse(body, err) + if responseErr == nil { + responseErr = limitsMutationConfirmedContext(ctx, value, change, backend.projectID != "") + } + return value, responseErr +} + +func limitsMutationConfirmed(value gjson.Result, change limitsChange, project bool) error { + return limitsMutationConfirmedContext(context.Background(), value, change, project) +} + +func limitsMutationConfirmedContext(ctx context.Context, value gjson.Result, change limitsChange, project bool) error { + scope := "organization" + if project { + scope = "project" + } + resource := "spend_limit" + if change.Kind == "alert_create" || change.Kind == "alert_update" || change.Kind == "alert_delete" { + resource = "spend_alert" + } + if change.Kind == "rate_update" { + resource = "rate_limit" + } + deleting := change.Kind == "limit_delete" || change.Kind == "alert_delete" + object := scope + "." + resource + if deleting { + object += ".deleted" + } + invalid := errors.New("The API returned an incomplete confirmation. Refresh and inspect the resource before another change.") + if value.Get("object").Type != gjson.String || value.Get("object").Str != object { + return invalid + } + if err := limitsUniqueJSONContext(ctx, value); err != nil { + if ctx.Err() != nil { + return ctx.Err() + } + return invalid + } + if resource != "spend_limit" { + id := value.Get("id") + if id.Type != gjson.String || id.Str == "" || change.Kind != "alert_create" && id.Str != change.ID { + return invalid + } + } + if deleting { + if value.Get("deleted").Type != gjson.True { + return invalid + } + return ctx.Err() + } + if resource == "rate_limit" { + if value.Get("model").Type != gjson.String || value.Get("model").Str == "" || + value.Get("max_requests_per_1_minute").Type != gjson.Number || value.Get("max_tokens_per_1_minute").Type != gjson.Number { + return invalid + } + } else { + if value.Get("threshold_amount").Type != gjson.Number || value.Get("currency").Type != gjson.String || + value.Get("currency").Str == "" || value.Get("interval").Type != gjson.String || value.Get("interval").Str == "" { + return invalid + } + if resource == "spend_alert" { + channel := value.Get("notification_channel") + if !channel.IsObject() || channel.Get("type").Type != gjson.String || !channel.Get("recipients").IsArray() { + return invalid + } + } + } + mismatch := errors.New("The API returned different values from the reviewed change. Inspect the returned record before another change.") + if resource == "rate_limit" { + for field, requested := range change.RateValues { + if err := ctx.Err(); err != nil { + return err + } + returned, err := limitsInteger(value.Get(field)) + if err != nil || returned != requested { + return mismatch + } + } + } else { + returned, err := limitsInteger(value.Get("threshold_amount")) + if err != nil || returned != change.Amount || value.Get("currency").Str != "USD" || value.Get("interval").Str != "month" { + return mismatch + } + if resource == "spend_alert" { + channel := value.Get("notification_channel") + if channel.Get("type").Str != "email" { + return mismatch + } + recipients := map[string]int{} + for _, address := range change.Recipients { + recipients[address]++ + } + var recipientErr error + channel.Get("recipients").ForEach(func(_, address gjson.Result) bool { + if recipientErr = ctx.Err(); recipientErr != nil { + return false + } + if address.Type != gjson.String || recipients[address.Str] == 0 { + recipientErr = mismatch + return false + } + recipients[address.Str]-- + return true + }) + if recipientErr != nil { + return recipientErr + } + for _, count := range recipients { + if count != 0 { + return mismatch + } + } + if change.SubjectPrefixSet { + prefix := channel.Get("subject_prefix") + if !prefix.Exists() || change.SubjectPrefix == nil && prefix.Type != gjson.Null || + change.SubjectPrefix != nil && (prefix.Type != gjson.String || prefix.Str != *change.SubjectPrefix) { + return mismatch + } + } + } + } + return ctx.Err() +} + +func (backend *limitsAPIBackend) preflight(ctx context.Context, change limitsChange) error { + if err := ctx.Err(); err != nil { + return err + } + if err := limitsValidateChangeContext(ctx, change); err != nil { + return err + } + if backend.projectID != "" { + if _, err := backend.get(ctx, "project", ""); err != nil { + return err + } + } + if change.Kind == "alert_create" { + return ctx.Err() + } + var current gjson.Result + var err error + switch change.Kind { + case "limit_set", "limit_delete": + current, err = backend.get(ctx, "spend_limit", "") + if change.Kind == "limit_set" && change.BeforeMissing && limitsHTTPStatus(err, http.StatusNotFound) { + // BeforeMissing records explicit review of an unconfirmed baseline, not proven absence. + return ctx.Err() + } + case "alert_update", "alert_delete": + current, err = backend.get(ctx, "spend_alert", change.ID) + case "rate_update": + if backend.projectID == "" { + return errors.New("Rate-limit changes require a project scope.") + } + var items []gjson.Result + var complete bool + items, complete, err = backend.list(ctx, true) + if err == nil && !complete { + return errors.New("Rate-limit editing requires a complete list. Reopen with a larger --max-items value or -1.") + } + for _, item := range items { + if item.Get("id").Str == change.ID { + if current.Exists() { + return errors.New("The rate-limit ID appears more than once. Refresh before editing.") + } + current = item + } + } + } + if err != nil { + return err + } + if change.BeforeMissing || !current.Exists() { + return errors.New("The configuration changed. Refresh before applying this change.") + } + if err := limitsEditableBaselineContext(ctx, current, change.Kind); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + before, err := limitsCanonicalJSONContext(ctx, change.Before) + if err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + now, err := limitsCanonicalJSONContext(ctx, current) + if err != nil { + return err + } + if before != now { + return errors.New("The configuration changed. Refresh before applying this change.") + } + return ctx.Err() +} + +func limitsAlertPayload(change limitsChange) ([]byte, error) { + channel := map[string]any{"type": "email", "recipients": change.Recipients} + if change.SubjectPrefixSet { + channel["subject_prefix"] = change.SubjectPrefix + } + return json.Marshal(map[string]any{ + "currency": "USD", "interval": "month", "threshold_amount": change.Amount, "notification_channel": channel, + }) +} diff --git a/pkg/custom/limits_service_test.go b/pkg/custom/limits_service_test.go new file mode 100644 index 00000000..ff52eb8c --- /dev/null +++ b/pkg/custom/limits_service_test.go @@ -0,0 +1,812 @@ +package custom + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + "time" + + localcommands "github.com/openai/openai-cli/internal/commands" + requestconfig "github.com/openai/openai-cli/internal/request" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +const limitsTestLimit = `{"object":"organization.spend_limit","threshold_amount":10000,"currency":"USD","interval":"month","enforcement":{"status":"enforcing"}}` +const limitsTestAlert = `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":100,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}` +const limitsTestRate = `{"id":"rate_demo","object":"project.rate_limit","model":"synthetic-model","max_requests_per_1_minute":100,"max_tokens_per_1_minute":200,"future":{"precision":0.1234567890123456789}}` + +func limitsTestBackend(t *testing.T, project string, maximum int64, handler http.HandlerFunc) *limitsAPIBackend { + t.Helper() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") != "Bearer synthetic-limits-admin" { + t.Error("limits service did not select the admin credential") + } + w.Header().Set("Content-Type", "application/json") + handler(w, r) + })) + t.Cleanup(server.Close) + return &limitsAPIBackend{client: openai.NewClient(option.WithBaseURL(server.URL), + option.WithAPIKey("synthetic-limits-project"), option.WithAdminAPIKey("synthetic-limits-admin"), option.WithMaxRetries(0)), + projectID: project, maxItems: maximum} +} + +func limitsTestWrite(t *testing.T, w http.ResponseWriter, body string) { + t.Helper() + if _, err := io.WriteString(w, body); err != nil { + t.Errorf("write synthetic response: %v", err) + } +} + +func TestLimitsServicePartialLoadPreservesRawValues(t *testing.T) { + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitsTestWrite(t, w, `{"object":"organization.spend_limit","threshold_amount":9007199254740993,"future":{"decimal":0.1234567890123456789}}`) + return + } + w.WriteHeader(http.StatusForbidden) + limitsTestWrite(t, w, `{"error":{"message":"synthetic-private-field","type":"permission_error"}}`) + }) + snapshot, err := backend.Load(t.Context()) + require.Error(t, err) + require.False(t, snapshot.AlertsComplete) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "9007199254740993", value.Get("spend_limit.threshold_amount").Raw) + require.Equal(t, "0.1234567890123456789", value.Get("spend_limit.future.decimal").Raw) + require.Equal(t, "api_error", value.Get("source_errors.0.status").Str) + require.Equal(t, int64(403), value.Get("source_errors.0.status_code").Int()) + require.NotContains(t, value.Raw, "synthetic-private-field") + require.Equal(t, "[]", value.Get("spend_alerts").Raw) +} + +func TestLimitsServiceDoesNotInferLimitAbsenceAfterProjectFailure(t *testing.T) { + var limitRequests atomic.Int32 + backend := limitsTestBackend(t, "proj_demo", 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + w.WriteHeader(http.StatusNotFound) + limitsTestWrite(t, w, `{"error":{"message":"synthetic missing project"}}`) + } else if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitRequests.Add(1) + w.WriteHeader(http.StatusNotFound) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + snapshot, err := backend.Load(t.Context()) + require.Error(t, err) + require.Zero(t, limitRequests.Load()) + require.False(t, snapshot.LimitAbsent) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) +} + +func TestLimitsServicePaginationCapsAndCycle(t *testing.T) { + for _, test := range []struct { + maximum int64 + count int + complete, failure bool + requests int32 + }{ + {0, 0, false, false, 1}, {1, 1, false, false, 1}, {-1, 2, false, true, 2}, + } { + t.Run(fmt.Sprint(test.maximum), func(t *testing.T) { + var requests atomic.Int32 + backend := limitsTestBackend(t, "", test.maximum, func(w http.ResponseWriter, r *http.Request) { + requests.Add(1) + if test.maximum == 0 && r.URL.Query().Get("limit") != "1" { + t.Error("zero cap must inspect one-record page") + } + limitsTestWrite(t, w, `{"object":"list","data":[`+limitsTestAlert+`],"has_more":true,"last_id":"alert_demo"}`) + }) + items, complete, err := backend.list(t.Context(), false) + require.Len(t, items, test.count) + require.Equal(t, test.complete, complete) + require.Equal(t, test.failure, err != nil) + require.Equal(t, test.requests, requests.Load()) + }) + } +} + +func TestLimitsServiceRejectsStaleLimitAndDuplicateBaselineWithoutWriting(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + } + limitsTestWrite(t, w, strings.Replace(limitsTestLimit, "10000", "20000", 1)) + }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "limit_set", Amount: 30000, Before: gjson.Parse(limitsTestLimit)}) + require.ErrorContains(t, err, "configuration changed") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.False(t, failure.Attempted) + duplicate := strings.Replace(limitsTestAlert, `"type":"email"`, `"type":"email","type":"email"`, 1) + _, err = backend.Mutate(t.Context(), limitsChange{Kind: "alert_update", ID: "alert_demo", Amount: 300, + Recipients: []string{"finance@example.test"}, Before: gjson.Parse(duplicate)}) + require.ErrorContains(t, err, "duplicate") + require.Zero(t, writes.Load()) +} + +func TestLimitsServiceAlertPrefixPresenceAndNoMutationRetries(t *testing.T) { + empty, subject := "", "synthetic subject" + for _, test := range []struct { + name string + set bool + value *string + want string + }{ + {"absent", false, nil, ""}, {"null", true, nil, "null"}, {"empty", true, &empty, `""`}, {"text", true, &subject, `"synthetic subject"`}, + } { + t.Run(test.name, func(t *testing.T) { + var writes atomic.Int32 + var sent []byte + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + limitsTestWrite(t, w, limitsTestAlert) + return + } + writes.Add(1) + var err error + sent, err = io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + w.WriteHeader(http.StatusInternalServerError) + limitsTestWrite(t, w, `{"error":{"message":"synthetic failed write"}}`) + }) + backend.client = openai.NewClient(append(backend.client.Options, option.WithMaxRetries(3))...) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "alert_update", ID: "alert_demo", Before: gjson.Parse(limitsTestAlert), + Amount: 0, Recipients: []string{"ops@example.test"}, SubjectPrefixSet: test.set, SubjectPrefix: test.value}) + require.Error(t, err) + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.True(t, failure.Attempted) + require.Equal(t, int32(1), writes.Load()) + value := gjson.ParseBytes(sent) + require.Equal(t, "0", value.Get("threshold_amount").Raw) + require.Equal(t, test.want, value.Get("notification_channel.subject_prefix").Raw) + }) + } +} + +func TestLimitsServiceRateUpdatePreservesZeroNegativeAndUnknownResponseFields(t *testing.T) { + var body []byte + backend := limitsTestBackend(t, "proj_demo", 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + } else if r.Method == http.MethodGet { + limitsTestWrite(t, w, `{"object":"list","data":[`+limitsTestRate+`],"has_more":false}`) + } else { + var err error + body, err = io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + var updated map[string]json.RawMessage + var changed map[string]json.RawMessage + if err := json.Unmarshal([]byte(limitsTestRate), &updated); err != nil { + t.Error(err) + } + if err := json.Unmarshal(body, &changed); err != nil { + t.Error(err) + } + for key, value := range changed { + updated[key] = value + } + encoded, err := json.Marshal(updated) + if err != nil { + t.Error(err) + } + limitsTestWrite(t, w, string(encoded)) + } + }) + value, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(limitsTestRate), + RateValues: map[string]int64{"max_requests_per_1_minute": 0, "max_tokens_per_1_minute": -1, + "max_requests_per_1_day": 123, "max_images_per_1_minute": 4, + "max_audio_megabytes_per_1_minute": 5, "batch_1_day_max_input_tokens": 9007199254740993}}) + require.NoError(t, err) + require.Equal(t, "9007199254740993", gjson.GetBytes(body, "batch_1_day_max_input_tokens").Raw) + require.JSONEq(t, `{"max_requests_per_1_minute":0,"max_tokens_per_1_minute":-1,"max_requests_per_1_day":123,"max_images_per_1_minute":4,"max_audio_megabytes_per_1_minute":5,"batch_1_day_max_input_tokens":9007199254740993}`, string(body)) + require.Equal(t, "0.1234567890123456789", value.Get("future.precision").Raw) +} + +func TestLimitsServiceCancellationAndRepeatedSubmission(t *testing.T) { + started := make(chan struct{}) + release := make(chan struct{}) + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + limitsTestWrite(t, w, limitsTestLimit) + return + } + close(started) + select { + case <-r.Context().Done(): + case <-release: + } + }) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + change := limitsChange{Kind: "limit_set", Amount: 20000, Before: gjson.Parse(limitsTestLimit)} + finished := make(chan error, 1) + go func() { _, err := backend.Mutate(ctx, change); finished <- err }() + <-started + _, err := backend.Mutate(t.Context(), change) + require.ErrorContains(t, err, "already in progress") + cancel() + close(release) + require.ErrorIs(t, <-finished, context.Canceled) +} + +func TestLimitsSnapshotErrorEnvelopeNeverIncludesProviderText(t *testing.T) { + snapshot := limitsSnapshot{Errors: []limitsSourceFailure{ + {Resource: "spend_limit", Err: context.Canceled}, + {Resource: "spend_alerts", Err: errors.New("synthetic-private-url")}, + }} + value, err := snapshot.JSON() + require.NoError(t, err) + require.True(t, json.Valid([]byte(value.Raw))) + require.Equal(t, "canceled", value.Get("source_errors.0.status").Str) + require.Equal(t, "request_failed", value.Get("source_errors.1.status").Str) + require.NotContains(t, value.Raw, "synthetic-private-url") +} + +func TestLimitsBackendConstructorPreservesRequestContextAndTLSClient(t *testing.T) { + requests := make(chan http.Header, 3) + server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests <- r.Header.Clone() + w.Header().Set("Content-Type", "application/json") + if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitsTestWrite(t, w, limitsTestLimit) + } else if strings.HasSuffix(r.URL.Path, "/costs") { + limitsTestWrite(t, w, `{"object":"page","data":[],"has_more":false,"next_page":null}`) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + })) + t.Cleanup(server.Close) + root := &cli.Command{Name: "openai", Metadata: map[string]any{requestconfig.MTLSHTTPClientMetadata: server.Client()}, + Flags: []cli.Flag{&cli.StringFlag{Name: "base-url"}, &cli.StringFlag{Name: "api-key"}, + &cli.StringFlag{Name: "admin-api-key"}, &cli.StringFlag{Name: "organization"}, + &cli.StringFlag{Name: "project"}, NewRequestHeaderFlag(), &cli.Int64Flag{Name: "max-items", Value: 100}}, + Action: func(ctx context.Context, cmd *cli.Command) error { + backend, err := newLimitsBackend(cmd, false) + if err != nil { + return err + } + _, err = backend.Load(ctx) + return err + }} + require.NoError(t, root.Run(t.Context(), []string{"openai", "--base-url", server.URL, + "--api-key", "synthetic-project-key", "--admin-api-key", "synthetic-admin-key", + "--organization", "org_demo", "--project", "proj_header", "--header", "X-Limits-Demo: preserved"})) + for range 3 { + header := <-requests + require.Equal(t, "Bearer synthetic-admin-key", header.Get("Authorization")) + require.Equal(t, "org_demo", header.Get("OpenAI-Organization")) + require.Equal(t, "proj_header", header.Get("OpenAI-Project")) + require.Equal(t, "preserved", header.Get("X-Limits-Demo")) + } +} + +func TestLimitsServiceRefusesIncompleteRatePreflight(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "proj_demo", 1, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + } + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[`+limitsTestRate+`],"has_more":true,"last_id":"rate_demo"}`) + } + }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "rate_update", ID: "rate_demo", Before: gjson.Parse(limitsTestRate), + RateValues: map[string]int64{"max_requests_per_1_minute": 1}}) + require.ErrorContains(t, err, "complete list") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.False(t, failure.Attempted) + require.Zero(t, writes.Load()) +} + +func TestLimitsServiceMutationOperationsUseTypedRoutes(t *testing.T) { + for _, project := range []bool{false, true} { + for _, kind := range []string{"limit_set", "limit_delete", "alert_create", "alert_update", "alert_delete"} { + t.Run(fmt.Sprintf("%t/%s", project, kind), func(t *testing.T) { + projectID, scope, prefix := "", "organization", "/organization" + if project { + projectID, scope, prefix = "proj_demo", "project", "/organization/projects/proj_demo" + } + limit := strings.ReplaceAll(limitsTestLimit, "organization.spend_limit", scope+".spend_limit") + alert := strings.ReplaceAll(limitsTestAlert, "organization.spend_alert", scope+".spend_alert") + type request struct{ method, path, body string } + sent := make(chan request, 1) + backend := limitsTestBackend(t, projectID, 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + switch r.URL.Path { + case "/organization/projects/proj_demo": + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + case prefix + "/spend_limit": + limitsTestWrite(t, w, limit) + case prefix + "/spend_alerts/alert_demo": + limitsTestWrite(t, w, alert) + default: + t.Errorf("unexpected read: %s", r.URL.Path) + w.WriteHeader(400) + } + return + } + body, err := io.ReadAll(r.Body) + if err != nil { + t.Error(err) + } + sent <- request{r.Method, r.URL.Path, string(body)} + switch kind { + case "limit_set": + limitsTestWrite(t, w, strings.Replace(limit, `"threshold_amount":10000`, `"threshold_amount":12345`, 1)) + case "alert_create", "alert_update": + limitsTestWrite(t, w, strings.Replace(alert, `"threshold_amount":100`, `"threshold_amount":12345`, 1)) + case "limit_delete": + limitsTestWrite(t, w, `{"object":"`+scope+`.spend_limit.deleted","deleted":true}`) + case "alert_delete": + limitsTestWrite(t, w, `{"id":"alert_demo","object":"`+scope+`.spend_alert.deleted","deleted":true}`) + } + }) + change := limitsChange{Kind: kind, Amount: 12345, Before: gjson.Parse(limit)} + path, method := prefix+"/spend_limit", http.MethodPost + if strings.HasPrefix(kind, "alert_") { + change.Before, change.ID = gjson.Parse(alert), "alert_demo" + change.Recipients = []string{"finance@example.test"} + path = prefix + "/spend_alerts/alert_demo" + if kind == "alert_create" { + path = prefix + "/spend_alerts" + change.Before, change.ID = gjson.Result{}, "" + } + } + if strings.HasSuffix(kind, "_delete") { + method = http.MethodDelete + } + _, err := backend.Mutate(t.Context(), change) + require.NoError(t, err) + got := <-sent + require.Equal(t, method, got.method) + require.Equal(t, path, got.path) + if method == http.MethodPost { + require.Equal(t, "12345", gjson.Get(got.body, "threshold_amount").Raw) + require.Equal(t, "USD", gjson.Get(got.body, "currency").Str) + require.Equal(t, "month", gjson.Get(got.body, "interval").Str) + } else { + require.Empty(t, got.body) + } + }) + } + } +} + +func TestLimitsServiceIncompleteMutationConfirmationIsUncertain(t *testing.T) { + for _, test := range []struct{ kind, response string }{ + {"limit_set", `{}`}, + {"limit_set", `{"object":"project.spend_limit","threshold_amount":100,"currency":"USD","interval":"month"}`}, + {"alert_create", `{"object":"organization.spend_alert","threshold_amount":100,"currency":"USD","interval":"month"}`}, + {"alert_update", strings.ReplaceAll(limitsTestAlert, "alert_demo", "alert_other")}, + {"limit_delete", `{"object":"organization.spend_limit.deleted","deleted":false}`}, + {"alert_delete", `{"id":"alert_other","object":"organization.spend_alert.deleted","deleted":true}`}, + } { + t.Run(test.kind+test.response, func(t *testing.T) { + var writes atomic.Int32 + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + writes.Add(1) + limitsTestWrite(t, w, test.response) + return + } + if strings.Contains(r.URL.Path, "spend_alerts") { + limitsTestWrite(t, w, limitsTestAlert) + } else { + limitsTestWrite(t, w, limitsTestLimit) + } + }) + change := limitsChange{Kind: test.kind, Amount: 100, Before: gjson.Parse(limitsTestLimit)} + if strings.HasPrefix(test.kind, "alert_") { + change.Before, change.ID, change.Recipients = gjson.Parse(limitsTestAlert), "alert_demo", []string{"finance@example.test"} + } + _, err := backend.Mutate(t.Context(), change) + require.ErrorContains(t, err, "incomplete confirmation") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.True(t, failure.Attempted) + require.Equal(t, int32(1), writes.Load()) + }) + } +} + +func TestLimitsServiceWrongDiscriminatorRemainsUnavailable(t *testing.T) { + for _, project := range []bool{false, true} { + t.Run(fmt.Sprint(project), func(t *testing.T) { + projectID := "" + if project { + projectID = "proj_demo" + } + var limitRequests atomic.Int32 + backend := limitsTestBackend(t, projectID, 100, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/proj_demo"): + limitsTestWrite(t, w, `{"id":"proj_demo","object":"project.spend_limit","future":123}`) + case strings.HasSuffix(r.URL.Path, "/spend_limit"): + limitRequests.Add(1) + limitsTestWrite(t, w, `{"object":"project.spend_limit","threshold_amount":10000,"future":123}`) + default: + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + snapshot, err := backend.Load(t.Context()) + require.Error(t, err) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) + require.Equal(t, "request_failed", value.Get("source_errors.0.status").Str) + if project { + require.Zero(t, limitRequests.Load()) + require.Equal(t, "project", value.Get("source_errors.0.resource").Str) + require.Equal(t, "123", value.Get("project.future").Raw) + } else { + require.Equal(t, int32(1), limitRequests.Load()) + require.Equal(t, "spend_limit", value.Get("source_errors.0.resource").Str) + require.Equal(t, "123", value.Get("spend_limit.future").Raw) + } + }) + } +} + +func TestLimitsSnapshotProjectFailureOverridesPriorLimitPresence(t *testing.T) { + for _, absent := range []bool{false, true} { + snapshot := limitsSnapshot{ + ProjectID: "proj_demo", Limit: gjson.Parse(limitsTestLimit), LimitAbsent: absent, + Errors: []limitsSourceFailure{{Resource: "project", Err: errors.New("synthetic project failure")}}, + } + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) + require.Equal(t, "10000", value.Get("spend_limit.threshold_amount").Raw) + } +} + +func TestLimitsBackendValidationUsesSafeLocalErrors(t *testing.T) { + for _, test := range []struct { + project bool + args []string + message string + }{ + {false, []string{"openai", "--max-items=-2"}, "--max-items must be -1, zero, or a positive integer."}, + {true, []string{"openai"}, "Set --project-id to inspect project limits."}, + } { + var validationErr error + root := &cli.Command{ + Name: "openai", Flags: []cli.Flag{&cli.Int64Flag{Name: "max-items", Value: 100}, &cli.StringFlag{Name: "project-id"}}, + Action: func(_ context.Context, cmd *cli.Command) error { + _, validationErr = newLimitsBackend(cmd, test.project) + return nil + }, + } + require.NoError(t, root.Run(t.Context(), test.args)) + var localErr *localcommands.LocalError + require.ErrorAs(t, validationErr, &localErr) + require.EqualError(t, validationErr, test.message) + } +} + +func limitsTestCostsBackend(t *testing.T, project string, maximum int64, costs http.HandlerFunc) *limitsAPIBackend { + t.Helper() + backend := limitsTestBackend(t, project, maximum, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/costs"): + costs(w, r) + case strings.HasSuffix(r.URL.Path, "/proj_demo"): + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + case strings.HasSuffix(r.URL.Path, "/spend_limit"): + limit := limitsTestLimit + if project != "" { + limit = strings.ReplaceAll(limit, "organization.spend_limit", "project.spend_limit") + } + limitsTestWrite(t, w, limit) + default: + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + backend.now = func() time.Time { return time.Date(2026, 10, 19, 12, 30, 0, 0, time.UTC) } + return backend +} + +func limitsTestCostBucket(start int64, results string) string { + return fmt.Sprintf(`{"object":"bucket","start_time":%d,"end_time":%d,"results":[%s],"future":{"exact":0.1234567890123456789}}`, start, start+86400, results) +} + +func limitsTestCostResult(amount, currency, project string) string { + return fmt.Sprintf(`{"object":"organization.costs.result","amount":{"value":%s,"currency":%q},"project_id":%q}`, amount, currency, project) +} + +func TestLimitsCostsExactPaginationAndUTCProjectScope(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + end := time.Date(2026, 10, 20, 0, 0, 0, 0, time.UTC).Unix() + for _, project := range []string{"", "proj_demo"} { + t.Run("project="+project, func(t *testing.T) { + var calls atomic.Int32 + backend := limitsTestCostsBackend(t, project, 100, func(w http.ResponseWriter, r *http.Request) { + calls.Add(1) + if r.Method != http.MethodGet || r.URL.Query().Get("start_time") != fmt.Sprint(start) || + r.URL.Query().Get("end_time") != fmt.Sprint(end) || r.URL.Query().Get("bucket_width") != "1d" { + t.Error("Costs request changed its method or UTC period") + } + if got := r.URL.Query().Get("project_ids[]"); got != project { + t.Errorf("project filter = %q, want %q", got, project) + } + if r.URL.Query().Get("group_by[]") != "" { + t.Error("Costs summary unexpectedly grouped results") + } + if r.URL.Query().Get("page") == "" { + bucket := limitsTestCostBucket(start, limitsTestCostResult("9007199254740993.000000000000000001", "usd", project)) + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":true,"next_page":"next_demo"}`) + } else { + if r.URL.Query().Get("page") != "next_demo" { + t.Error("Costs did not retain its cursor") + } + bucket := limitsTestCostBucket(start+86400, limitsTestCostResult("2e-1", "USD", project)+","+limitsTestCostResult("-0.1", "usd", project)) + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":false,"next_page":null}`) + } + }) + snapshot, err := backend.Load(t.Context()) + require.NoError(t, err) + require.True(t, snapshot.CostsComplete) + require.Equal(t, "9007199254740993.100000000000000001", snapshot.CostsTotal) + require.Equal(t, int32(2), calls.Load()) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, start, value.Get("costs.start_time").Int()) + require.Equal(t, end, value.Get("costs.end_time").Int()) + require.Equal(t, "UTC", value.Get("costs.timezone").Str) + require.Equal(t, snapshot.CostsTotal, value.Get("costs.total").Str) + require.Equal(t, "0.1234567890123456789", value.Get("costs.buckets.0.future.exact").Raw) + }) + } +} + +func TestLimitsCostsBoundsAndLaterFailuresPreservePartialData(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + for _, test := range []struct { + name string + maximum int64 + mode string + calls int32 + wantTotal string + wantError bool + }{ + {"zero", 0, "", 1, "", false}, + {"bound", 1, "", 1, "0.10", false}, + {"later HTTP error", -1, "failure", 2, "0.10", true}, + {"cursor cycle", -1, "cycle", 2, "0.30", true}, + {"missing cursor", -1, "missing", 1, "0.10", true}, + {"empty continuation", -1, "empty", 2, "0.10", true}, + } { + t.Run(test.name, func(t *testing.T) { + var calls atomic.Int32 + backend := limitsTestCostsBackend(t, "", test.maximum, func(w http.ResponseWriter, r *http.Request) { + call := calls.Add(1) + if test.maximum == 0 && r.URL.Query().Get("limit") != "1" { + t.Error("zero maximum must make a one-bucket request") + } + if call == 2 && test.mode == "failure" { + w.WriteHeader(http.StatusForbidden) + limitsTestWrite(t, w, `{"error":{"message":"synthetic-private-cost-detail"}}`) + return + } + cursor := `"next_demo"` + if test.mode == "missing" { + cursor = "null" + } + data := limitsTestCostBucket(start+int64(call-1)*86400, limitsTestCostResult(fmt.Sprintf("0.%d", call), "usd", "")) + if call == 2 && test.mode == "empty" { + data = "" + } + limitsTestWrite(t, w, `{"object":"page","data":[`+data+`],"has_more":true,"next_page":`+cursor+`}`) + }) + snapshot, err := backend.Load(t.Context()) + require.Equal(t, test.wantError, err != nil) + require.Equal(t, test.calls, calls.Load()) + require.False(t, snapshot.CostsComplete) + require.Equal(t, test.wantTotal, snapshot.CostsTotal) + require.True(t, snapshot.Limit.Exists()) + value, jsonErr := snapshot.JSON() + require.NoError(t, jsonErr) + require.NotContains(t, value.Raw, "synthetic-private-cost-detail") + if test.mode == "failure" { + require.Equal(t, "api_error", value.Get("source_errors.0.status").Str) + require.Equal(t, int64(403), value.Get("source_errors.0.status_code").Int()) + } + }) + } +} + +func TestLimitsCostsSummaryFailuresRetainRawBuckets(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + for _, test := range []struct{ name, result, reason string }{ + {"huge exponent", limitsTestCostResult("1e9223372036854775807", "usd", ""), "unsupported_precision"}, + {"overflow exponent", limitsTestCostResult("1e999999999999999999999999999999", "usd", ""), "unsupported_precision"}, + {"currency", limitsTestCostResult("0.1", "EUR", ""), "unsupported_currency"}, + {"missing amount", `{"object":"organization.costs.result","amount":{"currency":"usd"}}`, "invalid_amount"}, + {"duplicate amount", `{"object":"organization.costs.result","amount":{"currency":"usd","value":0.1,"value":0.2}}`, "ambiguous_fields"}, + {"wrong result", `{"object":"organization.usage.completions.result","amount":{"currency":"usd","value":0.1}}`, "unexpected_result"}, + } { + t.Run(test.name, func(t *testing.T) { + bucket := limitsTestCostBucket(start, test.result) + backend := limitsTestCostsBackend(t, "", 100, func(w http.ResponseWriter, _ *http.Request) { + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":false,"next_page":null}`) + }) + snapshot, err := backend.Load(t.Context()) + var summary *limitsCostSummaryError + require.ErrorAs(t, err, &summary) + require.Equal(t, test.reason, summary.Reason) + require.ErrorContains(t, err, "costs.buckets") + require.False(t, snapshot.CostsComplete) + require.Empty(t, snapshot.CostsTotal) + require.Equal(t, bucket, snapshot.Costs[0].Raw) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("costs.total").Raw) + require.Equal(t, "summary_unavailable", value.Get("source_errors.0.status").Str) + require.Equal(t, test.reason, value.Get("source_errors.0.summary_reason").Str) + require.True(t, value.Get("spend_limit_configured").Bool()) + }) + } +} + +func TestLimitsUnknown404RemainsUnconfirmed(t *testing.T) { + for _, project := range []string{"", "proj_demo"} { + t.Run("project="+project, func(t *testing.T) { + backend := limitsTestBackend(t, project, 100, func(w http.ResponseWriter, r *http.Request) { + switch { + case strings.HasSuffix(r.URL.Path, "/proj_demo"): + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + case strings.HasSuffix(r.URL.Path, "/spend_limit"): + w.WriteHeader(http.StatusNotFound) + limitsTestWrite(t, w, `{"error":{"code":"not_found","message":"synthetic private unavailable state"}}`) + case strings.HasSuffix(r.URL.Path, "/costs"): + limitsTestWrite(t, w, `{"object":"page","data":[],"has_more":false,"next_page":null}`) + default: + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + }) + snapshot, err := backend.Load(t.Context()) + var apiErr *openai.Error + require.ErrorAs(t, err, &apiErr) + require.Equal(t, http.StatusNotFound, apiErr.StatusCode) + require.False(t, snapshot.LimitAbsent) + require.True(t, snapshot.LimitUnconfirmed) + value, err := snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "null", value.Get("spend_limit_configured").Raw) + require.Equal(t, int64(404), value.Get("source_errors.0.status_code").Int()) + }) + } +} + +func TestLimitsReviewedUnconfirmedSetRechecksBeforeWriting(t *testing.T) { + for _, test := range []struct { + name string + status int + project bool + projectMissing bool + write bool + }{ + {"organization still unconfirmed", 404, false, false, true}, + {"project still unconfirmed", 404, true, false, true}, + {"new configuration", 200, false, false, false}, + {"access changed", 403, false, false, false}, + {"project disappeared", 404, true, true, false}, + } { + t.Run(test.name, func(t *testing.T) { + var writes atomic.Int32 + project, scope := "", "organization" + if test.project { + project, scope = "proj_demo", "project" + } + backend := limitsTestBackend(t, project, 100, func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/proj_demo") { + if test.projectMissing { + w.WriteHeader(http.StatusNotFound) + limitsTestWrite(t, w, `{"error":{"message":"missing"}}`) + return + } + limitsTestWrite(t, w, `{"id":"proj_demo","object":"organization.project"}`) + return + } + if r.Method == http.MethodGet { + w.WriteHeader(test.status) + if test.status == 200 { + limitsTestWrite(t, w, limitsTestLimit) + } else { + limitsTestWrite(t, w, `{"error":{"code":"not_found","message":"unconfirmed"}}`) + } + return + } + writes.Add(1) + limitsTestWrite(t, w, `{"object":"`+scope+`.spend_limit","threshold_amount":12345,"currency":"USD","interval":"month"}`) + }) + _, err := backend.Mutate(t.Context(), limitsChange{Kind: "limit_set", Amount: 12345, BeforeMissing: true}) + if test.write { + require.NoError(t, err) + require.Equal(t, int32(1), writes.Load()) + } else { + require.Error(t, err) + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.False(t, failure.Attempted) + require.Zero(t, writes.Load()) + } + }) + } +} + +func TestLimitsMismatchedMutationKeepsReturnedRecordAndUncertainty(t *testing.T) { + var writes atomic.Int32 + response := strings.TrimSuffix(limitsTestLimit, "}") + `,"future":{"exact":123456789012345678901234567890}}` + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + limitsTestWrite(t, w, limitsTestLimit) + return + } + writes.Add(1) + limitsTestWrite(t, w, response) + }) + value, err := backend.Mutate(t.Context(), limitsChange{Kind: "limit_set", Amount: 12345, Before: gjson.Parse(limitsTestLimit)}) + require.ErrorContains(t, err, "different values") + var failure *limitsMutationFailure + require.ErrorAs(t, err, &failure) + require.True(t, failure.Attempted) + require.Equal(t, int32(1), writes.Load()) + require.Equal(t, response, value.Raw) +} + +func TestLimitsCostsCancellationRetainsEarlierBuckets(t *testing.T) { + start := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC).Unix() + started := make(chan struct{}) + backend := limitsTestCostsBackend(t, "", -1, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Query().Get("page") == "" { + bucket := limitsTestCostBucket(start, limitsTestCostResult("0.1", "usd", "")) + limitsTestWrite(t, w, `{"object":"page","data":[`+bucket+`],"has_more":true,"next_page":"next_demo"}`) + return + } + close(started) + <-r.Context().Done() + }) + type outcome struct { + snapshot limitsSnapshot + err error + } + finished := make(chan outcome, 1) + ctx, cancel := context.WithCancel(t.Context()) + t.Cleanup(cancel) + go func() { snapshot, err := backend.Load(ctx); finished <- outcome{snapshot, err} }() + <-started + cancel() + result := <-finished + require.ErrorIs(t, result.err, context.Canceled) + require.Len(t, result.snapshot.Costs, 1) + require.False(t, result.snapshot.CostsComplete) + require.Empty(t, result.snapshot.CostsTotal) + value, err := result.snapshot.JSON() + require.NoError(t, err) + require.Equal(t, "canceled", value.Get("source_errors.0.status").Str) + require.Equal(t, "0.1", value.Get("costs.buckets.0.results.0.amount.value").Raw) +} diff --git a/pkg/custom/limits_values.go b/pkg/custom/limits_values.go new file mode 100644 index 00000000..6ee70bb4 --- /dev/null +++ b/pkg/custom/limits_values.go @@ -0,0 +1,443 @@ +package custom + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "math" + "math/big" + "net/mail" + "strconv" + "strings" + + "github.com/tidwall/gjson" +) + +// parseLimitsDollars converts explicit decimal USD input to integer cents. +func parseLimitsDollars(value string, minimum int64) (int64, error) { + invalid := fmt.Errorf("Enter a USD amount using digits and at most two decimal places.") + if value == "" { + return 0, invalid + } + dollars, cents, decimal := strings.Cut(value, ".") + if dollars == "" || decimal && (cents == "" || len(cents) > 2) { + return 0, invalid + } + for _, part := range []string{dollars, cents} { + for i := range len(part) { + if part[i] < '0' || part[i] > '9' { + return 0, invalid + } + } + } + whole, err := strconv.ParseUint(dollars, 10, 64) + if err != nil || whole > uint64(math.MaxInt64)/100 { + return 0, fmt.Errorf("The USD amount is too large.") + } + var fraction uint64 + if cents != "" { + fraction = uint64(cents[0]-'0') * 10 + if len(cents) == 2 { + fraction += uint64(cents[1] - '0') + } + } + if whole*100 > uint64(math.MaxInt64)-fraction { + return 0, fmt.Errorf("The USD amount is too large.") + } + result := int64(whole*100 + fraction) + if result < minimum { + return 0, fmt.Errorf("The USD amount must be at least %s.", formatLimitsDollars(minimum)) + } + return result, nil +} + +func formatLimitsDollars(cents int64) string { + digits := strconv.FormatInt(cents, 10) + sign := "" + if strings.HasPrefix(digits, "-") { + sign, digits = "-", digits[1:] + } + for len(digits) < 3 { + digits = "0" + digits + } + return sign + digits[:len(digits)-2] + "." + digits[len(digits)-2:] +} + +func parseLimitsRecipients(value string) ([]string, error) { + parts := strings.Split(value, ",") + recipients := make([]string, 0, len(parts)) + for _, part := range parts { + address := strings.TrimSpace(part) + parsed, err := mail.ParseAddress(address) + if err != nil || parsed.Name != "" || parsed.Address != address || strings.ContainsAny(address, "\r\n") { + return nil, fmt.Errorf("Enter email addresses separated by commas, without display names.") + } + recipients = append(recipients, address) + } + return recipients, nil +} + +func limitsUniqueObject(value gjson.Result) error { + return limitsUniqueObjectContext(context.Background(), value) +} + +func limitsUniqueObjectContext(ctx context.Context, value gjson.Result) error { + seen := map[string]bool{} + var err error + value.ForEach(func(key, _ gjson.Result) bool { + if err = ctx.Err(); err != nil { + return false + } + if seen[key.Str] { + err = errors.New("The response has duplicate fields. Use the original API command to inspect it.") + return false + } + seen[key.Str] = true + return true + }) + if ctx.Err() != nil { + return ctx.Err() + } + return err +} + +func limitsUniqueJSON(value gjson.Result) error { + return limitsUniqueJSONContext(context.Background(), value) +} + +func limitsUniqueJSONContext(ctx context.Context, value gjson.Result) error { + pending := []gjson.Result{value} + for len(pending) > 0 { + if err := ctx.Err(); err != nil { + return err + } + item := pending[len(pending)-1] + pending = pending[:len(pending)-1] + if item.IsObject() { + if err := limitsUniqueObjectContext(ctx, item); err != nil { + return err + } + } + if item.IsObject() || item.IsArray() { + item.ForEach(func(_, field gjson.Result) bool { + if ctx.Err() != nil { + return false + } + if field.IsObject() || field.IsArray() { + pending = append(pending, field) + } + return true + }) + } + } + return ctx.Err() +} + +func limitsKnownFields(value gjson.Result, fields ...string) bool { + known := make(map[string]bool, len(fields)) + for _, field := range fields { + known[field] = true + } + valid := true + value.ForEach(func(key, _ gjson.Result) bool { + valid = known[key.Str] + return valid + }) + return valid +} + +func limitsInteger(value gjson.Result) (int64, error) { + if value.Type != gjson.Number { + return 0, errors.New("The returned amount or rate is not an integer.") + } + integer, err := strconv.ParseInt(value.Raw, 10, 64) + if err != nil { + return 0, errors.New("The returned amount or rate cannot be represented by this form.") + } + return integer, nil +} + +func limitsEditableBaseline(value gjson.Result, kind string) error { + return limitsEditableBaselineContext(context.Background(), value, kind) +} + +func limitsEditableBaselineContext(ctx context.Context, value gjson.Result, kind string) error { + if !value.IsObject() || !gjson.Valid(value.Raw) { + return errors.New("The returned configuration cannot be edited by this form.") + } + if err := limitsUniqueJSONContext(ctx, value); err != nil { + return err + } + object := value.Get("object") + if object.Type != gjson.String { + return errors.New("The returned configuration has no recognized object type.") + } + switch kind { + case "limit_set", "limit_delete": + if object.Str != "organization.spend_limit" && object.Str != "project.spend_limit" { + return errors.New("The returned object is not a spend limit.") + } + if kind == "limit_delete" { + return nil + } + if !limitsKnownFields(value, "object", "threshold_amount", "currency", "interval", "enforcement") { + return errors.New("This spend limit contains unfamiliar configuration. Use the original API command to edit it.") + } + enforcement := value.Get("enforcement") + if enforcement.Exists() && enforcement.Type != gjson.Null { + status := enforcement.Get("status") + if !enforcement.IsObject() || !limitsKnownFields(enforcement, "status") || status.Type != gjson.String || + status.Str != "inactive" && status.Str != "enforcing" { + return errors.New("This spend limit has unfamiliar enforcement configuration. Use the original API command to edit it.") + } + } + case "alert_update", "alert_delete": + if object.Str != "organization.spend_alert" && object.Str != "project.spend_alert" { + return errors.New("The returned object is not a spend alert.") + } + if value.Get("id").Type != gjson.String || value.Get("id").Str == "" { + return errors.New("The returned spend alert has no identifier.") + } + if kind == "alert_delete" { + return nil + } + if !limitsKnownFields(value, "id", "object", "threshold_amount", "currency", "interval", "notification_channel") { + return errors.New("This alert contains unfamiliar configuration. Use the original API command to edit it.") + } + channel := value.Get("notification_channel") + if !channel.IsObject() || !limitsKnownFields(channel, "type", "recipients", "subject_prefix") || channel.Get("type").Str != "email" { + return errors.New("This alert has unfamiliar notification settings. Use the original API command to edit it.") + } + recipients := channel.Get("recipients") + if !recipients.IsArray() || len(recipients.Array()) == 0 { + return errors.New("This alert has no editable email recipients.") + } + for _, address := range recipients.Array() { + if err := ctx.Err(); err != nil { + return err + } + if address.Type != gjson.String { + return errors.New("This alert has an unrecognized email recipient.") + } + parsed, err := parseLimitsRecipients(address.Str) + if err != nil || len(parsed) != 1 || parsed[0] != address.Str { + return errors.New("This alert has an unrecognized email recipient.") + } + } + prefix := channel.Get("subject_prefix") + if prefix.Exists() && prefix.Type != gjson.Null && prefix.Type != gjson.String { + return errors.New("This alert has an unrecognized email subject prefix.") + } + case "rate_update": + if object.Str != "project.rate_limit" || value.Get("id").Type != gjson.String || value.Get("id").Str == "" || + value.Get("model").Type != gjson.String || value.Get("model").Str == "" { + return errors.New("The returned rate limit has no recognized identity.") + } + for _, field := range limitsRateFields { + item := value.Get(field) + if item.Exists() || field == "max_requests_per_1_minute" || field == "max_tokens_per_1_minute" { + if _, err := limitsInteger(item); err != nil { + return err + } + } + } + return nil + default: + return errors.New("Unknown limits edit operation.") + } + if value.Get("currency").Str != "USD" || value.Get("interval").Str != "month" { + return errors.New("This form can edit only returned monthly USD configurations.") + } + _, err := limitsInteger(value.Get("threshold_amount")) + return err +} + +var limitsRateFields = []string{ + "max_requests_per_1_minute", "max_tokens_per_1_minute", "max_requests_per_1_day", + "max_images_per_1_minute", "max_audio_megabytes_per_1_minute", "batch_1_day_max_input_tokens", +} + +func limitsValidateChange(change limitsChange) error { + return limitsValidateChangeContext(context.Background(), change) +} + +func limitsValidateChangeContext(ctx context.Context, change limitsChange) error { + if err := ctx.Err(); err != nil { + return err + } + if change.BeforeMissing && change.Before.Exists() { + return errors.New("The displayed configuration has an inconsistent presence state. Refresh before editing.") + } + switch change.Kind { + case "limit_set": + if change.Amount < 1 { + return errors.New("A hard spend limit must be at least USD 0.01.") + } + case "alert_create", "alert_update": + if change.Amount < 0 { + return errors.New("An alert threshold cannot be negative.") + } + if len(change.Recipients) == 0 { + return errors.New("An alert requires at least one email recipient.") + } + for _, address := range change.Recipients { + if err := ctx.Err(); err != nil { + return err + } + parsed, err := parseLimitsRecipients(address) + if err != nil || len(parsed) != 1 || parsed[0] != address { + return errors.New("Enter valid email recipients without display names.") + } + } + case "rate_update": + if len(change.RateValues) == 0 { + return errors.New("Set at least one rate-limit value.") + } + for key := range change.RateValues { + known := false + for _, field := range limitsRateFields { + known = known || key == field + } + if !known { + return errors.New("This form does not support the requested rate-limit field.") + } + } + case "limit_delete", "alert_delete": + default: + return errors.New("Unknown limits operation.") + } + if change.Kind == "alert_create" || change.Kind == "limit_set" && change.BeforeMissing { + return nil + } + if err := limitsEditableBaselineContext(ctx, change.Before, change.Kind); err != nil { + return err + } + if change.Kind == "alert_update" || change.Kind == "alert_delete" || change.Kind == "rate_update" { + if change.ID == "" || change.Before.Get("id").Str != change.ID { + return errors.New("The selected identifier does not match the displayed configuration.") + } + } + return nil +} + +func limitsCanonicalJSON(value gjson.Result) (string, error) { + return limitsCanonicalJSONContext(context.Background(), value) +} + +func limitsCanonicalJSONContext(ctx context.Context, value gjson.Result) (string, error) { + if err := limitsUniqueJSONContext(ctx, value); err != nil { + return "", err + } + decoder := json.NewDecoder(strings.NewReader(value.Raw)) + decoder.UseNumber() + var decoded any + if err := decoder.Decode(&decoded); err != nil { + return "", errors.New("The configuration cannot be compared safely.") + } + encoded, err := json.Marshal(decoded) + if ctx.Err() != nil { + return "", ctx.Err() + } + return string(encoded), err +} + +// limitsCostTotal summarizes raw USD amounts, never integer threshold cents. +// Unsupported records remain in the snapshot even when a total cannot be calculated. +func limitsCostTotal(ctx context.Context, buckets []gjson.Result, start, end int64, projectID string) (string, error) { + total := new(big.Rat) + precision := 2 + var ranges [][2]int64 + invalid := &limitsCostSummaryError{Reason: "invalid_bucket"} + for _, bucket := range buckets { + if err := limitsUniqueJSONContext(ctx, bucket); err != nil { + if ctx.Err() != nil { + return "", ctx.Err() + } + return "", &limitsCostSummaryError{Reason: "ambiguous_fields"} + } + if !bucket.IsObject() || bucket.Get("object").Str != "bucket" || !bucket.Get("results").IsArray() { + return "", invalid + } + from, fromErr := limitsInteger(bucket.Get("start_time")) + to, toErr := limitsInteger(bucket.Get("end_time")) + if fromErr != nil || toErr != nil || from < start || to > end || from >= to || to-from > 86400 { + return "", invalid + } + for _, previous := range ranges { + if from < previous[1] && previous[0] < to { + return "", &limitsCostSummaryError{Reason: "overlapping_buckets"} + } + } + ranges = append(ranges, [2]int64{from, to}) + var resultErr error + bucket.Get("results").ForEach(func(_, result gjson.Result) bool { + if resultErr = ctx.Err(); resultErr != nil { + return false + } + if !result.IsObject() || result.Get("object").Str != "organization.costs.result" { + resultErr = &limitsCostSummaryError{Reason: "unexpected_result"} + return false + } + if result.Get("amount.currency").Type != gjson.String || !strings.EqualFold(result.Get("amount.currency").Str, "USD") { + resultErr = &limitsCostSummaryError{Reason: "unsupported_currency"} + return false + } + project := result.Get("project_id") + if projectID != "" && project.Exists() && project.Type != gjson.Null && (project.Type != gjson.String || project.Str != projectID) { + resultErr = &limitsCostSummaryError{Reason: "unexpected_project"} + return false + } + amount, scale, err := limitsCostDecimal(result.Get("amount.value")) + if err != nil { + resultErr = err + return false + } + total.Add(total, amount) + precision = max(precision, scale) + return true + }) + if resultErr != nil { + return "", resultErr + } + } + if err := ctx.Err(); err != nil { + return "", err + } + return total.FloatString(precision), nil +} + +func limitsCostDecimal(value gjson.Result) (*big.Rat, int, error) { + invalid := &limitsCostSummaryError{Reason: "invalid_amount"} + if value.Type != gjson.Number || !gjson.Valid(value.Raw) { + return nil, 0, invalid + } + coefficient := value.Raw + exponent := int64(0) + if at := strings.IndexAny(coefficient, "eE"); at >= 0 { + var err error + exponent, err = strconv.ParseInt(coefficient[at+1:], 10, 64) + if err != nil { + return nil, 0, &limitsCostSummaryError{Reason: "unsupported_precision"} + } + coefficient = coefficient[:at] + } + // Bound only derived display arithmetic. Raw API records are never truncated or rejected. + const summaryDigits = 10000 + if len(coefficient) > summaryDigits || exponent > summaryDigits || exponent < -summaryDigits { + return nil, 0, &limitsCostSummaryError{Reason: "unsupported_precision"} + } + scale := int64(0) + if at := strings.IndexByte(coefficient, '.'); at >= 0 { + scale = int64(len(coefficient) - at - 1) + } + scale = max(0, scale-exponent) + if scale > summaryDigits || int64(len(coefficient))+max(0, exponent) > summaryDigits { + return nil, 0, &limitsCostSummaryError{Reason: "unsupported_precision"} + } + amount, ok := new(big.Rat).SetString(value.Raw) + if !ok { + return nil, 0, invalid + } + return amount, int(scale), nil +} diff --git a/pkg/custom/limits_values_test.go b/pkg/custom/limits_values_test.go new file mode 100644 index 00000000..0b66ea14 --- /dev/null +++ b/pkg/custom/limits_values_test.go @@ -0,0 +1,181 @@ +package custom + +import ( + "context" + "math" + "strings" + "testing" + + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +func TestLimitsDollarsExactBoundaries(t *testing.T) { + for _, test := range []struct { + input string + want int64 + }{ + {"0", 0}, {"0.00", 0}, {"0.01", 1}, {"1.2", 120}, {"00100.01", 10001}, + {"90071992547409.93", 9007199254740993}, {"92233720368547758.07", math.MaxInt64}, + } { + t.Run(test.input, func(t *testing.T) { + got, err := parseLimitsDollars(test.input, 0) + require.NoError(t, err) + require.Equal(t, test.want, got) + roundTrip, err := parseLimitsDollars(formatLimitsDollars(got), 0) + require.NoError(t, err) + require.Equal(t, got, roundTrip) + }) + } + for _, input := range []string{"", ".01", "1.", "1.001", "1.2.3", "+1", "-0", "-1", "$1", "1,000", "1e2", "NaN", " 1", "1\n", "12", "92233720368547758.08", "92233720368547759", "18446744073709551616"} { + t.Run("reject "+input, func(t *testing.T) { _, err := parseLimitsDollars(input, 0); require.Error(t, err) }) + } + _, err := parseLimitsDollars("0", 1) + require.Error(t, err) + require.Equal(t, "-92233720368547758.08", formatLimitsDollars(math.MinInt64)) + require.Equal(t, "-0.01", formatLimitsDollars(-1)) +} + +func TestLimitsRecipientsDoNotInterpretDisplayNamesOrControls(t *testing.T) { + got, err := parseLimitsRecipients("finance@example.test, ops+alerts@example.test") + require.NoError(t, err) + require.Equal(t, []string{"finance@example.test", "ops+alerts@example.test"}, got) + for _, input := range []string{"", "finance", "Finance ", "finance@example.test,", "finance@example.test\r\nBcc: other@example.test", "a@example.test\x1b"} { + _, err := parseLimitsRecipients(input) + require.Error(t, err, input) + } +} + +func TestLimitsEditBaselineRejectsAmbiguousOrUnrepresentableConfiguration(t *testing.T) { + valid := `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}` + require.NoError(t, limitsEditableBaseline(gjson.Parse(valid), "alert_update")) + for _, raw := range []string{ + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"],"recipients":["other@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"],"delivery_policy":{"mode":"unknown"}}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":9223372036854775808,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":1.0,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":1,"currency":"EUR","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"]}}`, + `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":1,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["finance@example.test"],"subject_prefix":false}}`, + } { + require.Error(t, limitsEditableBaseline(gjson.Parse(raw), "alert_update"), raw) + } + require.Error(t, limitsUniqueJSON(gjson.Parse(`{"records":[{"id":"a","\u0069d":"b"}]}`))) +} + +func TestLimitsBaselineComparisonRetainsPrecisionAndIgnoresObjectOrder(t *testing.T) { + a, err := limitsCanonicalJSON(gjson.Parse(`{"a":9007199254740993,"b":{"x":0.1234567890123456789}}`)) + require.NoError(t, err) + b, err := limitsCanonicalJSON(gjson.Parse(`{ "b": {"x": 0.1234567890123456789}, "a": 9007199254740993 }`)) + require.NoError(t, err) + require.Equal(t, a, b) + c, err := limitsCanonicalJSON(gjson.Parse(`{"a":9007199254740992,"b":{"x":0.1234567890123456789}}`)) + require.NoError(t, err) + require.NotEqual(t, a, c) +} + +func TestLimitsCostsExactDecimalValues(t *testing.T) { + for _, test := range []struct { + input, want string + scale int + }{ + {"0.1", "1/10", 1}, {"0.20", "1/5", 2}, {"-0.015", "-3/200", 3}, + {"9007199254740993.000000000000000001", "9007199254740993000000000000000001/1000000000000000000", 18}, + {"1.23e2", "123", 0}, {"1.23e-2", "123/10000", 4}, {"1E+2", "100", 0}, {"-0", "0", 0}, + } { + t.Run(test.input, func(t *testing.T) { + value, scale, err := limitsCostDecimal(gjson.Parse(test.input)) + require.NoError(t, err) + require.Equal(t, test.want, value.RatString()) + require.Equal(t, test.scale, scale) + }) + } + for _, input := range []string{"null", `"0.1"`, "true", `{}`, `[]`} { + _, _, err := limitsCostDecimal(gjson.Parse(input)) + require.Error(t, err) + } + for _, input := range []string{ + "1e9223372036854775807", "1e-9223372036854775808", "1e999999999999999999999999999999999", + "1e10001", "1e-10001", "1." + strings.Repeat("1", 10001), strings.Repeat("1", 10001), + } { + _, _, err := limitsCostDecimal(gjson.Parse(input)) + var summary *limitsCostSummaryError + require.ErrorAs(t, err, &summary) + require.Equal(t, "unsupported_precision", summary.Reason) + } +} + +func TestLimitsCostsTotalRejectsAmbiguousScopeAndBuckets(t *testing.T) { + const start, end int64 = 86400, 4 * 86400 + valid := gjson.Parse(limitsTestCostBucket(start, limitsTestCostResult("0.1", "usd", "proj_demo"))) + for _, test := range []struct { + name, project string + buckets []gjson.Result + reason string + }{ + {"overlap", "", []gjson.Result{valid, valid}, "overlapping_buckets"}, + {"outside period", "", []gjson.Result{gjson.Parse(limitsTestCostBucket(0, ""))}, "invalid_bucket"}, + {"non-object", "", []gjson.Result{gjson.Parse(`42`)}, "invalid_bucket"}, + {"wrong project", "proj_other", []gjson.Result{valid}, "unexpected_project"}, + {"malformed results", "", []gjson.Result{gjson.Parse(`{"object":"bucket","start_time":86400,"end_time":172800,"results":null}`)}, "invalid_bucket"}, + } { + t.Run(test.name, func(t *testing.T) { + value, err := limitsCostTotal(t.Context(), test.buckets, start, end, test.project) + var summary *limitsCostSummaryError + require.ErrorAs(t, err, &summary) + require.Empty(t, value) + require.Equal(t, test.reason, summary.Reason) + }) + } + value, err := limitsCostTotal(t.Context(), []gjson.Result{valid, + gjson.Parse(limitsTestCostBucket(start+86400, limitsTestCostResult("-0.3", "USD", "proj_demo")+","+limitsTestCostResult("2e-1", "usd", "proj_demo"))), + }, start, end, "proj_demo") + require.NoError(t, err) + require.Equal(t, "0.00", value) + value, err = limitsCostTotal(t.Context(), nil, start, end, "") + require.NoError(t, err) + require.Equal(t, "0.00", value) +} + +func TestLimitsMutationConfirmationMatchesReviewedFields(t *testing.T) { + change := limitsChange{Kind: "alert_update", ID: "alert_demo", Amount: 12345, + Recipients: []string{"finance@example.test", "ops@example.test"}, SubjectPrefixSet: true} + matching := `{"id":"alert_demo","object":"organization.spend_alert","threshold_amount":12345,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["ops@example.test","finance@example.test"],"subject_prefix":null},"future":{"exact":0.1234567890123456789}}` + require.NoError(t, limitsMutationConfirmed(gjson.Parse(matching), change, false)) + for _, response := range []string{ + strings.Replace(matching, `"threshold_amount":12345`, `"threshold_amount":12344`, 1), + strings.Replace(matching, `"threshold_amount":12345`, `"threshold_amount":12345.0`, 1), + strings.Replace(matching, `"currency":"USD"`, `"currency":"EUR"`, 1), + strings.Replace(matching, `"interval":"month"`, `"interval":"day"`, 1), + strings.Replace(matching, `"type":"email"`, `"type":"unknown"`, 1), + strings.Replace(matching, `"ops@example.test"`, `"other@example.test"`, 1), + strings.Replace(matching, `"ops@example.test","finance@example.test"`, `"finance@example.test","finance@example.test"`, 1), + strings.Replace(matching, `"subject_prefix":null`, `"subject_prefix":""`, 1), + strings.Replace(matching, `,"subject_prefix":null`, ``, 1), + strings.Replace(matching, `"threshold_amount":12345`, `"threshold_amount":12345,"threshold_amount":0`, 1), + } { + require.Error(t, limitsMutationConfirmed(gjson.Parse(response), change, false)) + } + empty := "" + change.SubjectPrefix = &empty + require.NoError(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(matching, `"subject_prefix":null`, `"subject_prefix":""`, 1)), change, false)) + change.SubjectPrefixSet = false + require.NoError(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(matching, `,"subject_prefix":null`, ``, 1)), change, false)) + rateChange := limitsChange{Kind: "rate_update", ID: "rate_demo", RateValues: map[string]int64{"max_requests_per_1_minute": 9007199254740993}} + rate := strings.Replace(limitsTestRate, `"max_requests_per_1_minute":100`, `"max_requests_per_1_minute":9007199254740993`, 1) + require.NoError(t, limitsMutationConfirmed(gjson.Parse(rate), rateChange, true)) + require.Error(t, limitsMutationConfirmed(gjson.Parse(strings.Replace(rate, "9007199254740993", "9007199254740992", 1)), rateChange, true)) + require.Error(t, limitsMutationConfirmed(gjson.Parse(limitsTestRate), rateChange, true)) +} + +func TestLimitsValueChecksPreserveCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + require.ErrorIs(t, limitsUniqueJSONContext(ctx, gjson.Parse(limitsTestLimit)), context.Canceled) + require.ErrorIs(t, limitsEditableBaselineContext(ctx, gjson.Parse(limitsTestLimit), "limit_set"), context.Canceled) + require.ErrorIs(t, limitsValidateChangeContext(ctx, limitsChange{Kind: "limit_set", Amount: 100, BeforeMissing: true}), context.Canceled) + _, err := limitsCanonicalJSONContext(ctx, gjson.Parse(limitsTestLimit)) + require.ErrorIs(t, err, context.Canceled) + _, err = limitsCostTotal(ctx, nil, 86400, 172800, "") + require.ErrorIs(t, err, context.Canceled) + require.ErrorIs(t, limitsMutationConfirmedContext(ctx, gjson.Parse(limitsTestLimit), limitsChange{Kind: "limit_set", Amount: 10000}, false), context.Canceled) +} diff --git a/pkg/custom/limits_workspace.go b/pkg/custom/limits_workspace.go new file mode 100644 index 00000000..e22e64ab --- /dev/null +++ b/pkg/custom/limits_workspace.go @@ -0,0 +1,1003 @@ +package custom + +import ( + "context" + "errors" + "fmt" + "os" + "strings" + "sync" + "time" + + "charm.land/bubbles/v2/viewport" + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/colorprofile" + "github.com/charmbracelet/x/ansi" + "github.com/charmbracelet/x/term" + "github.com/openai/openai-cli/internal/clihelp" + clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-cli/internal/readable" + clirequest "github.com/openai/openai-cli/internal/request" + "github.com/openai/openai-cli/internal/terminal" + "github.com/openai/openai-go/v3" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +type limitsWorkResult struct { + snapshot limitsSnapshot + value gjson.Result + err error + mutation bool + operation string +} +type limitsMenuItem struct { + label, kind string + value gjson.Result +} + +type limitsWorkspace struct { + detailsHeading string + detailsReturnPage string + detailsReturnOffset int + previewExecutable string + previewCommand, commandToPrint string + invocation string + lastSavedOperation, workerSavedOperation string + mutationError error + workerSaved gjson.Result + ctx context.Context + cancel context.CancelFunc + backend limitsBackend + output *limitsWorkspaceOutput + snapshot limitsSnapshot + width, height, section, selection, focus int + page, note, formError string + details string + busy, quitting, interrupted, locked, apply bool + refreshRequired, refreshingDraft, color, dark bool + lastSaved gjson.Result + draft limitsDraft + change limitsChange + viewport viewport.Model + requestArgs []string + omitPreview bool + shell string + work sync.WaitGroup + mu sync.Mutex + workerSnapshot *limitsSnapshot + workerError error + mutationPending bool +} + +type limitsInterrupt struct{ error } + +func (*limitsInterrupt) ExitCode() int { return 130 } +func (e *limitsInterrupt) Unwrap() error { return e.error } + +// limitsWorkspaceOutput records failures that the terminal renderer may ignore. +// Its output context stays active so shutdown can still restore terminal modes. +type limitsWorkspaceOutput struct { + clioutput.Writer + term.File + stop context.CancelFunc + mu sync.Mutex + err error +} + +var _ term.File = (*limitsWorkspaceOutput)(nil) + +func (w *limitsWorkspaceOutput) Write(data []byte) (int, error) { + n, err := w.Writer.Write(data) + if err != nil { + w.mu.Lock() + if w.err == nil { + w.err = err + } + w.mu.Unlock() + w.stop() + } + return n, err +} + +func (w *limitsWorkspaceOutput) WriteString(data string) (int, error) { + return w.Write([]byte(data)) +} + +func (w *limitsWorkspaceOutput) Err() error { + w.mu.Lock() + defer w.mu.Unlock() + return w.err +} + +func runLimitsWorkspace(ctx context.Context, cmd *cli.Command, backend limitsBackend, opts ShowJSONOpts) error { + file := opts.Stdout.(term.File) + width, height, err := term.GetSize(file.Fd()) + if err != nil { + return err + } + programContext, stop := context.WithCancel(ctx) + defer stop() + m := &limitsWorkspace{ctx: programContext, cancel: stop, backend: backend, width: width, height: height, page: "home", + color: os.Getenv("NO_COLOR") == "", dark: true, + viewport: viewport.New(viewport.WithWidth(max(1, width)), viewport.WithHeight(max(1, height-5)))} + m.viewport.SoftWrap, m.viewport.FillHeight = false, false + m.requestArgs, m.omitPreview = clirequest.NextCommandOptions(cmd.Root()) + m.shell = terminal.ParentShell(ctx) + m.invocation = clihelp.DisplayInvocation(cmd.Root()) + m.previewExecutable, err = os.Executable() + if err != nil || strings.HasPrefix(m.invocation, "go run ") { + m.omitPreview = true + } + m.output = &limitsWorkspaceOutput{ + Writer: clioutput.Writer{Context: context.Background(), Sink: opts.Stdout}, File: file, stop: stop} + options := []tea.ProgramOption{tea.WithInput(cmd.Root().Reader), tea.WithOutput(m.output), tea.WithContext(programContext), tea.WithWindowSize(width, height)} + if !m.color { + options = append(options, tea.WithColorProfile(colorprofile.NoTTY)) + } + program := tea.NewProgram(m, options...) + _, runErr := program.Run() + stop() + m.work.Wait() + return finishLimitsWorkspace(ctx, m, runErr, opts) +} + +// Finalization retains independent operation, cancellation, and output failures. +// A broken output pipe must not hide an uncertain mutation or failed source. +func finishLimitsWorkspace(ctx context.Context, m *limitsWorkspace, runErr error, opts ShowJSONOpts) error { + m.mu.Lock() + workerErr, pending, mutationErr := m.workerError, m.mutationPending, m.mutationError + if m.workerSnapshot != nil { + m.snapshot = *m.workerSnapshot + } + if m.workerSaved.Exists() { + m.lastSaved, m.lastSavedOperation = m.workerSaved, m.workerSavedOperation + } + m.mu.Unlock() + var causes []error + removedLimitErr := m.removedLimitFailure() + ignored := []error{removedLimitErr} + if ctx.Err() == nil && !m.interrupted { + ignored = append(ignored, context.Canceled) + } + sourcesIncomplete := false + for _, failure := range m.snapshot.Errors { + if !limitsOnlyErrors(failure.Err, ignored...) { + sourcesIncomplete = true + causes = append(causes, failure.Err) + } + } + // Load returns an aggregate of the recorded source errors. Keep each source + // once, and do not reintroduce cancellation caused by normal workspace exit. + accounted := append([]error{}, ignored...) + for _, failure := range m.snapshot.Errors { + accounted = append(accounted, failure.Err) + } + if limitsOnlyErrors(workerErr, accounted...) { + workerErr = nil + } + // Closing the workspace cancels its worker. That internal cancellation is + // not a failed read. Preserve the caller's cancellation separately below. + if workerErr != nil { + causes = append(causes, workerErr) + } + if m.refreshRequired || pending || m.locked { + causes = append(causes, mutationErr) + } + incomplete := m.refreshRequired || sourcesIncomplete || workerErr != nil + message, humanDetail := "", "" + if pending || m.locked { + message = "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + if mutationErr != nil { + humanDetail = limitsMutationRecovery(mutationErr, m.invocation, false) + } + } else if m.refreshRequired { + message = limitsPreflightRecovery(mutationErr, false, m.invocation) + "\n" + m.readRecovery() + if m.lastSaved.Exists() { + message += "\nA preceding change was confirmed. The latest change was not sent." + } + } else if incomplete { + message = "The limits overview is incomplete. Refresh before another change." + if m.lastSaved.Exists() { + message = "A change was confirmed, but the overview refresh failed. Inspect the resource before another change." + } + } + var outputErr error + if m.output != nil { + outputErr = m.output.Err() + } + interrupted := m.interrupted || errors.Is(runErr, tea.ErrInterrupted) + if ctx.Err() != nil { + causes = append(causes, ctx.Err()) + if message == "" { + message = "The limits workspace was canceled or timed out." + if m.lastSaved.Exists() { + message += " A preceding change was confirmed. Inspect the resource before another change." + } + } + } + if interrupted { + causes = append(causes, context.Canceled) + if message == "" { + message = "Limits workspace interrupted." + if m.lastSaved.Exists() { + message += " A preceding change was confirmed. Inspect the resource before another change." + } + } + } + // The output guard stops Tea after a sink failure. Preserve independent + // program failures, without treating that expected stop as another failure. + programErr := runErr + var expectedStops []error + if m.quitting || interrupted || outputErr != nil || ctx.Err() != nil { + expectedStops = append(expectedStops, tea.ErrProgramKilled, context.Canceled) + } + if interrupted { + expectedStops = append(expectedStops, tea.ErrInterrupted) + } + if limitsOnlyErrors(programErr, expectedStops...) { + programErr = nil + } + if programErr != nil { + causes = append(causes, programErr) + } + operationErr := errors.Join(causes...) + if message == "" && operationErr != nil { + message = "The limits workspace could not complete. Inspect the current settings before another change." + } + if outputErr != nil || interrupted || ctx.Err() != nil || pending || m.locked || programErr != nil { + if message != "" { + operationErr = &limitsWorkflowFailure{message: message, humanDetail: humanDetail, cause: operationErr} + } + failure := errors.Join(operationErr, outputErr) + if interrupted { + return &limitsInterrupt{failure} + } + return failure + } + var receiptErr error + if m.lastSaved.Exists() { + opts.Operation, opts.OutputKind = m.lastSavedOperation, OutputResponse + opts.Title = "Last confirmed result" + receiptErr = ShowJSON(m.lastSaved, opts) + } + var overviewErr error + if incomplete { + overviewErr = &limitsWorkflowFailure{message: message, cause: operationErr} + if m.commandToPrint == "" { + return errors.Join(overviewErr, receiptErr) + } + } + if receiptErr != nil { + return errors.Join(overviewErr, receiptErr) + } + if removedLimitErr != nil && resolvedOutputFormat(opts) == "text" { + if _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString("Removal confirmed. Current hard-limit configuration was not returned (HTTP 404).\n"); err != nil { + return errors.Join(overviewErr, err) + } + } + if m.commandToPrint != "" { + _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString(m.commandToPrint + "\n") + if err != nil { + err = &limitsWorkflowFailure{message: "Could not print the limits command. Check terminal output and try again.", cause: err} + } + return errors.Join(overviewErr, err) + } + if m.lastSaved.Exists() { + return nil + } + _, err := (clioutput.Writer{Context: ctx, Sink: opts.Stdout}).WriteString("Limits workspace closed.\n") + return err +} + +func (m *limitsWorkspace) readRecovery() string { + invocation := m.invocation + if invocation == "" { + invocation = "openai" + } + resource := "admin organization " + if m.snapshot.ProjectID != "" { + resource += "projects " + } + switch m.change.Kind { + case "alert_create", "alert_update", "alert_delete": + resource += "spend-alerts list" + case "rate_update": + resource += "rate-limits list-rate-limits" + default: + resource += "spend-limit retrieve" + } + return "Read options: " + invocation + " help " + resource + "\nKeep the original connection, authentication, and scope options." +} + +func (m *limitsWorkspace) Init() tea.Cmd { return tea.Batch(m.load(), tea.RequestBackgroundColor) } + +func (m *limitsWorkspace) launch(change *limitsChange) tea.Cmd { + m.busy = true + result := make(chan limitsWorkResult, 1) + operation := "" + if change != nil { + operation = limitsChangeOperation(change.Kind, m.snapshot.ProjectID != "") + } + m.work.Add(1) + if change != nil { + m.mu.Lock() + m.mutationPending = true + m.mu.Unlock() + } + go func() { + defer m.work.Done() + var r limitsWorkResult + if change == nil { + r.snapshot, r.err = m.backend.Load(m.ctx) + } else { + r.mutation, r.operation = true, operation + r.value, r.err = m.backend.Mutate(m.ctx, *change) + } + m.mu.Lock() + m.workerError = r.err + if !r.mutation { + m.workerSnapshot = &r.snapshot + } + if r.mutation && r.err == nil { + m.workerSaved, m.workerSavedOperation = r.value, r.operation + } + if r.mutation { + m.mutationError = r.err + } + var rejected *limitsMutationFailure + if r.mutation && (r.err == nil || errors.As(r.err, &rejected) && !rejected.Attempted) { + m.mutationPending = false + } + m.mu.Unlock() + result <- r + }() + return func() tea.Msg { return <-result } +} + +func (m *limitsWorkspace) load() tea.Cmd { return m.launch(nil) } +func (m *limitsWorkspace) small() bool { return m.width < 40 || m.height < 12 } + +func (m *limitsWorkspace) reconcileDraft() { + change := m.draft.change + var current gjson.Result + switch change.Kind { + case "limit_set", "limit_delete": + current = m.snapshot.Limit + if change.Kind == "limit_set" && (m.snapshot.LimitAbsent || m.snapshot.LimitUnconfirmed) && m.limitCanBeSet() { + change.Before, change.BeforeMissing = gjson.Result{}, true + m.draft.change, m.refreshRequired, m.formError = change, false, "Baseline refreshed. Review the before and after values." + return + } + case "alert_create": + m.refreshRequired, m.formError = false, "Review the draft before creating the alert." + return + case "alert_update", "alert_delete": + for _, record := range m.snapshot.Alerts { + if record.Get("id").String() == change.ID { + if current.Exists() { + m.formError = "Duplicate alert IDs. Inspect with the API command." + return + } + current = record + } + } + case "rate_update": + for _, record := range m.snapshot.Rates { + if record.Get("id").String() == change.ID { + if current.Exists() { + m.formError = "Duplicate rate IDs. Inspect with the API command." + return + } + current = record + } + } + } + if !current.Exists() || limitsEditableBaseline(current, change.Kind) != nil { + m.formError = "Cannot refresh this draft. Inspect the resource with the API command." + return + } + change.Before, change.BeforeMissing = current, false + m.draft.change, m.refreshRequired, m.formError = change, false, "Baseline refreshed. Review the before and after values." +} + +func (m *limitsWorkspace) menu() []limitsMenuItem { + var rows []limitsMenuItem + switch m.section { + case 0: + if m.limitCanBeSet() { + rows = append(rows, limitsMenuItem{label: "Set hard spend limit", kind: "limit_set", value: m.snapshot.Limit}) + if m.snapshot.Limit.Exists() { + rows = append(rows, limitsMenuItem{label: "Remove hard spend limit", kind: "limit_delete", value: m.snapshot.Limit}) + } + } + case 1: + rows = append(rows, limitsMenuItem{label: "Create spend alert", kind: "alert_create"}) + for _, alert := range m.snapshot.Alerts { + rows = append(rows, limitsMenuItem{label: "Edit " + alert.Get("id").String() + " " + limitsThresholdLabel(alert), kind: "alert_update", value: alert}) + } + case 2: + for _, rate := range m.snapshot.Rates { + rows = append(rows, limitsMenuItem{label: rate.Get("model").String() + " " + rate.Get("id").String(), kind: "rate_update", value: rate}) + } + } + return rows +} + +func (m *limitsWorkspace) begin(kind string, value gjson.Result) { + if m.locked { + m.note = "Writes are locked after a failed save. Inspect the resource before reopening." + return + } + for _, failure := range m.snapshot.Errors { + if failure.Resource == "project" { + m.note = "Project scope unavailable. Refresh: r" + return + } + } + change := limitsChange{Kind: kind, ID: value.Get("id").String(), Before: value, BeforeMissing: kind == "limit_set" && (m.snapshot.LimitAbsent || m.snapshot.LimitUnconfirmed)} + if kind == "limit_set" || kind == "limit_delete" { + change.ID = "" + } + draft, err := newLimitsDraft(change) + if err != nil { + m.note = "This record cannot be edited safely. Use i to inspect the original fields." + return + } + m.draft, m.change, m.focus, m.formError = draft, change, 0, "" + if len(draft.inputs) == 0 { + m.review() + return + } + m.page = "form" + m.setFocus(0) +} + +func (m *limitsWorkspace) setFocus(next int) { + for i := range m.draft.inputs { + m.draft.inputs[i].Blur() + } + count := len(m.draft.inputs) + 1 + if len(m.draft.inputs) == 3 { + count++ + } + m.focus = (next + count) % count + if m.focus < len(m.draft.inputs) { + m.draft.inputs[m.focus].Focus() + } +} + +func (m *limitsWorkspace) review() { + if m.refreshRequired { + m.formError = "Refresh with r before reviewing this draft." + return + } + change, err := m.draft.validate() + if err != nil { + m.formError = err.Error() + return + } + m.change, m.page, m.apply, m.formError = change, "review", false, "" + m.setReview() +} + +func (m *limitsWorkspace) setReview() { + m.previewCommand, _ = limitsCommand(m.change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable) + m.setDetails(limitsReview(m.change, m.snapshot.ProjectID, m.requestArgs, m.omitPreview, m.shell, m.previewExecutable)) +} + +func (m *limitsWorkspace) detailsHeight() int { + if m.page == "review" && m.previewCommand != "" { + return max(1, m.height-6) + } + return max(1, m.height-5) +} + +func (m *limitsWorkspace) setDetails(content string) { + m.details = content + m.viewport.SetWidth(max(1, m.width-2)) + m.viewport.SetHeight(m.detailsHeight()) + m.viewport.SetContent(ansi.Hardwrap(content, max(1, m.width-2), true)) + m.viewport.GotoTop() +} + +func (m *limitsWorkspace) Update(message tea.Msg) (tea.Model, tea.Cmd) { + if m.quitting { + return m, tea.Quit + } + switch msg := message.(type) { + case tea.BackgroundColorMsg: + m.dark = msg.IsDark() + return m, nil + case tea.WindowSizeMsg: + m.width, m.height = msg.Width, msg.Height + m.apply = false + if m.page == "review" { + m.setReview() + } + if m.page == "details" { + m.detailsReturnOffset = 0 + m.setDetails(m.details) + } + m.viewport.SetWidth(max(1, msg.Width-2)) + m.viewport.SetHeight(m.detailsHeight()) + return m, nil + case limitsWorkResult: + m.busy = false + if msg.mutation { + if msg.err != nil { + var rejected *limitsMutationFailure + m.locked = !(errors.As(msg.err, &rejected) && !rejected.Attempted) + m.refreshRequired, m.apply, m.page = !m.locked, false, "form" + m.formError = "Save not confirmed. Press ? for recovery. Exit before another change." + if !m.locked { + m.formError = limitsPreflightHint(rejected.Err, m.invocation) + } + return m, nil + } + m.lastSaved, m.lastSavedOperation = msg.value, msg.operation + m.page, m.note, m.selection = "home", "Saved. Refreshing the overview.", 0 + return m, m.load() + } + m.snapshot = msg.snapshot + m.note = "Website controls and source errors: ?" + if m.lastSaved.Exists() { + m.note = "Change confirmed. Last result prints when you exit." + } + if msg.err != nil || len(msg.snapshot.Errors) != 0 { + m.note = "Some sources failed. Press ? for recovery; r to refresh." + if expected := m.removedLimitFailure(); expected != nil && len(msg.snapshot.Errors) == 1 && limitsOnlyErrors(msg.err, expected) { + m.note = "Removal confirmed. Current limit not returned." + } + } + if m.refreshingDraft { + m.refreshingDraft = false + m.reconcileDraft() + } + m.selection = min(m.selection, max(0, len(m.menu())-1)) + return m, nil + case tea.KeyPressMsg: + key := msg.String() + if key == "ctrl+c" || (m.page == "home" && (key == "q" || key == "esc")) { + m.interrupted, m.quitting = key == "ctrl+c", true + m.cancel() + return m, tea.Quit + } + if m.busy { + return m, nil + } + if key == "esc" { + m.apply = false + switch m.page { + case "details": + m.page, m.detailsReturnPage = m.detailsReturnPage, "" + switch m.page { + case "review": + m.setReview() + m.viewport.SetYOffset(m.detailsReturnOffset) + case "form": + m.setFocus(m.focus) + default: + m.page = "home" + } + case "review": + m.page = "form" + m.setFocus(m.focus) + default: + m.page = "home" + } + return m, nil + } + if m.small() { + return m, nil + } + if key == "f1" || key == "?" && m.page != "form" || key == "?" && m.locked { + if m.page != "details" { + m.detailsReturnPage, m.detailsReturnOffset = m.page, m.viewport.YOffset() + } + m.apply = false + m.page, m.detailsHeading = "details", "Limits workspace help" + m.setDetails(m.help()) + return m, nil + } + switch m.page { + case "home": + switch key { + + case "r": + m.note = "Refreshing…" + return m, m.load() + case "left", "right", "tab", "shift+tab": + count, delta := 2, 1 + if m.snapshot.ProjectID != "" { + count = 3 + } + if key == "left" || key == "shift+tab" { + delta = -1 + } + m.section, m.selection = (m.section+count+delta)%count, 0 + case "up": + m.selection = max(0, m.selection-1) + case "down": + m.selection = min(max(0, len(m.menu())-1), m.selection+1) + case "enter", "d", "i": + rows := m.menu() + if len(rows) == 0 { + return m, nil + } + row := rows[m.selection] + if key == "i" { + if row.value.Exists() { + m.detailsReturnPage = "home" + m.page, m.detailsHeading = "details", "Original API record" + m.setDetails(limitsSafeJSON(row.value)) + } + } else if key == "d" { + if row.kind == "alert_update" { + m.begin("alert_delete", row.value) + } + } else { + m.begin(row.kind, row.value) + } + } + case "form": + if key == "r" && m.refreshRequired { + m.refreshingDraft = true + return m, m.load() + } + switch key { + case "tab": + m.setFocus(m.focus + 1) + return m, nil + case "shift+tab": + m.setFocus(m.focus - 1) + return m, nil + case "enter": + if m.focus == len(m.draft.inputs) && len(m.draft.inputs) == 3 { + m.draft.prefixMode = (m.draft.prefixMode + 1) % 3 + } else if m.focus >= len(m.draft.inputs) { + if !m.locked { + m.review() + } + } else { + m.setFocus(m.focus + 1) + } + return m, nil + } + case "review": + if key == "ctrl+p" { + if m.previewCommand != "" && !m.locked && !m.refreshRequired && m.ctx.Err() == nil && (m.output == nil || m.output.Err() == nil) { + m.commandToPrint, m.quitting = m.previewCommand, true + m.cancel() + return m, tea.Quit + } + return m, nil + } + if key == "tab" || key == "shift+tab" { + if m.viewport.AtBottom() { + m.apply = !m.apply + } + return m, nil + } + if key == "enter" { + if !m.apply { + m.page = "form" + m.setFocus(m.focus) + return m, nil + } + if m.viewport.AtBottom() && !m.locked && m.ctx.Err() == nil && (m.output == nil || m.output.Err() == nil) { + m.apply = false + change := m.change + return m, m.launch(&change) + } + return m, nil + } + } + } + if m.page == "review" || m.page == "details" { + var command tea.Cmd + m.viewport, command = m.viewport.Update(message) + if !m.viewport.AtBottom() { + m.apply = false + } + return m, command + } + if m.page == "form" && m.focus < len(m.draft.inputs) && !m.small() { + var command tea.Cmd + m.draft.inputs[m.focus], command = m.draft.inputs[m.focus].Update(message) + return m, command + } + return m, nil +} + +func limitsThresholdLabel(value gjson.Result) string { + amount, err := limitsInteger(value.Get("threshold_amount")) + if err == nil && limitsUniqueJSON(value) == nil && value.Get("currency").String() == "USD" && value.Get("interval").String() == "month" { + return "USD " + formatLimitsDollars(amount) + " / month" + } + return "Inspect raw units" +} + +func (m *limitsWorkspace) heading() string { + if m.page == "form" || m.page == "review" { + return limitsChangeTitle(m.change.Kind) + } + if m.page == "details" { + if m.detailsHeading != "" { + return m.detailsHeading + } + return "Original API record" + } + return "Limits" +} + +func (m *limitsWorkspace) scope() string { + if m.snapshot.ProjectID != "" { + return "Project " + m.snapshot.ProjectID + } + return "Organization" +} + +func (m *limitsWorkspace) sourceSummary() string { + switch m.section { + case 0: + if m.snapshot.LimitUnconfirmed { + if m.removedLimitFailure() != nil { + return "Removal confirmed. Current limit unknown." + } + return "Hard limit unknown. Review before replacing." + } + if !m.limitReadable() { + return "Hard spend limit unavailable. Refresh: r" + } + if m.snapshot.LimitAbsent { + return "No hard spend limit configured." + } + if m.snapshot.Limit.Exists() { + return "Hard limit: " + limitsThresholdLabel(m.snapshot.Limit) + } + return "Hard spend limit unavailable. Refresh to retry." + case 1: + if !m.snapshot.AlertsComplete { + return fmt.Sprintf("Incomplete. Alerts loaded: %d", len(m.snapshot.Alerts)) + } + return fmt.Sprintf("Complete. Alerts loaded: %d", len(m.snapshot.Alerts)) + default: + if !m.snapshot.RatesComplete { + return fmt.Sprintf("Incomplete. Rate records loaded: %d", len(m.snapshot.Rates)) + } + return fmt.Sprintf("Complete. Rate records loaded: %d", len(m.snapshot.Rates)) + } +} + +func (m *limitsWorkspace) limitReadable() bool { + for _, failure := range m.snapshot.Errors { + if failure.Resource == "spend_limit" || failure.Resource == "project" { + return false + } + } + return true +} + +func limitsSafeJSON(value gjson.Result) string { return limitsSafeLine(value.Raw) } + +func limitsPreflightHint(err error, invocation ...string) string { + return limitsPreflightRecovery(err, true, invocation...) +} + +func limitsPreflightRecovery(err error, inWorkspace bool, invocation ...string) string { + bin := "openai" + if len(invocation) != 0 && invocation[0] != "" { + bin = invocation[0] + } + refresh := "Press r to refresh." + validation := "Recheck the request fields and scope, then press r to refresh." + baseline := "Press r to refresh this draft's baseline." + conflict := "Press r to refresh and review the new baseline." + interrupted := "Press r to refresh before trying again." + wait := "Wait before refreshing. Check API error details if this continues." + if !inWorkspace { + refresh = "Reopen the limits workspace and review current settings before another change." + validation = "Recheck the request fields and scope.\n" + refresh + baseline, conflict, interrupted = refresh, refresh, refresh + wait = "Wait before reopening the limits workspace. Check API error details if this continues." + } + var apiErr *openai.Error + if errors.As(err, &apiErr) { + switch apiErr.StatusCode { + case 400, 422: + return fmt.Sprintf("The API returned HTTP %d. No write sent.\n%s", apiErr.StatusCode, validation) + case 401: + return "Admin authentication failed. No write sent.\nKey setup: " + bin + " help setup admin" + case 403: + return "The API denied access. No write sent.\nCheck Admin-key permissions and organization or project access.\nAccess setup: " + bin + " help setup admin" + case 404: + return "The requested resource was not returned. No write sent.\nCheck the API address, scope, and resource ID. " + refresh + case 429: + switch apiErr.Code { + case "organization_spend_limit_exceeded": + return "The API reports an organization spend limit was reached. No write sent.\nLimit options: " + bin + " help admin organization spend-limit retrieve" + case "project_spend_limit_exceeded": + return "The API reports a project spend limit was reached. No write sent.\nLimit options: " + bin + " help admin organization projects spend-limit retrieve" + case "insufficient_quota", "billing_hard_limit_reached", "credit_balance_exhausted", "organization_usage_limit_exceeded": + return "The API reports a billing or usage limit. No write sent.\nCheck billing and approved usage: https://platform.openai.com/settings/organization/limits" + default: + return "The API returned HTTP 429. No write sent.\n" + wait + } + default: + return fmt.Sprintf("The API returned HTTP %d. No write sent.\n%s If this continues, check API availability and access.", apiErr.StatusCode, refresh) + } + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "The check was interrupted. No write sent. " + interrupted + } + if err == nil { + return "No change sent. " + baseline + } + // Only exact locally defined messages select specific recovery guidance. + switch err.Error() { + case "Rate-limit editing requires a complete list. Reopen with a larger --max-items value or -1.": + return "No write sent. Reopen with --max-items -1 to check every rate record." + case "The configuration changed. Refresh before applying this change.": + return "Record changed. No write sent. " + conflict + default: + return "No change sent. " + baseline + } +} + +func (m *limitsWorkspace) limitCanBeSet() bool { + for _, failure := range m.snapshot.Errors { + if failure.Resource == "project" || failure.Resource == "spend_limit" && !m.snapshot.LimitUnconfirmed { + return false + } + } + return m.snapshot.Limit.Exists() || m.snapshot.LimitAbsent || m.snapshot.LimitUnconfirmed +} + +func (m *limitsWorkspace) costsSummary() string { + if m.snapshot.CostsTotal == "" { + return "Current-month Costs unavailable. ?: details" + } + label := "Reported Costs" + if !m.snapshot.CostsComplete { + label += " (incomplete)" + } + return label + ": USD " + m.snapshot.CostsTotal +} + +func (m *limitsWorkspace) help() string { + invocation := m.invocation + if invocation == "" { + invocation = "openai" + } + text := "Spend manages hard monthly USD limits.\nAlerts notify recipients; they do not enforce caps.\nRates manages six project rate fields.\n\nEvery change requires a before/after review.\nRead to the end, then Tab to Apply.\nBack remains the default action.\nCtrl+P prints the reviewed command and exits. Printing sends no change.\n\nUse r to refresh. Use i to inspect original fields.\nUse --max-items -1 to load complete lists.\n\n" + if m.locked { + m.mu.Lock() + failure := m.mutationError + m.mu.Unlock() + text = "The latest save was not confirmed. It may have succeeded.\nWrites remain locked, including after refresh.\n" + m.readRecovery() + "\n" + limitsMutationHint(failure, invocation) + "\n\n" + text + } else if m.refreshRequired { + m.mu.Lock() + failure := m.mutationError + m.mu.Unlock() + text = limitsPreflightHint(failure, invocation) + "\n\n" + text + } else if m.formError != "" && (m.page == "form" || m.detailsReturnPage == "form") { + text = "Draft details\n" + readable.Text(m.formError) + "\n\n" + text + } + for _, failure := range m.snapshot.Errors { + // Source names come from our loader, never response text. + text += "Source unavailable: " + failure.Resource + "\n" + limitsReadHint(failure.Err, invocation) + "\n\n" + } + if len(m.snapshot.Errors) != 0 { + route := "admin limits show" + if m.snapshot.ProjectID != "" { + route = "projects limits show" + } + text += "Overview options: " + invocation + " help " + route + "\nKeep the original connection, authentication, and scope options.\nUse --format json to inspect available records and source_errors.\n\n" + } + if m.snapshot.LimitUnconfirmed { + text += "The current hard limit could not be confirmed.\nSet hard spend limit requires review and can create or replace a limit.\n\n" + } + if m.snapshot.CostsStart != 0 && m.snapshot.CostsEnd != 0 { + text += m.costsSummary() + "\nQueried period: " + time.Unix(m.snapshot.CostsStart, 0).UTC().Format(time.RFC3339) + " to " + time.Unix(m.snapshot.CostsEnd, 0).UTC().Format(time.RFC3339) + " (exclusive).\nToday's Costs remain partial.\nCosts can lag and do not determine enforcement state.\n\n" + } + text += "Soft budgets, independent enforcement settings, account tiers, credits, and organization rate catalogs need the website:\nhttps://platform.openai.com/settings/organization/limits\n\nThe public API exposes absolute-value alerts, not website percentage alerts or triggered status.\nRemoving a hard limit stops that cap; inspect soft budgets on the website.\n\nHistorical Costs command:\n" + invocation + " help admin usage costs\n\nThe workspace checks current state before saving. This check is not atomic.\nThe workspace sends each confirmed mutation once, without automatic retries.\nDirect API commands omit these safeguards and can retry.\n" + return text +} + +func limitsReadHint(err error, invocation string) string { + // Read failures have no pending mutation. Reuse the classification without + // presenting a failed read as a failed save. + var summary *limitsCostSummaryError + if errors.As(err, &summary) { + return summary.Error() + "\nCosts options: " + invocation + " help admin usage costs" + } + var apiErr *openai.Error + if errors.As(err, &apiErr) { + switch apiErr.StatusCode { + case 401, 403, 404, 429: + return strings.ReplaceAll(limitsPreflightHint(err, invocation), "No write sent.", "No change was attempted.") + case 400, 422: + return fmt.Sprintf("The API returned HTTP %d. No change was attempted.\nRecheck the request fields and scope, then refresh.", apiErr.StatusCode) + default: + return fmt.Sprintf("The API returned HTTP %d. No change was attempted.\nPress r to refresh. If this continues, check API availability and access.", apiErr.StatusCode) + } + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "The read was interrupted. Press r to refresh." + } + return "The source could not be read. Check the API address and access, then refresh." +} + +func limitsMutationHint(err error, invocation string) string { + return limitsMutationRecovery(err, invocation, true) +} + +func limitsMutationRecovery(err error, invocation string, inWorkspace bool) string { + var apiErr *openai.Error + if errors.As(err, &apiErr) { + switch apiErr.StatusCode { + case 401, 403, 404, 429: + if !inWorkspace && apiErr.StatusCode == 404 { + return "The requested resource was not returned. The change is not confirmed.\nCheck the API address, scope, and resource ID. Read the current record before another change." + } + return strings.ReplaceAll(limitsPreflightRecovery(err, inWorkspace, invocation), "No write sent.", "The change is not confirmed.") + case 400, 422: + return fmt.Sprintf("The API returned HTTP %d. The change is not confirmed.\nRecheck the reviewed fields. Read the current record before another change.", apiErr.StatusCode) + default: + return fmt.Sprintf("The API returned HTTP %d. The change is not confirmed.\nRead the current record before another change.", apiErr.StatusCode) + } + } + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return "The response was interrupted. The change is not confirmed.\nRead the current record before another change." + } + if err != nil { + switch err.Error() { + case "The API returned different values from the reviewed change. Inspect the returned record before another change.": + return "The API returned different values from the reviewed change. Read the current record before another change." + case "The API returned an incomplete confirmation. Refresh and inspect the resource before another change.": + return "The API response did not confirm the change. Read the current record before another change." + } + } + return "The response did not confirm the change. Check the API address and connection before reading its current state." +} + +func (m *limitsWorkspace) removedLimitFailure() error { + if m.lastSavedOperation != limitsChangeOperation("limit_delete", m.snapshot.ProjectID != "") || m.lastSaved.Get("deleted").Type != gjson.True || !m.snapshot.LimitUnconfirmed { + return nil + } + for _, failure := range m.snapshot.Errors { + if failure.Resource == "spend_limit" && limitsHTTPStatus(failure.Err, 404) { + return failure.Err + } + } + return nil +} + +// Inspect each branch; errors.Is alone also matches joins with unrelated errors. +func limitsOnlyErrors(err error, expected ...error) bool { + if err == nil { + return false + } + if joined, ok := err.(interface{ Unwrap() []error }); ok { + children := joined.Unwrap() + if len(children) == 0 { + return false + } + for _, child := range children { + if !limitsOnlyErrors(child, expected...) { + return false + } + } + return true + } + if wrapped, ok := err.(interface{ Unwrap() error }); ok { + return limitsOnlyErrors(wrapped.Unwrap(), expected...) + } + for _, target := range expected { + if target != nil && errors.Is(err, target) { + return true + } + } + return false +} diff --git a/pkg/custom/limits_workspace_test.go b/pkg/custom/limits_workspace_test.go new file mode 100644 index 00000000..a19979f0 --- /dev/null +++ b/pkg/custom/limits_workspace_test.go @@ -0,0 +1,1869 @@ +package custom + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math" + "net/http" + "os" + "os/exec" + "path/filepath" + "reflect" + "runtime" + "strings" + "sync/atomic" + "syscall" + "testing" + "time" + + "charm.land/bubbles/v2/viewport" + tea "charm.land/bubbletea/v2" + "github.com/charmbracelet/colorprofile" + "github.com/charmbracelet/x/ansi" + clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-cli/internal/terminal" + "github.com/openai/openai-go/v3" + "github.com/openai/openai-go/v3/option" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" +) + +const limitsControllerAlert = `{"object":"organization.spend_alert","id":"alert_synthetic","threshold_amount":10000,"currency":"USD","interval":"month","notification_channel":{"type":"email","recipients":["synthetic@example.invalid"]}}` +const limitsControllerRate = `{"object":"project.rate_limit","id":"rl_synthetic","model":"synthetic-model","max_requests_per_1_minute":100,"max_tokens_per_1_minute":200,"max_images_per_1_minute":0}` + +type limitsControllerBackend struct { + snapshot limitsSnapshot + loadErr, mutationErr error + release <-chan struct{} + changes chan limitsChange + writes, reads atomic.Int32 +} + +func (b *limitsControllerBackend) Load(ctx context.Context) (limitsSnapshot, error) { + b.reads.Add(1) + if err := ctx.Err(); err != nil { + return limitsSnapshot{}, err + } + return b.snapshot, b.loadErr +} + +func (b *limitsControllerBackend) Mutate(ctx context.Context, change limitsChange) (gjson.Result, error) { + b.writes.Add(1) + b.changes <- change + if b.release != nil { + select { + case <-b.release: + case <-ctx.Done(): + return gjson.Result{}, ctx.Err() + } + } + return gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month"}`), b.mutationErr +} + +func newLimitsController(t *testing.T, width, height int) (*limitsWorkspace, *limitsControllerBackend) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + b := &limitsControllerBackend{snapshot: limitsSnapshot{LimitAbsent: true, AlertsComplete: true}, changes: make(chan limitsChange, 8)} + m := &limitsWorkspace{ctx: ctx, cancel: cancel, backend: b, snapshot: b.snapshot, width: width, height: height, page: "home", shell: "bash", + viewport: viewport.New(viewport.WithWidth(width-2), viewport.WithHeight(height-5))} + m.viewport.SoftWrap, m.viewport.FillHeight = false, false + t.Cleanup(func() { cancel(); m.work.Wait() }) + return m, b +} + +func limitsControllerKey(m *limitsWorkspace, code rune) tea.Cmd { + _, command := m.Update(tea.KeyPressMsg{Code: code}) + return command +} + +func limitsControllerReview(t *testing.T, m *limitsWorkspace) { + t.Helper() + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" || len(m.draft.inputs) != 1 { + t.Fatalf("limit form did not open: page=%s", m.page) + } + m.draft.inputs[0].SetValue("100.01") + limitsControllerKey(m, tea.KeyTab) + limitsControllerKey(m, tea.KeyEnter) + if m.page != "review" || m.apply || m.busy { + t.Fatalf("review did not start on Back: page=%s apply=%t busy=%t", m.page, m.apply, m.busy) + } +} + +func limitsControllerRun(t *testing.T, command tea.Cmd) limitsWorkResult { + t.Helper() + if command == nil { + t.Fatal("missing asynchronous operation") + } + result, ok := command().(limitsWorkResult) + if !ok { + t.Fatal("operation returned an unexpected message") + } + return result +} + +func limitsControllerStarted(t *testing.T, m *limitsWorkspace, b *limitsControllerBackend) limitsChange { + t.Helper() + select { + case change := <-b.changes: + return change + case <-m.ctx.Done(): + t.Fatal("confirmed mutation did not reach the backend") + return limitsChange{} + } +} + +func TestLimitsWorkspaceReviewRequiresExplicitApply(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + limitsControllerReview(t, m) + m.work.Wait() + if b.writes.Load() != 0 { + t.Fatal("opening or reviewing a draft wrote to the backend") + } + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" || b.writes.Load() != 0 { + t.Fatal("default Back action submitted the draft") + } + m.setFocus(len(m.draft.inputs)) + limitsControllerKey(m, tea.KeyEnter) + m.viewport.GotoTop() + if m.viewport.AtBottom() { + t.Fatal("fixture must require scrolling through its review") + } + limitsControllerKey(m, tea.KeyTab) + if m.apply { + t.Fatal("Apply became active before the review reached its end") + } + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + if !m.apply || b.writes.Load() != 0 { + t.Fatal("selecting Apply must not submit") + } + result := limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter)) + change := limitsControllerStarted(t, m, b) + if b.writes.Load() != 1 || change.Kind != "limit_set" || change.Amount != 10001 || !change.BeforeMissing { + t.Fatalf("incorrect confirmed mutation: %+v; writes=%d", change, b.writes.Load()) + } + _, refresh := m.Update(result) + m.Update(limitsControllerRun(t, refresh)) + if m.page != "home" || m.busy || b.reads.Load() != 1 || b.writes.Load() != 1 { + t.Fatal("successful save did not refresh once without repeating the mutation") + } +} + +func TestLimitsWorkspaceResizeClearsApplyAndBlocksTinySubmission(t *testing.T) { + for _, size := range [][2]int{{80, 24}, {40, 12}, {39, 12}, {40, 11}} { + m, b := newLimitsController(t, 80, 24) + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + m.Update(tea.WindowSizeMsg{Width: size[0], Height: size[1]}) + if m.apply { + t.Errorf("resize to %v retained Apply selection", size) + } + if m.small() { + if view := m.View().Content; !strings.Contains(view, "Resize") || !strings.Contains(view, "Ctrl+C") { + t.Errorf("tiny screen lacks resize/cancel guidance: %q", view) + } + m.apply = true // A hidden action must remain blocked even with stale focus. + if command := limitsControllerKey(m, tea.KeyEnter); command != nil || m.busy { + t.Errorf("tiny screen %v submitted an invisible action", size) + } + } else { + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" { + t.Errorf("resize to %v did not reset confirmation to Back", size) + } + } + m.work.Wait() + if b.writes.Load() != 0 { + t.Errorf("resize to %v caused a write", size) + } + } +} + +func TestLimitsWorkspaceBusyAndFailedSavePreventRepeatedWrites(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + release := make(chan struct{}) + b.release, b.mutationErr = release, errors.New("synthetic uncertain write failure") + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + command := limitsControllerKey(m, tea.KeyEnter) + limitsControllerStarted(t, m, b) + for _, key := range []rune{tea.KeyEnter, tea.KeyTab, tea.KeyEnter, 'r', tea.KeyEscape} { + if next := limitsControllerKey(m, key); next != nil { + t.Error("busy workspace started another operation") + } + } + if b.writes.Load() != 1 || !m.busy { + t.Fatal("pending write lost its submission guard") + } + close(release) + m.Update(limitsControllerRun(t, command)) + if !m.locked || m.apply || m.page != "form" || m.draft.inputs[0].Value() != "100.01" || !strings.Contains(m.formError, "Press ? for recovery") { + t.Fatalf("failed save discarded draft or unlocked writes: page=%s locked=%t error=%q", m.page, m.locked, m.formError) + } + m.setFocus(len(m.draft.inputs)) + limitsControllerKey(m, tea.KeyEnter) + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, tea.KeyEnter) + m.work.Wait() + if b.writes.Load() != 1 || m.busy || !m.locked { + t.Fatal("failed save permitted an unconfirmed repeat") + } + limitsControllerKey(m, '?') + if m.page != "details" || !strings.Contains(m.details, "It may have succeeded") || !strings.Contains(m.details, "help admin organization spend-limit retrieve") { + t.Fatal("failed save did not provide uncertainty guidance and a safe read") + } +} + +func limitsControllerPrintedCommand(t *testing.T, m *limitsWorkspace) string { + t.Helper() + m.viewport.GotoBottom() + view := ansi.Strip(m.View().Content) + for _, visible := range []string{"Ctrl+P: print command and exit", "> Back", "Apply change"} { + if !strings.Contains(view, visible) { + t.Fatalf("review clipped %q at %d columns: %q", visible, m.width, view) + } + } + for _, line := range strings.Split(view, "\n") { + if ansi.StringWidth(line) > m.width { + t.Fatalf("review exceeds %d columns: %q", m.width, line) + } + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit == nil || !m.quitting || m.interrupted || m.commandToPrint == "" { + t.Fatal("Ctrl+P did not freeze the reviewed command and quit normally") + } + var out bytes.Buffer + if err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard}); err != nil { + t.Fatal(err) + } + if out.String() != m.commandToPrint+"\n" { + t.Fatalf("finalization changed the command or added prose: %q", out.String()) + } + return out.String() +} + +func TestLimitsWorkspacePrintFreezesReviewedCommand(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + limitsControllerReview(t, m) + if !strings.Contains(m.details, "Command preview (inspection only)") { + t.Fatal("wrapped review did not identify the command as inspection-only") + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit == nil || m.commandToPrint == "" { + t.Fatal("review did not offer command printing") + } + frozen := m.commandToPrint + m.change.Amount, m.previewCommand = 99999, "changed after exit" + m.Update(tea.WindowSizeMsg{Width: 80, Height: 24}) + var out bytes.Buffer + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Stdout: &out, Stderr: io.Discard}) + if err != nil || out.String() != frozen+"\n" || backend.writes.Load() != 0 { + t.Fatalf("print changed reviewed bytes or sent a mutation: output=%q err=%v", out.String(), err) + } +} + +func TestLimitsWorkspacePrintOmissionsAndUnavailableStates(t *testing.T) { + for _, scenario := range []string{"private context", "unsupported shell", "literal prefix", "go run", "locked", "refresh required", "busy", "small terminal", "canceled", "output failure"} { + t.Run(scenario, func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.change, m.page = limitsChange{Kind: "limit_set", Amount: 12525}, "review" + omitted := true + switch scenario { + case "private context": + m.omitPreview = true + case "unsupported shell": + m.shell = "unknown-shell" + case "literal prefix": + prefix := "\\@literal" + m.change = limitsChange{Kind: "alert_create", SubjectPrefixSet: true, SubjectPrefix: &prefix} + case "go run": + m.invocation, m.omitPreview = "go run ./cmd/openai", true + default: + omitted = false + } + m.setReview() + switch scenario { + case "locked": + m.locked = true + case "refresh required": + m.refreshRequired = true + case "busy": + m.busy = true + case "small terminal": + m.Update(tea.WindowSizeMsg{Width: 39, Height: 12}) + case "canceled": + m.cancel() + case "output failure": + m.output = &limitsWorkspaceOutput{err: errors.New("synthetic sink failure")} + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit != nil || m.commandToPrint != "" || m.quitting || backend.writes.Load() != 0 { + t.Fatal("Ctrl+P exported an omitted command or bypassed an unavailable state") + } + if omitted && (m.previewCommand != "" || strings.Contains(m.View().Content, "Ctrl+P:")) { + t.Fatal("an omitted command retained its print action") + } + }) + } +} + +func TestLimitsWorkspacePrintPreservesFinalizationFailures(t *testing.T) { + for _, scenario := range []string{"caller cancellation", "interrupt", "program failure", "mutation failure", "terminal failure", "print failure", "short print"} { + t.Run(scenario, func(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + limitsControllerReview(t, m) + m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + ctx := t.Context() + want := errors.New("synthetic independent failure") + var out bytes.Buffer + opts := ShowJSONOpts{Context: ctx, Format: "text", Stdout: &out, Stderr: io.Discard} + var runErr error + switch scenario { + case "caller cancellation": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + want = context.Canceled + case "interrupt": + m.interrupted = true + want = context.Canceled + case "program failure": + runErr = want + case "mutation failure": + m.locked, m.mutationError = true, want + case "terminal failure": + m.output = &limitsWorkspaceOutput{err: want} + case "print failure": + opts.Stdout = limitsFailingWriter{err: want} + case "short print": + want = io.ErrShortWrite + opts.Stdout = limitsWorkspaceWriteFunc(func(data []byte) (int, error) { return 1, nil }) + } + err := finishLimitsWorkspace(ctx, m, runErr, opts) + if !errors.Is(err, want) || out.Len() != 0 { + t.Fatalf("print bypassed an independent failure: output=%q err=%v", out.String(), err) + } + if scenario == "interrupt" { + var interrupt *limitsInterrupt + if !errors.As(err, &interrupt) || interrupt.ExitCode() != 130 { + t.Fatal("print changed the interrupt exit status") + } + } + if scenario == "print failure" || scenario == "short print" { + if !strings.Contains(err.Error(), "Could not print the limits command") { + t.Fatalf("print failure lacks useful recovery: %v", err) + } + } + }) + } +} + +func TestLimitsWorkspacePrintPreservesSourceFailures(t *testing.T) { + for _, source := range []string{"costs", "first-cap-404", "costs-with-receipt"} { + for _, printFails := range []bool{false, true} { + t.Run(fmt.Sprintf("%s/print-fails-%t", source, printFails), func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + var sourceErr error = errors.New("synthetic Costs failure") + resource := "costs" + if source == "first-cap-404" { + sourceErr = &openai.Error{StatusCode: http.StatusNotFound} + resource = "spend_limit" + m.snapshot.LimitAbsent, m.snapshot.LimitUnconfirmed = false, true + } + m.snapshot.Errors = []limitsSourceFailure{{Resource: resource, Err: sourceErr}} + limitsControllerReview(t, m) + if source == "first-cap-404" && (!m.change.BeforeMissing || !strings.Contains(m.details, "creates or replaces")) { + t.Fatal("first-cap review lost its explicit uncertainty warning") + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + if quit == nil || m.commandToPrint == "" { + t.Fatal("a nonfatal source failure removed the print action") + } + if source == "costs-with-receipt" { + m.lastSaved = gjson.Parse("{\"object\":\"organization.spend_limit\",\"threshold_amount\":10001,\"currency\":\"USD\",\"interval\":\"month\"}") + m.lastSavedOperation = limitsChangeOperation("limit_set", false) + } + var out bytes.Buffer + opts := ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard} + if printFails { + opts.Stdout = limitsFailingWriter{err: syscall.EPIPE} + } + err := finishLimitsWorkspace(t.Context(), m, nil, opts) + if !errors.Is(err, sourceErr) || backend.writes.Load() != 0 { + t.Fatalf("printing hid the source failure or sent a mutation: %v", err) + } + if printFails { + if !errors.Is(err, syscall.EPIPE) || clioutput.IsBrokenPipe(err) || out.Len() != 0 { + t.Fatalf("print failure masked the independent source failure: %v", err) + } + return + } + if source == "costs-with-receipt" { + if !strings.HasSuffix(out.String(), m.commandToPrint+"\n") || !strings.Contains(strings.TrimSuffix(out.String(), m.commandToPrint+"\n"), "USD 100.01") { + t.Fatalf("source failure lost the confirmed receipt or complete command: %q", out.String()) + } + } else if out.String() != m.commandToPrint+"\n" { + t.Fatalf("source failure suppressed or changed the command: %q", out.String()) + } + }) + } + } +} + +func TestLimitsWorkspacePrintPreservesConfirmedReceipt(t *testing.T) { + for _, scenario := range []string{"confirmed result", "receipt failure", "confirmed removal"} { + t.Run(scenario, func(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + limitsControllerReview(t, m) + m.Update(tea.KeyPressMsg{Code: 'p', Mod: tea.ModCtrl}) + m.lastSaved = gjson.Parse("{\"object\":\"organization.spend_limit\",\"threshold_amount\":10001,\"currency\":\"USD\",\"interval\":\"month\"}") + m.lastSavedOperation = limitsChangeOperation("limit_set", false) + if scenario == "confirmed removal" { + m.lastSaved = gjson.Parse("{\"object\":\"organization.spend_limit.deleted\",\"deleted\":true}") + m.lastSavedOperation = limitsChangeOperation("limit_delete", false) + m.snapshot.LimitUnconfirmed = true + m.snapshot.Errors = []limitsSourceFailure{{Resource: "spend_limit", Err: &openai.Error{StatusCode: http.StatusNotFound}}} + } + var out bytes.Buffer + opts := ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard} + if scenario == "receipt failure" { + opts.Stdout = limitsFailingWriter{err: syscall.EPIPE} + } + err := finishLimitsWorkspace(t.Context(), m, nil, opts) + if scenario == "receipt failure" { + if !errors.Is(err, syscall.EPIPE) || out.Len() != 0 { + t.Fatalf("print ignored a receipt failure: %v", err) + } + return + } + if err != nil || !strings.HasSuffix(out.String(), m.commandToPrint+"\n") { + t.Fatalf("confirmed receipt suppressed the command: %q err=%v", out.String(), err) + } + want := "USD 100.01" + if scenario == "confirmed removal" { + want = "Removal confirmed. Current hard-limit configuration was not returned (HTTP 404)." + } + if !strings.Contains(strings.TrimSuffix(out.String(), m.commandToPrint+"\n"), want) { + t.Fatalf("printing lost the preceding confirmed result: %q", out.String()) + } + }) + } +} + +// This checks emitted PowerShell bytes; native PowerShell execution is separate. +func TestLimitsWorkspacePrintedPowerShellCommandRetainsCallOperator(t *testing.T) { + for _, width := range []int{40, 80} { + t.Run(fmt.Sprintf("columns-%d", width), func(t *testing.T) { + m, backend := newLimitsController(t, width, 12) + m.change, m.page, m.shell = limitsChange{Kind: "limit_set", Amount: 12525}, "review", "pwsh" + m.previewExecutable = "C:\\Program Files\\" + strings.Repeat("long ", 20) + "Owner's Tools\\openai.exe" + m.requestArgs = []string{"--organization=org owner's"} + m.setReview() + command := limitsControllerPrintedCommand(t, m) + want := "& '" + strings.ReplaceAll(m.previewExecutable, "'", "''") + "' '--organization=org owner''s' admin organization spend-limit update --currency USD --interval month --threshold-amount 12525\n" + if command != want || backend.writes.Load() != 0 { + t.Fatalf("print changed PowerShell syntax or sent a mutation: %q", command) + } + }) + } +} + +func limitsPreviewExecutable(t *testing.T, name string) (string, string) { + t.Helper() + directory := filepath.Join(t.TempDir(), "CLI preview's directory") + require.NoError(t, os.Mkdir(directory, 0700)) + path := filepath.Join(directory, name) + // This executable records shell arguments and stdin. It cannot call the API. + const script = `#!/bin/sh +set -eu +printf '%s\000' "$@" > "$LIMITS_PREVIEW_ARGS" +printf '%s' "${OPENAI_UNTRUSTED_STDIN-unset}" > "$LIMITS_PREVIEW_PROTECTED" +/bin/cat > "$LIMITS_PREVIEW_STDIN" +` + require.NoError(t, os.WriteFile(path, []byte(script), 0700)) + return path, directory +} + +type limitsPreviewCapture struct { + args []string + stdin []byte + protected string +} + +func executeLimitsPreview(t *testing.T, shell, command, pathDirectory string) limitsPreviewCapture { + t.Helper() + path := "/bin/" + shell + if _, err := os.Stat(path); err != nil { + if strings.Contains(","+os.Getenv("OPENAI_CLI_REQUIRE_NATIVE_SHELLS")+",", ","+shell+",") { + t.Fatalf("required native shell %s is unavailable", shell) + } + t.Skipf("native %s is unavailable", shell) + } + directory := t.TempDir() + argsPath, stdinPath, protectedPath := filepath.Join(directory, "argv"), filepath.Join(directory, "stdin"), filepath.Join(directory, "protected") + args := []string{"--noprofile", "--norc", "-c", command} + if shell == "zsh" { + args = []string{"-f", "-c", command} + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + process := exec.CommandContext(ctx, path, args...) + process.WaitDelay = 250 * time.Millisecond + process.Dir = t.TempDir() + process.Env = []string{"PATH=" + pathDirectory + ":/usr/bin:/bin", "LANG=en_US.UTF-8", + "LIMITS_PREVIEW_ARGS=" + argsPath, "LIMITS_PREVIEW_STDIN=" + stdinPath, + "LIMITS_PREVIEW_PROTECTED=" + protectedPath} + var stdout, stderr bytes.Buffer + process.Stdout, process.Stderr = &stdout, &stderr + require.NoError(t, process.Run(), "shell=%s stderr=%q", shell, stderr.String()) + require.NoError(t, ctx.Err()) + require.Empty(t, stdout.String()) + require.Empty(t, stderr.String()) + rawArgs, err := os.ReadFile(argsPath) + require.NoError(t, err) + require.True(t, bytes.HasSuffix(rawArgs, []byte{0})) + input, err := os.ReadFile(stdinPath) + require.NoError(t, err) + protected, err := os.ReadFile(protectedPath) + require.NoError(t, err) + return limitsPreviewCapture{strings.Split(string(rawArgs[:len(rawArgs)-1]), "\x00"), input, string(protected)} +} + +func TestLimitsWorkspacePrintedCommandNativeShellPreservesArguments(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("Bash/Zsh preview execution uses a POSIX recorder; native Windows requires separate coverage") + } + for _, shell := range []string{"bash", "zsh"} { + t.Run(shell, func(t *testing.T) { + binary, directory := limitsPreviewExecutable(t, strings.Repeat("long ", 20)+"limits'\nrecorder") + marker := filepath.Join(t.TempDir(), "MUST_NOT_EXIST") + subject := "Owner's \"alert\"; $(touch " + marker + "); `touch " + marker + "` $HOME \\path\nnext line" + channel, err := json.Marshal(map[string]any{"type": "email", "recipients": []string{"finance@example.test"}, "subject_prefix": subject}) + if err != nil { + t.Fatal(err) + } + requestArgs := []string{"--organization=org owner's", "--project=proj root's", "--base-url=https://example.invalid/proxy/v1"} + cases := []struct { + name string + projectID string + change limitsChange + want []string + }{ + {name: "organization-limit", change: limitsChange{Kind: "limit_set", Amount: 12525}, + want: []string{"admin", "organization", "spend-limit", "update", "--currency", "USD", "--interval", "month", "--threshold-amount", "12525"}}, + {name: "project-limit", projectID: "proj target's", change: limitsChange{Kind: "limit_set", Amount: 12525}, + want: []string{"admin", "organization", "projects", "spend-limit", "update", "--project-id", "proj target's", "--currency", "USD", "--interval", "month", "--threshold-amount", "12525"}}, + {name: "organization-limit-delete", change: limitsChange{Kind: "limit_delete"}, + want: []string{"admin", "organization", "spend-limit", "delete"}}, + {name: "project-limit-delete", projectID: "proj target's", change: limitsChange{Kind: "limit_delete"}, + want: []string{"admin", "organization", "projects", "spend-limit", "delete", "--project-id", "proj target's"}}, + {name: "alert-create", change: limitsChange{Kind: "alert_create", Amount: 7550, Recipients: []string{"finance@example.test"}, SubjectPrefixSet: true, SubjectPrefix: &subject}, + want: []string{"admin", "organization", "spend-alerts", "create", "--currency", "USD", "--interval", "month", "--threshold-amount", "7550", "--notification-channel", string(channel)}}, + {name: "alert-update", projectID: "proj target's", change: limitsChange{Kind: "alert_update", ID: "alert owner's", Amount: 15025, Recipients: []string{"finance@example.test"}, SubjectPrefixSet: true, SubjectPrefix: &subject}, + want: []string{"admin", "organization", "projects", "spend-alerts", "update", "--project-id", "proj target's", "--alert-id", "alert owner's", "--currency", "USD", "--interval", "month", "--threshold-amount", "15025", "--notification-channel", string(channel)}}, + {name: "alert-delete", projectID: "proj target's", change: limitsChange{Kind: "alert_delete", ID: "alert owner's"}, + want: []string{"admin", "organization", "projects", "spend-alerts", "delete", "--project-id", "proj target's", "--alert-id", "alert owner's"}}, + {name: "rate-update", projectID: "proj target's", change: limitsChange{Kind: "rate_update", ID: "rl owner's", RateValues: map[string]int64{"max_requests_per_1_minute": -1}}, + want: []string{"admin", "organization", "projects", "rate-limits", "update-rate-limit", "--project-id", "proj target's", "--rate-limit-id", "rl owner's", "--max-requests-per-1-minute", "-1"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + for _, width := range []int{40, 80} { + t.Run(fmt.Sprintf("columns-%d", width), func(t *testing.T) { + m, backend := newLimitsController(t, width, 12) + m.change, m.page, m.shell = tc.change, "review", shell + m.snapshot.ProjectID, m.requestArgs, m.previewExecutable = tc.projectID, requestArgs, binary + m.setReview() + command := limitsControllerPrintedCommand(t, m) + if backend.writes.Load() != 0 { + t.Fatal("printing a command wrote to the backend") + } + capture := executeLimitsPreview(t, shell, command, directory) + want := append(append([]string{}, requestArgs...), tc.want...) + if !reflect.DeepEqual(capture.args, want) { + t.Fatalf("printed command changed arguments:\n got: %q\nwant: %q", capture.args, want) + } + if len(capture.stdin) != 0 || capture.protected != "unset" { + t.Fatal("printed command unexpectedly supplied stdin or changed its environment") + } + if _, err := os.Stat(marker); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("printed command executed subject text: %v", err) + } + }) + } + }) + } + }) + } +} + +func TestLimitsWorkspaceControlCFromEveryState(t *testing.T) { + for _, page := range []string{"home", "form", "review", "details", "busy"} { + t.Run(page, func(t *testing.T) { + m, b := newLimitsController(t, 40, 12) + var pending tea.Cmd + if page == "busy" { + b.release = make(chan struct{}) + change := limitsChange{Kind: "limit_set", Amount: 10001, BeforeMissing: true} + pending = m.launch(&change) + limitsControllerStarted(t, m, b) + } else { + m.page = page + } + _, quit := m.Update(tea.KeyPressMsg{Code: 'c', Mod: tea.ModCtrl}) + if quit == nil || !m.quitting || !m.interrupted || !errors.Is(m.ctx.Err(), context.Canceled) || m.View().Content != "" { + t.Fatal("Ctrl+C did not stop work and clear the owned view") + } + if pending != nil && !errors.Is(limitsControllerRun(t, pending).err, context.Canceled) { + t.Fatal("Ctrl+C did not cancel the pending backend request") + } + m.work.Wait() + }) + } +} + +func TestLimitsWorkspacePreflightFailureRequiresRefresh(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + b.mutationErr = &limitsMutationFailure{Err: errors.New("synthetic preflight conflict"), Attempted: false} + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + m.Update(limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter))) + limitsControllerStarted(t, m, b) + if m.locked || !m.refreshRequired || m.page != "form" || m.draft.inputs[0].Value() != "100.01" { + t.Fatal("preflight conflict did not retain a refreshable draft") + } + m.setFocus(len(m.draft.inputs)) + limitsControllerKey(m, tea.KeyEnter) + if m.page != "form" || m.busy || b.writes.Load() != 1 { + t.Fatal("preflight conflict allowed review before refresh") + } + b.snapshot.LimitAbsent = false + b.snapshot.Limit = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":20000,"currency":"USD","interval":"month","enforcement":{"status":"enforcing"}}`) + m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) + if m.refreshRequired || m.locked || m.draft.inputs[0].Value() != "100.01" || m.draft.change.Before.Raw != b.snapshot.Limit.Raw || m.draft.change.BeforeMissing { + t.Fatal("refresh discarded user input or failed to reconcile the baseline") + } + limitsControllerKey(m, tea.KeyEnter) + if m.page != "review" || m.apply || m.change.Before.Raw != b.snapshot.Limit.Raw || m.change.Amount != 10001 { + t.Fatal("refreshed review lost old/new distinction or selected Apply") + } + b.mutationErr = nil + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter)) + change := limitsControllerStarted(t, m, b) + if b.writes.Load() != 2 || change.Before.Raw != b.snapshot.Limit.Raw || change.Amount != 10001 { + t.Fatal("explicit retry used a stale baseline or changed the draft") + } +} + +func TestLimitsWorkspaceAttemptedFailureStaysLockedAfterRefresh(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + b.mutationErr = &limitsMutationFailure{Err: errors.New("synthetic uncertain response"), Attempted: true} + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + m.Update(limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter))) + limitsControllerStarted(t, m, b) + if !m.locked || m.refreshRequired { + t.Fatal("attempted write failure became a safe preflight rejection") + } + limitsControllerKey(m, tea.KeyEscape) + m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) + limitsControllerKey(m, tea.KeyEnter) + m.work.Wait() + if !m.locked || m.page != "home" || b.writes.Load() != 1 { + t.Fatal("refresh silently unlocked an uncertain write") + } +} + +func TestLimitsWorkspaceRefreshClearsStaleStatusAndRewrapsDetails(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.Update(limitsWorkResult{snapshot: limitsSnapshot{Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: errors.New("synthetic failure")}}}, err: errors.New("synthetic failure")}) + if !strings.Contains(m.note, "failed") { + t.Fatal("partial load did not report failure") + } + m.Update(limitsWorkResult{snapshot: limitsSnapshot{LimitAbsent: true, AlertsComplete: true}}) + if strings.Contains(m.note, "failed") { + t.Fatal("successful refresh retained stale failure status") + } + m.page = "details" + m.setDetails(strings.Repeat("界", 60)) + m.Update(tea.WindowSizeMsg{Width: 40, Height: 12}) + view := ansi.Strip(m.View().Content) + for _, line := range strings.Split(view, "\n") { + if ansi.StringWidth(line) > 40 { + t.Errorf("resized details overflowed display cells: %q", line) + } + } + if !strings.Contains(view, "Ctrl+C") { + t.Fatal("resized details hid cancellation") + } +} + +func TestLimitsWorkspaceNoLimitPreservesOtherEditBaselines(t *testing.T) { + for _, kind := range []string{"alert_update", "alert_delete", "rate_update"} { + t.Run(kind, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + value := gjson.Parse(limitsControllerAlert) + if kind == "rate_update" { + value = gjson.Parse(limitsControllerRate) + } + m.begin(kind, value) + if m.change.Kind != kind || m.change.BeforeMissing || m.change.Before.Raw != value.Raw || m.change.ID != value.Get("id").Str { + t.Fatalf("absent hard limit corrupted %s baseline: %+v", kind, m.change) + } + }) + } +} + +func TestLimitsDraftExactAmountsAndPrefixPresence(t *testing.T) { + for _, tc := range []struct { + kind, amount string + want int64 + valid bool + }{ + {"limit_set", "92233720368547758.07", math.MaxInt64, true}, + {"limit_set", "0.01", 1, true}, {"limit_set", "0", 0, false}, + {"alert_create", "0", 0, true}, {"alert_create", "1.001", 0, false}, + {"limit_set", "92233720368547758.08", 0, false}, + } { + draft, err := newLimitsDraft(limitsChange{Kind: tc.kind, BeforeMissing: true}) + if err != nil { + t.Fatal(err) + } + draft.inputs[0].SetValue(tc.amount) + if tc.kind == "alert_create" { + draft.inputs[1].SetValue("synthetic@example.invalid") + } + change, err := draft.validate() + if (err == nil) != tc.valid || tc.valid && change.Amount != tc.want { + t.Errorf("%s amount %q: cents=%d error=%v", tc.kind, tc.amount, change.Amount, err) + } + } + for _, tc := range []struct { + name, original, value string + mode int + set, null bool + }{ + {name: "omitted remains omitted"}, + {name: "null remains null", original: `,"subject_prefix":null`, set: true, null: true}, + {name: "empty remains empty", original: `,"subject_prefix":""`, set: true}, + {name: "preserve string", original: `,"subject_prefix":"Existing"`, value: "Existing", set: true}, + {name: "replace empty", original: `,"subject_prefix":"Existing"`, mode: 1, set: true}, + {name: "clear null", original: `,"subject_prefix":"Existing"`, mode: 2, set: true, null: true}, + {name: "edited value", value: "New prefix", set: true}, + {name: "edited original", original: `,"subject_prefix":"Existing"`, value: "New prefix", set: true}, + {name: "edited original to empty", original: `,"subject_prefix":"Existing"`, set: true}, + } { + t.Run(tc.name, func(t *testing.T) { + raw := strings.Replace(limitsControllerAlert, `["synthetic@example.invalid"]}`, `["synthetic@example.invalid"]`+tc.original+`}`, 1) + draft, err := newLimitsDraft(limitsChange{Kind: "alert_update", ID: "alert_synthetic", Before: gjson.Parse(raw)}) + if err != nil { + t.Fatal(err) + } + draft.prefixMode = tc.mode + draft.inputs[2].SetValue(tc.value) + change, err := draft.validate() + if err != nil || change.SubjectPrefixSet != tc.set || tc.set && (change.SubjectPrefix == nil) != tc.null { + t.Fatalf("prefix presence changed: %+v; error=%v", change, err) + } + if tc.set && !tc.null && *change.SubjectPrefix != tc.value { + t.Errorf("prefix content changed: %q", *change.SubjectPrefix) + } + }) + } +} + +func TestLimitsDraftRatesSendOnlyExplicitChanges(t *testing.T) { + draft, err := newLimitsDraft(limitsChange{Kind: "rate_update", ID: "rl_synthetic", Before: gjson.Parse(limitsControllerRate)}) + if err != nil { + t.Fatal(err) + } + if draft.inputs[2].Value() != "" || draft.inputs[3].Value() != "0" || draft.inputs[4].Value() != "" { + t.Fatal("draft conflated missing rates with explicit zero") + } + draft.inputs[0].SetValue("") // Blank retains the existing value. + draft.inputs[1].SetValue("201") + draft.inputs[4].SetValue("0") // Explicit zero remains a real change. + change, err := draft.validate() + want := map[string]int64{"max_tokens_per_1_minute": 201, "max_audio_megabytes_per_1_minute": 0} + if err != nil || !reflect.DeepEqual(change.RateValues, want) { + t.Fatalf("rate changes=%v error=%v; want %v", change.RateValues, err, want) + } + for _, tc := range []struct { + text string + value int64 + }{{"-1", -1}, {"-9223372036854775808", math.MinInt64}} { + draft.inputs[1].SetValue(tc.text) + change, err := draft.validate() + if err != nil || change.RateValues["max_tokens_per_1_minute"] != tc.value { + t.Errorf("signed rate %q was not preserved: %v, %v", tc.text, change.RateValues, err) + } + } + for _, invalid := range []string{"-", "1.5", "1e3", "9223372036854775808", "-9223372036854775809"} { + draft.inputs[1].SetValue(invalid) + if _, err := draft.validate(); err == nil { + t.Errorf("rate draft accepted %q", invalid) + } + } +} + +func TestLimitsWorkspaceActionVisibleAtMinimumSize(t *testing.T) { + for _, kind := range []string{"limit_set", "alert_create", "rate_update"} { + t.Run(kind, func(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + value := gjson.Result{} + if kind == "rate_update" { + value = gjson.Parse(limitsControllerRate) + } + m.begin(kind, value) + for focus := 0; focus <= len(m.draft.inputs); focus++ { + m.setFocus(focus) + view := ansi.Strip(m.View().Content) + if !strings.Contains(view, "Review change") || !strings.Contains(view, "Ctrl+C") { + t.Errorf("%s focus %d hides action or cancel: %q", kind, focus, view) + } + if lines := strings.Split(view, "\n"); len(lines) > 12 { + t.Errorf("minimum view uses %d rows", len(lines)) + } else { + for _, line := range lines { + if ansi.StringWidth(line) > 40 { + t.Errorf("minimum view exceeds 40 columns: %q", line) + } + } + } + } + if kind == "rate_update" { + m.draft.inputs[0].SetValue("101") + } else { + m.draft.inputs[0].SetValue("100.01") + if kind == "alert_create" { + m.draft.inputs[1].SetValue("synthetic@example.invalid") + m.draft.inputs[2].SetValue("Edited prefix") + if view := ansi.Strip(m.View().Content); !strings.Contains(view, "Prefix: Keep unless edited") { + t.Fatalf("edited prefix has misleading mode guidance: %q", view) + } + } + } + m.review() + if m.page != "review" { + t.Fatalf("fixture could not enter review: %s", m.formError) + } + if kind == "alert_create" && (!m.change.SubjectPrefixSet || m.change.SubjectPrefix == nil || *m.change.SubjectPrefix != "Edited prefix") { + t.Fatalf("review did not retain the edited prefix: %+v", m.change) + } + m.viewport.GotoBottom() + view := ansi.Strip(m.View().Content) + if !strings.Contains(view, "> Back") || !strings.Contains(view, "Apply change") || !strings.Contains(view, "Ctrl+C") { + t.Errorf("minimum review hides confirmation: %q", view) + } + }) + } +} + +func TestLimitsWorkspaceHelpRestoresDraftAndReview(t *testing.T) { + t.Run("form", func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.begin("alert_update", gjson.Parse(limitsControllerAlert)) + values := []string{"123.45", "finance@example.test", "Edited $HOME prefix"} + for i, value := range values { + m.draft.inputs[i].SetValue(value) + m.draft.inputs[i].SetCursor(i + 1) + } + for mode := 0; mode < 3; mode++ { + m.draft.prefixMode = mode + for focus := 0; focus < 5; focus++ { + m.setFocus(focus) + before, err := m.draft.validate() + require.NoError(t, err) + for _, key := range []rune{tea.KeyF1, tea.KeyF1, '?', tea.KeyEscape} { + require.Nil(t, limitsControllerKey(m, key)) + } + require.Equal(t, "form", m.page) + require.Equal(t, focus, m.focus) + require.Equal(t, mode, m.draft.prefixMode) + for i, value := range values { + require.Equal(t, value, m.draft.inputs[i].Value()) + require.Equal(t, i == focus, m.draft.inputs[i].Focused()) + require.Equal(t, i+1, m.draft.inputs[i].Position()) + } + after, err := m.draft.validate() + require.NoError(t, err) + require.Equal(t, limitsChangeBody(before), limitsChangeBody(after)) + } + } + require.Zero(t, backend.writes.Load()) + require.Zero(t, backend.reads.Load()) + }) + for _, resized := range []bool{false, true} { + t.Run(fmt.Sprintf("review_resize_%t", resized), func(t *testing.T) { + m, backend := newLimitsController(t, 80, 24) + limitsControllerReview(t, m) + command := m.previewCommand + m.viewport.GotoBottom() + review := m.viewport.View() + limitsControllerKey(m, tea.KeyTab) + require.True(t, m.apply) + limitsControllerKey(m, tea.KeyF1) + limitsControllerKey(m, tea.KeyF1) + if resized { + m.Update(tea.WindowSizeMsg{Width: 40, Height: 13}) + } + limitsControllerKey(m, tea.KeyEscape) + require.Equal(t, "review", m.page) + require.False(t, m.apply) + require.Equal(t, command, m.previewCommand) + require.Equal(t, "100.01", m.draft.inputs[0].Value()) + if resized { + require.Zero(t, m.viewport.YOffset()) + require.False(t, m.viewport.AtBottom()) + limitsControllerKey(m, tea.KeyTab) + require.False(t, m.apply, "resize must require rereading before Apply") + } else { + require.Equal(t, review, m.viewport.View()) + require.True(t, m.viewport.AtBottom()) + } + require.Zero(t, backend.writes.Load()) + }) + } + t.Run("home_and_inspection", func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.snapshot.Alerts = []gjson.Result{gjson.Parse(limitsControllerAlert)} + m.section, m.selection = 1, 1 + for _, key := range []rune{tea.KeyF1, tea.KeyF1, tea.KeyEscape, 'i', tea.KeyF1, tea.KeyEscape} { + require.Nil(t, limitsControllerKey(m, key)) + } + require.Equal(t, "home", m.page) + require.Equal(t, 1, m.section) + require.Equal(t, 1, m.selection) + require.False(t, m.quitting) + require.Zero(t, backend.writes.Load()) + }) +} + +func TestLimitsWorkspaceReviewEscapeMatchesBack(t *testing.T) { + for _, key := range []rune{tea.KeyEscape, tea.KeyEnter} { + t.Run(fmt.Sprintf("key_%d", key), func(t *testing.T) { + m, backend := newLimitsController(t, 40, 13) + m.begin("alert_update", gjson.Parse(limitsControllerAlert)) + m.draft.inputs[0].SetValue("125.25") + m.draft.inputs[2].SetValue("Changed prefix") + m.draft.prefixMode = 1 + m.setFocus(4) + limitsControllerKey(m, tea.KeyEnter) + require.Equal(t, "review", m.page) + require.False(t, m.apply) + if key == tea.KeyEscape { + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + require.True(t, m.apply) + } + limitsControllerKey(m, key) + require.Equal(t, "form", m.page) + require.Equal(t, 4, m.focus) + require.Equal(t, 1, m.draft.prefixMode) + require.Equal(t, "125.25", m.draft.inputs[0].Value()) + require.Equal(t, "Changed prefix", m.draft.inputs[2].Value()) + require.False(t, m.apply) + limitsControllerKey(m, tea.KeyEnter) + require.Equal(t, "review", m.page) + require.False(t, m.apply) + require.Equal(t, int64(12525), m.change.Amount) + require.Equal(t, "Changed prefix", *m.change.SubjectPrefix) + require.Zero(t, backend.writes.Load()) + }) + } +} + +func TestLimitsWorkspaceFormDiagnosticsFitAndRemainAccessible(t *testing.T) { + for _, width := range []int{40, 80} { + for _, height := range []int{12, 13, 14, 24} { + for _, state := range []string{"validation", "preflight", "uncertain"} { + t.Run(fmt.Sprintf("%dx%d_%s", width, height, state), func(t *testing.T) { + m, backend := newLimitsController(t, width, height) + m.begin("alert_create", gjson.Result{}) + m.draft.inputs[0].SetValue("125.25") + m.draft.inputs[1].SetValue("Name ") + m.review() + require.Equal(t, "form", m.page) + require.Contains(t, m.formError, "without display names") + if state != "validation" { + m.draft.inputs[1].SetValue("finance@example.test") + m.review() + m.mutationError = &limitsMutationFailure{ + Attempted: state == "uncertain", + Err: &openai.Error{StatusCode: 400, Message: "synthetic-private-response"}, + } + m.Update(limitsWorkResult{mutation: true, err: m.mutationError}) + } + outcome := strings.SplitN(m.formError, "\n", 2)[0] + for focus := 0; focus < 5; focus++ { + m.setFocus(focus) + for _, dark := range []bool{false, true} { + m.color, m.dark = false, dark + raw := m.View().Content + var monochrome bytes.Buffer + writer := colorprofile.Writer{Forward: &monochrome, Profile: colorprofile.NoTTY} + _, err := io.WriteString(&writer, raw) + require.NoError(t, err) + view := monochrome.String() + require.Contains(t, strings.Join(strings.Fields(view), " "), outcome) + for _, control := range []string{"Review change", "F1: help", "Esc: back", "Ctrl+C"} { + require.Contains(t, view, control) + } + if state == "preflight" { + require.Contains(t, view, "r: refresh") + } + require.NotContains(t, view, "\x1b") + require.NotContains(t, view, "synthetic-private-response") + require.LessOrEqual(t, len(strings.Split(view, "\n")), height) + for _, line := range strings.Split(view, "\n") { + require.LessOrEqual(t, ansi.StringWidth(line), width) + } + } + } + limitsControllerKey(m, tea.KeyF1) + require.Equal(t, "details", m.page) + require.NotContains(t, m.details, "synthetic-private-response") + if state == "validation" { + require.Contains(t, m.details, outcome) + } else if state == "preflight" { + require.Contains(t, m.details, outcome) + require.Contains(t, m.details, "request fields and scope") + } else { + require.Contains(t, m.details, "It may have succeeded") + } + m.viewport.GotoBottom() + require.True(t, m.viewport.AtBottom()) + limitsControllerKey(m, tea.KeyEscape) + require.Equal(t, "form", m.page) + require.Equal(t, 4, m.focus) + require.Equal(t, "125.25", m.draft.inputs[0].Value()) + require.Zero(t, backend.writes.Load()) + }) + } + } + } +} + +func TestLimitsWorkspaceListStatusStaysVisible(t *testing.T) { + for _, complete := range []bool{false, true} { + t.Run(fmt.Sprintf("complete_%t", complete), func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.snapshot.ProjectID = "proj_synthetic" + m.snapshot.Alerts = []gjson.Result{gjson.Parse(limitsControllerAlert)} + m.snapshot.Rates = []gjson.Result{gjson.Parse(limitsControllerRate)} + m.snapshot.AlertsComplete, m.snapshot.RatesComplete = complete, complete + m.snapshot.CostsTotal, m.snapshot.CostsComplete = "1.23", false + for _, section := range []int{1, 2} { + m.section = section + status, label, tab := "Incomplete.", "Alerts loaded: 1", "[Alerts]" + if complete { + status = "Complete." + } + if section == 2 { + label, tab = "Rate records loaded: 1", "[Rates]" + } + for _, colored := range []bool{false, true} { + for _, dark := range []bool{false, true} { + m.color, m.dark = colored, dark + view := ansi.Strip(m.View().Content) + require.Contains(t, view, status+" "+label) + require.Contains(t, view, tab) + require.Contains(t, view, "Reported Costs (incomplete): USD 1.23") + require.Contains(t, view, "q: quit") + require.NotContains(t, view, "true") + require.NotContains(t, view, "false") + require.LessOrEqual(t, len(strings.Split(view, "\n")), m.height) + for _, line := range strings.Split(view, "\n") { + require.LessOrEqual(t, ansi.StringWidth(line), m.width) + } + } + } + } + require.Zero(t, backend.writes.Load()) + }) + } +} + +func TestLimitsWorkspaceReviewShowsExactDollarChanges(t *testing.T) { + for _, tc := range []struct { + name, kind, before, amount, previous, next string + }{ + {"limit", "limit_set", limitsTestLimit, "125.25", "USD 100.00 / month", "USD 125.25 / month"}, + {"alert", "alert_update", limitsControllerAlert, "0.00", "USD 100.00 / month", "USD 0.00 / month"}, + {"new_alert", "alert_create", "", "0.01", "New alert", "USD 0.01 / month"}, + {"unknown_limit", "limit_set", "", "125.25", "Not confirmed", "USD 125.25 / month"}, + {"large_limit", "limit_set", strings.Replace(limitsTestLimit, "10000", "9223372036854775806", 1), "92233720368547758.07", "USD 92233720368547758.06 / month", "USD 92233720368547758.07 / month"}, + {"remove_limit", "limit_delete", limitsTestLimit, "", "USD 100.00 / month", "Remove this resource"}, + {"remove_alert", "alert_delete", limitsControllerAlert, "", "USD 100.00 / month", "Remove this resource"}, + } { + t.Run(tc.name, func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.snapshot.LimitAbsent = tc.before == "" + m.begin(tc.kind, gjson.Parse(tc.before)) + if len(m.draft.inputs) != 0 { + m.draft.inputs[0].SetValue(tc.amount) + if tc.kind == "alert_create" { + m.draft.inputs[1].SetValue("finance@example.test") + } + action := len(m.draft.inputs) + if action == 3 { + action++ + } + m.setFocus(action) + limitsControllerKey(m, tea.KeyEnter) + } + require.Equal(t, "review", m.page, m.formError) + require.False(t, m.apply) + require.Contains(t, m.details, "Before: "+tc.previous) + require.Contains(t, m.details, "After: "+tc.next) + if tc.before != "" { + require.Contains(t, m.details, tc.before) + } + if tc.amount != "" { + require.Contains(t, m.previewCommand, fmt.Sprintf("--threshold-amount %d", m.change.Amount)) + require.Contains(t, m.details, fmt.Sprintf(`"threshold_amount": %d`, m.change.Amount)) + } + require.Contains(t, m.details, "\n"+m.previewCommand+"\n") + require.Zero(t, backend.writes.Load()) + }) + } +} + +func TestLimitsDraftCommandPreviewPreservesPublicRoutes(t *testing.T) { + for _, tc := range []struct { + change limitsChange + wants []string + }{ + {limitsChange{Kind: "limit_set", Amount: 10001, BeforeMissing: true}, []string{"spend-limit update", "--currency USD", "--threshold-amount 10001"}}, + {limitsChange{Kind: "alert_update", ID: "alert_synthetic", Amount: 10001, Recipients: []string{"synthetic@example.invalid"}}, []string{"spend-alerts update", "--alert-id alert_synthetic", "--currency USD"}}, + {limitsChange{Kind: "rate_update", ID: "rl_synthetic", RateValues: map[string]int64{"max_requests_per_1_minute": 0}}, []string{"rate-limits update-rate-limit", "--rate-limit-id rl_synthetic", "--max-requests-per-1-minute 0"}}, + } { + preview := limitsReview(tc.change, "proj_synthetic", nil, false, "bash") + for _, want := range append(tc.wants, "openai admin organization projects", "--project-id proj_synthetic") { + if !strings.Contains(preview, want) { + t.Errorf("preview missing %q: %q", want, preview) + } + } + if tc.change.Kind != "rate_update" && limitsChangeBody(tc.change)["currency"] != "USD" { + t.Error("review body changed API currency spelling") + } + } +} + +func TestLimitsDraftPreviewNamesDirectCommandDifferences(t *testing.T) { + change := limitsChange{Kind: "limit_set", Amount: 10001, BeforeMissing: true} + for _, shell := range []string{"bash", "zsh", "fish", "pwsh"} { + t.Run(shell, func(t *testing.T) { + preview := limitsReview(change, "", nil, false, shell, "/synthetic path/openai") + for _, want := range []string{"Command preview (inspection only)", "Ctrl+P prints the complete command", "skips the workspace preflight", "can retry automatically", "creates or replaces"} { + if !strings.Contains(preview, want) { + t.Errorf("preview omitted %q", want) + } + } + if strings.Contains(preview, "Equivalent command") || strings.Contains(preview, "Not configured") { + t.Fatal("preview claimed unsupported equivalence or confirmed absence") + } + command, _ := limitsCommand(change, "", nil, false, shell, "/synthetic path/openai") + if shell == "pwsh" && !strings.HasPrefix(command, "& '/synthetic path/openai' ") { + t.Fatalf("PowerShell preview cannot invoke the quoted executable: %q", command) + } + }) + } + deletion := limitsReview(limitsChange{Kind: "limit_delete"}, "", nil, true, "bash") + if strings.Contains(deletion, "does not preserve a disabled budget") || !strings.Contains(deletion, "does not guarantee soft-budget behavior") { + t.Fatal("hard-limit removal claimed unverified soft-budget semantics") + } +} + +func TestLimitsWorkspaceUnknownLimitRequiresExplicitReview(t *testing.T) { + m, b := newLimitsController(t, 80, 24) + m.snapshot = limitsSnapshot{LimitUnconfirmed: true, Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: &openai.Error{StatusCode: 404}}}} + if rows := m.menu(); len(rows) != 1 || rows[0].kind != "limit_set" || !strings.Contains(m.sourceSummary(), "unknown") { + t.Fatal("unknown limit lost its explicit create-or-replace path or became confirmed absence") + } + limitsControllerReview(t, m) + if !m.change.BeforeMissing || m.apply || !strings.Contains(m.details, "The current limit could not be confirmed.") || b.writes.Load() != 0 { + t.Fatal("unknown-baseline change bypassed its warning and Back default") + } + m.page = "home" + m.snapshot.Errors = append(m.snapshot.Errors, limitsSourceFailure{Resource: "project", Err: errors.New("synthetic project failure")}) + if len(m.menu()) != 0 { + t.Fatal("unverified project scope allowed a limit change") + } + m.snapshot = limitsSnapshot{Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: &openai.Error{StatusCode: 401}}}} + if len(m.menu()) != 0 { + t.Fatal("an authentication failure allowed an assumed-absent limit change") + } +} + +func TestLimitsPreflightGuidancePreservesFailureScope(t *testing.T) { + for _, tc := range []struct { + status int + code, want, reject string + }{ + {401, "", "help setup admin", "help setup\n"}, + {403, "", "Check Admin-key permissions", "invalid key"}, + {404, "", "was not returned", "not configured"}, + {429, "organization_spend_limit_exceeded", "help admin organization spend-limit retrieve", "Wait before"}, + {429, "project_spend_limit_exceeded", "help admin organization projects spend-limit retrieve", "Wait before"}, + {429, "credit_balance_exhausted", "billing or usage limit", "Rate limited"}, + {429, "unknown_synthetic_code", "Wait before refreshing", "spend limit was reached"}, + } { + err := &openai.Error{StatusCode: tc.status, Code: tc.code, Message: "private synthetic response prose"} + got := limitsPreflightHint(err, "custom-openai") + if !strings.Contains(got, tc.want) || strings.Contains(got, tc.reject) || strings.Contains(got, err.Message) || !strings.Contains(got, "No write sent.") { + t.Errorf("status=%d code=%s yielded unsafe or incorrect recovery: %q", tc.status, tc.code, got) + } + for _, hint := range []string{limitsReadHint(err, "custom-openai"), limitsMutationHint(err, "custom-openai")} { + if !strings.Contains(hint, tc.want) || strings.Contains(hint, tc.reject) || strings.Contains(hint, err.Message) || strings.Contains(hint, "No write sent.") { + t.Errorf("status=%d code=%s lost its read or mutation classification: %q", tc.status, tc.code, hint) + } + } + } +} + +func TestLimitsWorkspaceHTTPFailureRecovery(t *testing.T) { + for _, status := range []int{400, 422, 409, 503} { + for _, operation := range []string{"read", "preflight", "mutation"} { + t.Run(fmt.Sprintf("%s/%d", operation, status), func(t *testing.T) { + var writes, preflightReads atomic.Int32 + var rejectPreflight atomic.Bool + backend := limitsTestBackend(t, "", 100, func(w http.ResponseWriter, r *http.Request) { + preflight := r.Method == http.MethodGet && operation == "preflight" && rejectPreflight.Load() && strings.HasSuffix(r.URL.Path, "/spend_limit") + if r.Method == http.MethodGet && (operation == "mutation" || operation == "preflight" && !preflight) { + if strings.HasSuffix(r.URL.Path, "/spend_limit") { + limitsTestWrite(t, w, limitsTestLimit) + } else if strings.HasSuffix(r.URL.Path, "/costs") { + limitsTestWrite(t, w, `{"object":"page","data":[],"has_more":false}`) + } else { + limitsTestWrite(t, w, `{"object":"list","data":[],"has_more":false}`) + } + return + } + if r.Method != http.MethodGet { + writes.Add(1) + } + if preflight { + preflightReads.Add(1) + } + w.WriteHeader(status) + limitsTestWrite(t, w, `{"error":{"message":"synthetic-private-message\u001b[2J","param":"synthetic-private-param\nline","code":"synthetic-private-code\u001b]0;title\u0007"}}`) + }) + m, _ := newLimitsController(t, 80, 24) + m.backend = backend + var result limitsWorkResult + if operation == "read" { + result = limitsControllerRun(t, m.load()) + } else { + if operation == "preflight" { + overview := limitsControllerRun(t, m.load()) + if overview.err != nil || len(overview.snapshot.Errors) != 0 || overview.snapshot.Limit.Raw != limitsTestLimit { + t.Fatal("preflight fixture did not load a successful overview") + } + m.Update(overview) + rejectPreflight.Store(true) + } else { + backend.client = openai.NewClient(append(backend.client.Options, option.WithMaxRetries(3))...) + m.snapshot.LimitAbsent, m.snapshot.Limit = false, gjson.Parse(limitsTestLimit) + } + limitsControllerReview(t, m) + m.viewport.GotoBottom() + limitsControllerKey(m, tea.KeyTab) + result = limitsControllerRun(t, limitsControllerKey(m, tea.KeyEnter)) + var attempted *limitsMutationFailure + wantAttempted, wantWrites := operation == "mutation", int32(0) + if wantAttempted { + wantWrites = 1 + } + if !errors.As(result.err, &attempted) || attempted.Attempted != wantAttempted || writes.Load() != wantWrites { + t.Fatal("failed operation changed its attempted state or mutation count") + } + } + m.Update(result) + if operation == "preflight" { + if m.locked || !m.refreshRequired || m.page != "form" || m.draft.inputs[0].Value() != "100.01" || m.change.Before.Raw != limitsTestLimit || m.change.Amount != 10001 { + t.Fatal("HTTP preflight failure discarded the draft or changed its recovery state") + } + if !strings.Contains(m.formError, fmt.Sprintf("HTTP %d", status)) || strings.Contains(m.formError, "synthetic-private") || strings.Contains(m.formError, "\x1b") { + t.Fatal("preflight form omitted the status or exposed response data") + } + m.setFocus(len(m.draft.inputs)) + if next := limitsControllerKey(m, tea.KeyEnter); next != nil || m.page != "form" || m.busy || !m.refreshRequired || preflightReads.Load() != 1 || writes.Load() != 0 { + t.Fatal("HTTP preflight failure allowed review or another request before refresh") + } + } + limitsControllerKey(m, tea.KeyF1) + if !strings.Contains(m.details, fmt.Sprintf("HTTP %d", status)) || strings.Contains(m.details, "synthetic-private") || strings.Contains(m.details, "\x1b") { + t.Fatal("recovery omitted the status or exposed response data") + } + if status == 400 || status == 422 { + fields := "request fields and scope" + if operation == "mutation" { + fields = "reviewed fields" + } + if !strings.Contains(m.details, fields) { + t.Fatal("validation recovery did not identify fields to recheck") + } + } + if operation == "read" { + if writes.Load() != 0 || !strings.Contains(m.details, "No change was attempted.") || strings.Contains(m.details, "It may have succeeded") { + t.Fatal("read failure was presented as an attempted mutation") + } + return + } + if operation == "preflight" { + if !strings.Contains(m.details, "No write sent.") || !strings.Contains(m.details, "press r to refresh") && !strings.Contains(m.details, "Press r to refresh") || strings.Contains(m.details, "It may have succeeded") { + t.Fatal("preflight recovery lost safe refresh guidance or claimed a possible mutation") + } + if (status == 409 || status == 503) && !strings.Contains(m.details, "API availability and access") { + t.Fatal("preflight recovery omitted availability and access guidance") + } + } else { + if !m.locked || !strings.Contains(m.details, "It may have succeeded") || strings.Contains(m.details, "No write sent.") { + t.Fatal("HTTP failure removed mutation uncertainty or the write lock") + } + limitsControllerKey(m, tea.KeyEscape) + if m.page != "form" { + t.Fatal("recovery help did not restore the locked draft") + } + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, tea.KeyEnter) + m.Update(limitsControllerRun(t, limitsControllerKey(m, 'r'))) + limitsControllerKey(m, tea.KeyEnter) + if !m.locked || writes.Load() != 1 { + t.Fatal("Enter or refresh repeated a failed mutation") + } + } + m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) + if !errors.Is(failure, result.err) || !errors.Is(failure, syscall.EPIPE) || clioutput.IsBrokenPipe(failure) { + t.Fatal("finalization discarded the operation or output failure") + } + baseline := "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + if operation == "preflight" { + baseline = fmt.Sprintf("The API returned HTTP %d. No write sent.\n", status) + if status == 400 || status == 422 { + baseline += "Recheck the request fields and scope.\nReopen the limits workspace and review current settings before another change." + } else { + baseline += "Reopen the limits workspace and review current settings before another change. If this continues, check API availability and access." + } + baseline += "\n" + m.readRecovery() + } + for _, args := range [][]string{nil, {"--format", "json", "--format-error", "text"}, {"--format", "text", "--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { + root := readableErrorTestCommand(t, args...) + var out bytes.Buffer + if err := ShowCommandError(root, failure, &out); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "synthetic-private") || strings.Contains(out.String(), "\x1b") { + t.Fatal("final diagnostic exposed response data") + } + if strings.Contains(strings.ToLower(out.String()), "press r") { + t.Fatal("final diagnostic directed the user to a closed workspace") + } + switch { + case root.String("transform-error") != "": + if out.String() != baseline+"\n" { + t.Fatal("extracted error message changed") + } + case errorOutputFormat(root) == "json": + var value map[string]any + if err := json.Unmarshal(out.Bytes(), &value); err != nil || len(value) != 1 || value["message"] != baseline { + t.Fatal("machine error message changed") + } + default: + if operation == "preflight" { + if !strings.Contains(out.String(), baseline) || strings.Contains(out.String(), "may have succeeded") { + t.Fatal("final text lost preflight status, outcome, or recovery guidance") + } + break + } + if !strings.Contains(out.String(), fmt.Sprintf("HTTP %d", status)) || !strings.Contains(out.String(), "may have succeeded") { + t.Fatal("final text lost HTTP status or mutation uncertainty") + } + if (status == 400 || status == 422) && !strings.Contains(out.String(), "reviewed fields") { + t.Fatal("final text omitted validation recovery") + } + } + } + }) + } + } +} + +func TestLimitsWorkspaceExitRecoveryUsesProcessActions(t *testing.T) { + for _, tc := range []struct { + name, want string + cause error + }{ + {"authentication", "help setup admin", &openai.Error{StatusCode: 401, Message: "synthetic-private-response"}}, + {"access", "organization or project access", &openai.Error{StatusCode: 403, Message: "synthetic-private-response"}}, + {"missing", "scope, and resource ID", &openai.Error{StatusCode: 404, Message: "synthetic-private-response"}}, + {"rate", "Wait before reopening", &openai.Error{StatusCode: 429, Code: "unknown_synthetic_code", Message: "synthetic-private-response"}}, + {"organization cap", "help admin organization spend-limit retrieve", &openai.Error{StatusCode: 429, Code: "organization_spend_limit_exceeded"}}, + {"project cap", "help admin organization projects spend-limit retrieve", &openai.Error{StatusCode: 429, Code: "project_spend_limit_exceeded"}}, + {"billing", "https://platform.openai.com/settings/organization/limits", &openai.Error{StatusCode: 429, Code: "credit_balance_exhausted"}}, + {"canceled", "check was interrupted", context.Canceled}, + {"deadline", "check was interrupted", context.DeadlineExceeded}, + {"changed", "Record changed", errors.New("The configuration changed. Refresh before applying this change.")}, + {"incomplete rates", "Reopen with --max-items -1", errors.New("Rate-limit editing requires a complete list. Reopen with a larger --max-items value or -1.")}, + {"unknown", "Reopen the limits workspace", errors.New("synthetic-private-response\x1b[2J")}, + } { + t.Run(tc.name, func(t *testing.T) { + m, backend := newLimitsController(t, 40, 12) + m.invocation, m.snapshot.ProjectID = "custom-openai", "synthetic-private-project" + limitsControllerReview(t, m) + m.mutationError = &limitsMutationFailure{Err: tc.cause, Attempted: false} + m.Update(limitsWorkResult{mutation: true, err: m.mutationError}) + limitsControllerKey(m, tea.KeyF1) + if tc.name == "missing" || tc.name == "canceled" || tc.name == "deadline" || tc.name == "changed" || tc.name == "unknown" { + if !strings.Contains(m.details, "Press r to refresh") { + t.Fatal("in-workspace recovery lost its keyboard action") + } + } + limitsControllerKey(m, tea.KeyEscape) + if m.page != "form" { + t.Fatal("recovery help did not restore the rejected draft") + } + limitsControllerKey(m, tea.KeyEscape) + limitsControllerKey(m, 'q') + m.lastSaved = gjson.Parse(strings.Replace(limitsTestLimit, "organization.spend_limit", "project.spend_limit", 1)) + m.lastSavedOperation = limitsChangeOperation("limit_set", true) + var receipt bytes.Buffer + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "json", ExplicitFormat: true, Stdout: &receipt, Stderr: io.Discard}) + var safe *limitsWorkflowFailure + if !errors.As(failure, &safe) || !errors.Is(failure, tc.cause) || backend.writes.Load() != 0 { + t.Fatal("final recovery lost the cause, failure, or zero-write outcome") + } + if !gjson.Valid(receipt.String()) || gjson.Get(receipt.String(), "threshold_amount").Int() != 10000 || !strings.Contains(safe.Error(), "A preceding change was confirmed.") { + t.Fatal("final recovery discarded the preceding confirmed result") + } + for _, args := range [][]string{nil, {"--format-error", "json"}, {"--format-error", "text", "--transform-error", "message"}} { + root := readableErrorTestCommand(t, args...) + var out bytes.Buffer + if err := ShowCommandError(root, failure, &out); err != nil { + t.Fatal(err) + } + text := out.String() + if strings.Contains(text, "synthetic-private") || strings.Contains(text, "\x1b") || strings.Contains(strings.ToLower(text), "press r") || strings.Contains(text, "may have succeeded") { + t.Fatal("final recovery leaked private data, keyboard actions, or mutation uncertainty") + } + for _, want := range []string{tc.want, "custom-openai help admin organization projects spend-limit retrieve", "original connection, authentication, and scope options"} { + if !strings.Contains(text, want) { + t.Fatalf("final recovery omitted %q", want) + } + } + if root.String("transform-error") != "" { + if text != safe.Error()+"\n" { + t.Fatal("extracted message differs from the canonical final recovery") + } + } else if errorOutputFormat(root) == "json" { + var value map[string]any + if err := json.Unmarshal(out.Bytes(), &value); err != nil || len(value) != 1 || value["message"] != safe.Error() { + t.Fatal("machine error schema or message changed") + } + } + } + }) + } + for _, status := range []int{404, 429} { + t.Run(fmt.Sprintf("uncertain/%d", status), func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + cause := &openai.Error{StatusCode: status, Message: "synthetic-private-response"} + m.locked, m.mutationError = true, &limitsMutationFailure{Err: cause, Attempted: true} + if status == 404 && !strings.Contains(m.help(), "Press r to refresh") { + t.Fatal("in-workspace mutation recovery lost its keyboard guidance") + } + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Stdout: io.Discard, Stderr: io.Discard}) + if !errors.Is(failure, cause) { + t.Fatal("final mutation recovery lost its cause") + } + baseline := "A limits change may have succeeded. Check its current state before repeating it.\n" + m.readRecovery() + var human, machine bytes.Buffer + if err := ShowCommandError(readableErrorTestCommand(t), failure, &human); err != nil { + t.Fatal(err) + } + if strings.Contains(strings.ToLower(human.String()), "press r") || strings.Contains(human.String(), "synthetic-private") || !strings.Contains(human.String(), baseline) { + t.Fatal("final mutation recovery lost uncertainty or exposed an invalid action") + } + if status == 404 && !strings.Contains(human.String(), "Read the current record before another change.") { + t.Fatal("uncertain 404 did not require a new read") + } + if err := ShowCommandError(readableErrorTestCommand(t, "--format-error", "json"), failure, &machine); err != nil { + t.Fatal(err) + } + var value map[string]any + if err := json.Unmarshal(machine.Bytes(), &value); err != nil || len(value) != 1 || value["message"] != baseline { + t.Fatal("final mutation recovery changed the machine contract") + } + }) + } + t.Run("interrupted preflight and output failure", func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + cause := &openai.Error{StatusCode: 404, Message: "synthetic-private-response"} + m.refreshRequired, m.interrupted = true, true + m.mutationError = &limitsMutationFailure{Err: errors.Join(cause, context.Canceled), Attempted: false} + m.output = &limitsWorkspaceOutput{err: &clioutput.WriteError{Err: syscall.EPIPE}} + failure := finishLimitsWorkspace(t.Context(), m, tea.ErrInterrupted, ShowJSONOpts{Context: t.Context(), Stdout: io.Discard, Stderr: io.Discard}) + var interrupt interface{ ExitCode() int } + if !errors.Is(failure, cause) || !errors.Is(failure, context.Canceled) || !errors.Is(failure, syscall.EPIPE) || clioutput.IsBrokenPipe(failure) || !errors.As(failure, &interrupt) || interrupt.ExitCode() != 130 { + t.Fatal("final preflight recovery lost a joined cause or interruption status") + } + var out bytes.Buffer + if err := ShowCommandError(readableErrorTestCommand(t), failure, &out); err != nil { + t.Fatal(err) + } + if strings.Contains(out.String(), "synthetic-private") || strings.Contains(strings.ToLower(out.String()), "press r") || !strings.Contains(out.String(), "No write sent.") || !strings.Contains(out.String(), "Reopen the limits workspace") { + t.Fatal("interrupted final preflight recovery lost its process-safe action") + } + }) +} + +func TestLimitsWorkspaceCanceledMutationRecovery(t *testing.T) { + for _, cause := range []error{context.Canceled, context.DeadlineExceeded} { + m, _ := newLimitsController(t, 80, 24) + m.locked, m.mutationError = true, &limitsMutationFailure{Err: cause, Attempted: true} + failure := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) + var out bytes.Buffer + if err := ShowCommandError(readableErrorTestCommand(t), failure, &out); err != nil { + t.Fatal(err) + } + if !errors.Is(failure, cause) || !strings.Contains(out.String(), "interrupted") || strings.Contains(out.String(), "Check the API address and connection") || !strings.Contains(out.String(), "may have succeeded") { + t.Fatal("canceled mutation lost its cause or gave connection advice") + } + } +} + +func TestLimitsWorkspaceRecoveryAccessibleFromLockedForm(t *testing.T) { + m, _ := newLimitsController(t, 40, 12) + m.begin("alert_create", gjson.Result{}) + m.locked = true + m.change.Kind = "alert_create" + m.formError = "Save not confirmed. Press ? for recovery. Exit before another change." + if !strings.Contains(m.View().Content, "F1: help") { + t.Fatal("minimum-size form hid the recovery action") + } + limitsControllerKey(m, tea.KeyF1) + if m.page != "details" || !strings.Contains(m.details, "It may have succeeded") || !strings.Contains(m.details, "help admin organization spend-alerts list") { + t.Fatal("recovery help did not explain uncertainty and a safe read") + } + if !strings.Contains(m.View().Content, "Ctrl+C") { + t.Fatal("recovery help hid cancellation") + } +} + +type limitsFailingWriter struct{ err error } + +func (w limitsFailingWriter) Write([]byte) (int, error) { return 0, w.err } + +type limitsWorkspaceWriteFunc func([]byte) (int, error) + +func (write limitsWorkspaceWriteFunc) Write(data []byte) (int, error) { return write(data) } + +func TestLimitsWorkspaceOutputPreservesFailuresAndCleanup(t *testing.T) { + for _, method := range []string{"Write", "WriteString"} { + for _, failure := range []string{"sink error", "short write"} { + t.Run(method+"/"+failure, func(t *testing.T) { + ctx, stop := context.WithCancel(t.Context()) + defer stop() + first, later := errors.New("synthetic output failure"), errors.New("later output failure") + want := first + if failure == "short write" { + want = io.ErrShortWrite + } + calls, cleanup := 0, "" + sink := limitsWorkspaceWriteFunc(func(data []byte) (int, error) { + calls++ + switch calls { + case 1: + if failure == "short write" { + return 2, nil + } + return 2, first + case 2: + cleanup = string(data) + return len(data), nil + default: + return 0, later + } + }) + out := &limitsWorkspaceOutput{ + Writer: clioutput.Writer{Context: context.Background(), Sink: sink}, + stop: stop} + write := func(value string) (int, error) { return out.Write([]byte(value)) } + if method == "WriteString" { + write = func(value string) (int, error) { return io.WriteString(out, value) } + } + n, original := write("result") + var writeErr *clioutput.WriteError + if n != 2 || !errors.Is(original, want) || !errors.As(original, &writeErr) { + t.Fatalf("lost partial write or sink classification: n=%d err=%v", n, original) + } + if !errors.Is(ctx.Err(), context.Canceled) || out.Err() != original { + t.Fatal("output failure did not cancel the workspace and retain the original error") + } + const restore = "\x1b[?25h" + if n, err := io.WriteString(out, restore); n != len(restore) || err != nil || cleanup != restore { + t.Fatalf("workspace cancellation prevented terminal cleanup: n=%d err=%v", n, err) + } + if _, err := out.Write([]byte("later")); !errors.Is(err, later) || out.Err() != original { + t.Fatal("a later write replaced the first output failure") + } + }) + } + } +} + +func TestLimitsWorkspaceOutputPreservesCancellation(t *testing.T) { + for _, when := range []string{"before write", "during write"} { + t.Run(when, func(t *testing.T) { + outputContext, cancelOutput := context.WithCancel(t.Context()) + defer cancelOutput() + workspaceContext, stop := context.WithCancel(t.Context()) + defer stop() + calls := 0 + out := &limitsWorkspaceOutput{ + Writer: clioutput.Writer{Context: outputContext, Sink: limitsWorkspaceWriteFunc(func(data []byte) (int, error) { + calls++ + cancelOutput() + return len(data), nil + })}, + stop: stop} + wantBytes, wantCalls := len("result"), 1 + if when == "before write" { + cancelOutput() + wantBytes, wantCalls = 0, 0 + } + n, err := io.WriteString(out, "result") + var writeErr *clioutput.WriteError + if n != wantBytes || calls != wantCalls || !errors.Is(err, context.Canceled) || errors.As(err, &writeErr) { + t.Fatalf("cancellation changed byte counts or became a sink failure: n=%d calls=%d err=%v", n, calls, err) + } + if out.Err() != err || !errors.Is(workspaceContext.Err(), context.Canceled) { + t.Fatal("cancellation did not stop the workspace and preserve its cause") + } + }) + } +} + +func TestLimitsWorkspaceOutputPreservesTerminalFile(t *testing.T) { + for _, borrowed := range []bool{false, true} { + t.Run(fmt.Sprintf("borrowed=%t", borrowed), func(t *testing.T) { + file, err := os.CreateTemp(t.TempDir(), "terminal-output") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { file.Close() }) + if _, err := file.WriteString("input"); err != nil { + t.Fatal(err) + } + if _, err := file.Seek(0, io.SeekStart); err != nil { + t.Fatal(err) + } + var sink bytes.Buffer + out := &limitsWorkspaceOutput{ + Writer: clioutput.Writer{Context: t.Context(), Sink: &sink}, File: file, + stop: func() { t.Fatal("successful output canceled the workspace") }} + if borrowed { + writer := terminal.BorrowedWriter{Writer: &sink, File: file} + out.Writer.Sink, out.File = writer, writer + } + if out.Fd() != file.Fd() { + t.Fatal("output wrapper changed the descriptor used for terminal detection") + } + if _, err := io.WriteString(out, "output"); err != nil || sink.String() != "output" { + t.Fatal("output wrapper bypassed its guarded sink", err) + } + input := make([]byte, len("input")) + if _, err := io.ReadFull(out, input); err != nil || string(input) != "input" { + t.Fatal("output wrapper changed terminal reads", err) + } + if out.Err() != nil { + t.Fatal("successful writes recorded an error", out.Err()) + } + if err := out.Close(); err != nil { + t.Fatal(err) + } + _, err = file.Stat() + if borrowed && err != nil || !borrowed && !errors.Is(err, os.ErrClosed) { + t.Fatal("output wrapper changed terminal file ownership", err) + } + }) + } +} + +func TestLimitsWorkspaceFinalizationPreservesIndependentFailures(t *testing.T) { + for _, scenario := range []string{"read and terminal output", "mutation and terminal output", "cancellation and terminal output", "refresh and receipt output", "program and terminal output"} { + t.Run(scenario, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + ctx := t.Context() + original := errors.New("private synthetic operation failure") + writeErr := &clioutput.WriteError{Err: syscall.EPIPE} + opts := ShowJSONOpts{Context: ctx, Stdout: io.Discard, Stderr: io.Discard, Format: "text"} + var runErr error + switch scenario { + case "program and terminal output": + runErr = errors.Join(original, tea.ErrProgramKilled) + m.output = &limitsWorkspaceOutput{err: writeErr} + case "read and terminal output": + m.workerError = original + m.output = &limitsWorkspaceOutput{err: writeErr} + case "mutation and terminal output": + m.mutationPending = true + m.mutationError = original + m.output = &limitsWorkspaceOutput{err: writeErr} + case "cancellation and terminal output": + var cancel context.CancelFunc + ctx, cancel = context.WithCancel(ctx) + cancel() + original = context.Canceled + m.interrupted = true + m.output = &limitsWorkspaceOutput{err: writeErr} + case "refresh and receipt output": + m.snapshot.Errors = []limitsSourceFailure{{Resource: "spend_alerts", Err: original}} + m.lastSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month"}`) + m.lastSavedOperation = limitsChangeOperation("limit_set", false) + opts.Stdout = limitsFailingWriter{err: syscall.EPIPE} + } + err := finishLimitsWorkspace(ctx, m, runErr, opts) + if !errors.Is(err, original) || !errors.Is(err, syscall.EPIPE) || clioutput.IsBrokenPipe(err) { + t.Fatalf("output failure hid an independent outcome: %v", err) + } + var safe *limitsWorkflowFailure + if !errors.As(err, &safe) || strings.Contains(safe.Error(), "private synthetic") || strings.Contains(safe.Error(), "shown above") { + t.Fatal("finalization lost safe outcome guidance or asserted an undisplayed receipt") + } + if scenario == "cancellation and terminal output" { + var interrupt *limitsInterrupt + if !errors.As(err, &interrupt) || interrupt.ExitCode() != 130 { + t.Fatal("output failure changed the interrupt status") + } + } + }) + } +} + +func TestLimitsWorkspaceReceiptUsesMutationProjection(t *testing.T) { + for _, format := range []string{"text", "json"} { + t.Run(format, func(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + m.workerSaved = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month","enforcement":{"status":"inactive"}}`) + m.workerSavedOperation = limitsChangeOperation("limit_set", false) + var out bytes.Buffer + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: format, ExplicitFormat: true, Stdout: &out, Stderr: io.Discard}) + if err != nil { + t.Fatal(err) + } + if format == "text" { + if !strings.Contains(out.String(), "USD 100.01") || !strings.Contains(out.String(), "inactive") { + t.Fatalf("receipt lost threshold units or returned state: %q", out.String()) + } + } else if !gjson.Valid(out.String()) || gjson.Get(out.String(), "threshold_amount").Raw != "10001" || strings.Contains(out.String(), "spend_threshold") { + t.Fatalf("machine receipt changed threshold data: %q", out.String()) + } + }) + } +} + +func TestLimitsWorkspaceCostsAndEnforcementStayDistinct(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.snapshot.Limit = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month","enforcement":{"status":"inactive"}}`) + m.snapshot.LimitAbsent = false + m.snapshot.CostsTotal, m.snapshot.CostsComplete = "123.000000000000000001", false + m.snapshot.CostsStart, m.snapshot.CostsEnd = 1790812800, 1792368000 + view := m.View().Content + if !strings.Contains(view, "Reported Costs (incomplete): USD 123.000000000000000001") || !strings.Contains(view, "Returned enforcement: inactive") { + t.Fatalf("workspace conflated Costs and returned enforcement: %q", view) + } + help := m.help() + if !strings.Contains(help, "Queried period:") || !strings.Contains(help, "Today's Costs remain partial.") || !strings.Contains(help, "Costs can lag and do not determine enforcement state") { + t.Fatal("Costs view omitted its coverage and enforcement limitations") + } + m.snapshot.CostsComplete = true + view = m.View().Content + if !strings.Contains(view, "Reported Costs: USD 123.000000000000000001") || strings.Contains(view, "(complete)") || strings.Contains(view, "(incomplete)") { + t.Fatalf("complete pagination must not imply final Costs: %q", view) + } + m.Update(tea.WindowSizeMsg{Width: 40, Height: 12}) + for _, complete := range []bool{false, true} { + m.snapshot.CostsComplete = complete + view = ansi.Strip(m.View().Content) + if !complete && !strings.Contains(view, "Reported Costs (incomplete)") { + t.Fatalf("narrow overview hid incomplete Costs: %q", view) + } + for _, line := range strings.Split(view, "\n") { + if ansi.StringWidth(line) > 40 { + t.Fatalf("minimum-size overview exceeded its width: %q", line) + } + } + } +} + +func TestLimitsWorkspaceConfirmedRemovalPreservesReceiptAfter404(t *testing.T) { + for _, otherFailure := range []bool{false, true} { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + missing := &openai.Error{StatusCode: 404} + m.lastSaved = gjson.Parse(`{"object":"organization.spend_limit.deleted","deleted":true}`) + m.lastSavedOperation = limitsChangeOperation("limit_delete", false) + m.snapshot = limitsSnapshot{LimitUnconfirmed: true, AlertsComplete: true, CostsComplete: true, + Errors: []limitsSourceFailure{{Resource: "spend_limit", Err: missing}}} + m.workerError = missing + independent := errors.New("synthetic Costs failure") + if otherFailure { + m.snapshot.Errors = append(m.snapshot.Errors, limitsSourceFailure{Resource: "costs", Err: independent}) + m.workerError = errors.Join(missing, independent) + } + var out bytes.Buffer + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: &out, Stderr: io.Discard}) + if otherFailure { + if !errors.Is(err, independent) { + t.Fatal("confirmed removal hid an independent refresh failure") + } + } else if err != nil || !strings.Contains(out.String(), "Removal confirmed. Current hard-limit configuration was not returned (HTTP 404).") { + t.Fatalf("expected post-removal 404 overturned a confirmed removal: %v, %q", err, out.String()) + } + if !strings.Contains(out.String(), "deleted") || strings.Contains(out.String(), "No hard spend limit configured") { + t.Fatalf("receipt disappeared or 404 became confirmed absence: %q", out.String()) + } + } +} + +func TestLimitsWorkspaceShutdownKeepsCompletedSourceFailure(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.quitting = true + original := errors.New("synthetic source failure before shutdown") + snapshot := limitsSnapshot{Errors: []limitsSourceFailure{ + {Resource: "spend_limit", Err: original}, {Resource: "costs", Err: context.Canceled}, + }} + m.workerSnapshot = &snapshot + m.workerError = errors.Join(original, context.Canceled) + err := finishLimitsWorkspace(t.Context(), m, nil, ShowJSONOpts{Context: t.Context(), Format: "text", Stdout: io.Discard, Stderr: io.Discard}) + if !errors.Is(err, original) || errors.Is(err, context.Canceled) { + t.Fatalf("normal shutdown hid the read failure or became an interruption: %v", err) + } +} + +func TestLimitsWorkspaceAmbiguousEnforcementStaysUnknown(t *testing.T) { + m, _ := newLimitsController(t, 80, 24) + m.snapshot.LimitAbsent = false + m.snapshot.Limit = gjson.Parse(`{"object":"organization.spend_limit","threshold_amount":10001,"currency":"USD","interval":"month","enforcement":{"status":"inactive","status":"enforcing"}}`) + view := ansi.Strip(m.View().Content) + if !strings.Contains(view, "Returned enforcement: ambiguous; inspect original") || strings.Contains(view, "Returned enforcement: inactive") { + t.Fatalf("workspace selected one conflicting enforcement value: %q", view) + } +} diff --git a/pkg/custom/limits_workspace_view.go b/pkg/custom/limits_workspace_view.go new file mode 100644 index 00000000..af652861 --- /dev/null +++ b/pkg/custom/limits_workspace_view.go @@ -0,0 +1,160 @@ +package custom + +import ( + "strings" + + tea "charm.land/bubbletea/v2" + "charm.land/lipgloss/v2" + "github.com/charmbracelet/x/ansi" + "github.com/openai/openai-cli/internal/readable" + "github.com/tidwall/gjson" +) + +func (m *limitsWorkspace) View() tea.View { + view := tea.NewView("") + view.AltScreen = true + if m.quitting { + return view + } + if m.small() { + view.Content = "Resize to at least 40 x 12.\nCtrl+C: quit" + return view + } + width := m.width - 2 + accent := lipgloss.NewStyle().Bold(true) + if m.color { + color := "#3159BC" + if m.dark { + color = "#8AA8FF" + } + accent = accent.Foreground(lipgloss.Color(color)) + } + line := func(s string) string { return ansi.Truncate(limitsSafeLine(s), width, "…") } + mark := func(s string, selected bool) string { + if selected { + return accent.Render("> " + line(s)) + } + return " " + line(s) + } + header := accent.Render(line(m.heading() + " · " + m.scope())) + var body []string + footer := "Esc: back Ctrl+C: quit" + switch m.page { + case "home": + sections := []string{"Spend", "Alerts"} + if m.snapshot.ProjectID != "" { + sections = append(sections, "Rates") + } + for i := range sections { + if i == m.section { + sections[i] = accent.Render("[" + sections[i] + "]") + } + } + body = append(body, strings.Join(sections, " "), line(m.sourceSummary()), line(m.costsSummary())) + if m.section == 0 && m.snapshot.Limit.Exists() { + state := m.snapshot.Limit.Get("enforcement.status") + status := "not reported" + if limitsUniqueJSON(m.snapshot.Limit) != nil { + status = "ambiguous; inspect original" + } else if state.Type == gjson.String { + status = state.String() + } + body = append(body, line("Returned enforcement: "+status)) + } + rows := m.menu() + room := max(1, m.height-6-len(body)) + if m.locked { + room = max(1, room-1) + } + start := max(0, m.selection-room+1) + for i := start; i < min(len(rows), start+room); i++ { + body = append(body, mark(rows[i].label, i == m.selection)) + } + if len(rows) == 0 { + body = append(body, "No editable records loaded.") + } + body = append(body, line(m.note)) + if m.locked { + body = append(body, "Writes locked. Inspect before retrying.") + } + footer = "←/→: section ↑/↓: select Enter: edit\nr: refresh i: inspect d: delete alert\nq: quit ?: help Ctrl+C: interrupt" + case "form": + body = append(body, "Tab: next field · Shift+Tab: previous") + count := len(m.draft.inputs) + var notice []string + if m.formError != "" { + first := limitsSafeLine(strings.SplitN(m.formError, "\n", 2)[0]) + notice = strings.Split(ansi.Wrap(first, width, ""), "\n") + if len(notice) > 2 { + notice = []string{notice[0], ansi.Truncate(strings.Join(notice[1:], " "), width, "…")} + } + } + // Reserve navigation, Review, diagnostics, and the optional prefix action before fields. + fieldRows := m.height - 4 - 2 - len(notice) + if count == 3 { + fieldRows-- + } + visible := max(1, fieldRows/2) + start := max(0, min(m.focus, count-1)-visible+1) + end := min(count, start+visible) + for i := start; i < end; i++ { + m.draft.inputs[i].SetWidth(max(1, width-4)) + marker, label := " ", line(m.draft.labels[i]) + if i == m.focus { + marker = "> " + label = accent.Render(label) + } + body = append(body, label, marker+ansi.Truncate(m.draft.inputs[i].View(), width, "…")) + } + if count == 3 { + modes := []string{"Keep unless edited", "Replace (empty is allowed)", "Clear to null"} + body = append(body, mark("Prefix: "+modes[m.draft.prefixMode], m.focus == count)) + } + body = append(body, mark("Review change", m.focus >= count && !(count == 3 && m.focus == count))) + body = append(body, notice...) + footer = "Enter: next / select\nF1: help Esc: back Ctrl+C: quit" + if m.refreshRequired { + footer = "r: refresh Enter: next field\nF1: help Esc: back Ctrl+C: quit" + } + case "review", "details": + body = append(body, m.viewport.View()) + if m.page == "review" { + back, apply := "> Back", " Apply change" + if m.apply { + back, apply = " Back", "> Apply change" + } + if !m.viewport.AtBottom() { + apply = " Read to the end before applying" + } + body = append(body, line(back+" "+apply)) + footer = "↑/↓: scroll Tab: action Enter: select\nF1: help Esc: back Ctrl+C: quit" + if m.previewCommand != "" { + footer = "↑/↓: scroll Tab: action Enter: select\nCtrl+P: print command and exit\nF1: help Esc: back Ctrl+C: quit" + } + } else { + footer = "↑/↓: scroll Esc: back Ctrl+C: quit" + } + } + if m.busy { + status := "Loading…" + if m.page == "review" { + status = "Saving… Do not repeat this change." + } + footer = line(status) + "\nCtrl+C: interrupt" + } + // The action row and controls occupy reserved rows. Long data stays in the viewport. + footerRows := strings.Count(footer, "\n") + 1 + available := max(0, m.height-2-footerRows) + text := strings.Join(body, "\n") + rows := strings.Split(text, "\n") + if len(rows) > available { + rows = rows[:available] + } + view.Content = header + "\n" + strings.Join(rows, "\n") + strings.Repeat("\n", max(1, available-len(rows)+1)) + footer + return view +} + +// Keep external text on one row after escaping terminal controls. +func limitsSafeLine(value string) string { + return strings.NewReplacer("\n", " ↵ ", "\t", " → ").Replace(readable.Text(value)) +} diff --git a/pkg/custom/local_errors.go b/pkg/custom/local_errors.go index 90b99969..0c12653f 100644 --- a/pkg/custom/local_errors.go +++ b/pkg/custom/local_errors.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "io" "net" "net/url" "os" @@ -28,11 +29,16 @@ import ( // Local diagnostics can contain rejected values, file names, and credentialed // URLs. Classify them into local prose; terminal escaping is not redaction. func localErrorMessage(root *cli.Command, failure error) string { - command := root - var contextual *commandError - if errors.As(failure, &contextual) && contextual.command != nil { - command = contextual.command - } + return localErrorMessageForDisplay(root, failure, false) +} + +// Keep human recovery separate from existing machine fallback messages. +func readableLocalErrorMessage(root *cli.Command, failure error) string { + return localErrorMessageForDisplay(root, failure, true) +} + +func localErrorMessageForDisplay(root *cli.Command, failure error, human bool) string { + command := commandErrorContext(root, failure) var timeout net.Error var networkError *url.Error var connectionError *net.OpError @@ -57,8 +63,16 @@ func localErrorMessage(root *cli.Command, failure error) string { var navigationFailure *listview.PaginationError var modelsListFailure *models.ListError var batchFailure *batchcommands.WorkflowError + var limitsFailure *limitsWorkflowFailure var finiteListFailure *clioutput.FiniteJSONListError + var spendResponseFailure *spendMutationResponseError + responseRecovery := "" + if human && errors.As(failure, &spendResponseFailure) { + responseRecovery = readableSpendMutationFailure(root, command) + } switch { + case errors.As(failure, &limitsFailure): + return limitsFailure.Error() case errors.As(failure, &batchFailure): return batchFailure.Error() case errors.As(failure, &adminSetupFailure): @@ -76,6 +90,8 @@ func localErrorMessage(root *cli.Command, failure error) string { return utilityFailure.Error() case errors.Is(failure, context.DeadlineExceeded), errors.As(failure, &timeout) && timeout.Timeout(): return "The request timed out. The API may have received it; check its status before repeating it." + case responseRecovery != "": + return responseRecovery case errors.As(failure, &modelsListFailure): return modelsListFailure.Error() case errors.As(failure, &finiteListFailure): @@ -141,9 +157,18 @@ func localErrorMessage(root *cli.Command, failure error) string { } // Transport wrappers can contain a vetted local configuration error, such // as the mTLS proxy restriction. Keep that guidance before this fallback. + if human && (errors.As(failure, &networkError) || errors.As(failure, &connectionError) || errors.Is(failure, io.ErrUnexpectedEOF)) { + if message := readableSpendMutationFailure(root, command); message != "" { + return message + } + } if errors.As(failure, &networkError) || errors.As(failure, &connectionError) { return "Could not connect to the API. Check your connection, proxy, and --base-url setting." } + var writeFailure *clioutput.WriteError + if human && errors.As(failure, &writeFailure) { + return "Could not write command output. Check your terminal, output file, or receiving command." + } // The usage buffer is deliberately not a fallback: it repeats raw values. return "The command could not be completed. Check your arguments with --help." } diff --git a/pkg/custom/readable_errors.go b/pkg/custom/readable_errors.go index b74fe94f..b83da02c 100644 --- a/pkg/custom/readable_errors.go +++ b/pkg/custom/readable_errors.go @@ -12,6 +12,7 @@ import ( "github.com/openai/openai-cli/internal/clihelp" localcommands "github.com/openai/openai-cli/internal/commands" + "github.com/openai/openai-cli/internal/commands/admin" batchcommands "github.com/openai/openai-cli/internal/commands/batches" clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-cli/internal/readable" @@ -109,17 +110,28 @@ func showCommandError(ctx context.Context, root *cli.Command, failure error, out isAPI := errors.As(failure, &apierr) var streamerr *ssestream.StreamError isStream := !errors.Is(failure, context.Canceled) && errors.As(failure, &streamerr) + var limitsFailure *limitsWorkflowFailure + if errors.As(failure, &limitsFailure) { + // Aggregate source failures retain their causes, but expose only the + // workflow's safe outcome. Individual API payloads are not the result. + isAPI, isStream = false, false + } if format == "text" && root.String("transform-error") == "" { message := "" var batchFailure *batchcommands.WorkflowError - if errors.As(failure, &batchFailure) { + if limitsFailure != nil { + message = limitsFailure.Error() + if limitsFailure.humanDetail != "" { + message += "\n" + limitsFailure.humanDetail + } + } else if errors.As(failure, &batchFailure) { message = batchFailure.Error() } else if isAPI && !errors.Is(failure, context.Canceled) { message = readableAPIErrorMessage(root, failure, apierr) } else if isStream { message = readableStreamErrorText } else { - message = localErrorMessage(root, failure) + message = readableLocalErrorMessage(root, failure) } return readable.WriteText(out, message) } @@ -161,10 +173,17 @@ func readableAPIErrorMessage(root *cli.Command, failure error, apierr *openai.Er switch apierr.StatusCode { case http.StatusUnauthorized: message = "Check API key, organization and project.\nKey setup: " + invocation + " help setup" + if required, _ := commandErrorContext(root, failure).Metadata[admin.CredentialsRequiredMetadata].(bool); required { + message = "Check --admin-api-key or OPENAI_ADMIN_KEY and any Authorization override.\n" + + "Check the organization and API address.\nKey setup: " + invocation + " help setup admin" + } case http.StatusForbidden: message = "Check your key's permissions.\nCheck project access to this resource." case http.StatusNotFound: message = "Check the resource or model ID.\nCheck your key's access to it." + if guidance := readableSpendNotFoundMessage(root, failure); guidance != "" { + message = guidance + } case http.StatusBadRequest, http.StatusUnprocessableEntity: var contextual *commandError var command *cli.Command @@ -175,7 +194,11 @@ func readableAPIErrorMessage(root *cli.Command, failure error, apierr *openai.Er case http.StatusTooManyRequests: message = "Wait before trying again." switch apierr.Code { - case "insufficient_quota", "billing_hard_limit_reached", "credit_balance_exhausted", "organization_spend_limit_exceeded", "project_spend_limit_exceeded", "organization_usage_limit_exceeded": + case "organization_spend_limit_exceeded": + message = readableOrganizationSpendLimitMessage(root) + case "organization_usage_limit_exceeded": + message = "Organization usage limit reached.\nCheck your organization's billing and usage limits." + case "insufficient_quota", "billing_hard_limit_reached", "credit_balance_exhausted", "project_spend_limit_exceeded": message = "API usage or billing limit reached.\nCheck your project's billing and limits." } case http.StatusConflict: diff --git a/pkg/custom/readable_errors_test.go b/pkg/custom/readable_errors_test.go index 62635daa..adaaa2e0 100644 --- a/pkg/custom/readable_errors_test.go +++ b/pkg/custom/readable_errors_test.go @@ -10,9 +10,11 @@ import ( "net" "net/http" "net/url" + "os" "strings" "testing" + clioutput "github.com/openai/openai-cli/internal/output" "github.com/openai/openai-cli/internal/requestflag" "github.com/openai/openai-go/v3" "github.com/openai/openai-go/v3/packages/ssestream" @@ -190,6 +192,65 @@ func TestCommandErrorStructuredLocalFallbackDoesNotEchoDiagnostic(t *testing.T) } } +func TestCommandErrorOutputFailureHumanGuidance(t *testing.T) { + const humanMessage = "Could not write command output. Check your terminal, output file, or receiving command." + const machineMessage = "The command could not be completed. Check your arguments with --help." + cause := errors.New("synthetic-private-prompt https://synthetic.invalid/?token=synthetic-secret \x1b]52;c;synthetic-secret\a") + writer := clioutput.Writer{Context: t.Context(), Sink: errorSink{cause}} + _, failure := writer.Write([]byte("synthetic output")) + var writeFailure *clioutput.WriteError + require.ErrorAs(t, failure, &writeFailure) + for _, test := range []struct { + name string + flags []string + want string + }{ + {"human", nil, humanMessage}, + {"quiet", []string{"--quiet"}, humanMessage}, + {"JSON", []string{"--format-error", "json"}, machineMessage}, + {"extraction", []string{"--format-error", "text", "--transform-error", "message"}, machineMessage}, + } { + t.Run(test.name, func(t *testing.T) { + var out bytes.Buffer + require.NoError(t, ShowCommandError(readableErrorTestCommand(t, test.flags...), failure, &out)) + if test.name == "JSON" { + var value map[string]string + require.NoError(t, json.Unmarshal(out.Bytes(), &value)) + require.Equal(t, map[string]string{"message": test.want}, value) + } else { + require.Equal(t, test.want+"\n", out.String()) + } + assertReadableErrorContainsNoPrivateDetails(t, out.String()) + require.ErrorIs(t, failure, cause) + }) + } + var value any + syntaxFailure := json.Unmarshal([]byte(`{"synthetic-private-prompt":`), &value) + require.Error(t, syntaxFailure) + for _, test := range []struct { + name string + cause error + }{ + {"cancellation", context.Canceled}, + {"deadline", context.DeadlineExceeded}, + {"syntax", syntaxFailure}, + {"file", &os.PathError{Op: "write", Path: "synthetic-private-output", Err: os.ErrPermission}}, + {"API", &openai.Error{StatusCode: http.StatusForbidden}}, + } { + t.Run("precedence/"+test.name, func(t *testing.T) { + root := readableErrorTestCommand(t) + var baseline, out bytes.Buffer + require.NoError(t, ShowCommandError(root, test.cause, &baseline)) + joined := errors.Join(failure, test.cause) + require.NoError(t, ShowCommandError(root, joined, &out)) + require.Equal(t, baseline.String(), out.String()) + assertReadableErrorContainsNoPrivateDetails(t, out.String()) + require.ErrorIs(t, joined, cause) + require.ErrorIs(t, joined, test.cause) + }) + } +} + func TestCommandErrorInvalidFormatStillPresentsSafely(t *testing.T) { root := readableErrorTestCommand(t, "--format-error", "synthetic-secret") var out bytes.Buffer @@ -209,6 +270,7 @@ func readableErrorTestCommand(t *testing.T, args ...string) *cli.Command { &cli.StringFlag{Name: "transform-error"}, &cli.StringFlag{Name: "transform"}, &cli.BoolFlag{Name: "raw-output"}, + &cli.BoolFlag{Name: "quiet"}, }, Action: func(context.Context, *cli.Command) error { return nil }, } diff --git a/pkg/custom/spend_error_guidance.go b/pkg/custom/spend_error_guidance.go new file mode 100644 index 00000000..7a5d371b --- /dev/null +++ b/pkg/custom/spend_error_guidance.go @@ -0,0 +1,142 @@ +package custom + +import ( + "encoding/json" + "errors" + "io" + + "github.com/openai/openai-cli/internal/clihelp" + localcommands "github.com/openai/openai-cli/internal/commands" + "github.com/urfave/cli/v3" +) + +func commandErrorContext(root *cli.Command, failure error) *cli.Command { + var contextual *commandError + if errors.As(failure, &contextual) && contextual.command != nil { + return contextual.command + } + return root +} + +const spendRecoveryContext = "Keep the original API address, authentication, organization, and project options for any check." + +// Response conversion happens after the mutation returned successfully. Keep +// that provenance separate from JSON errors raised while preparing a request. +type spendMutationResponseError struct{ cause error } + +func (e *spendMutationResponseError) Error() string { return e.cause.Error() } +func (e *spendMutationResponseError) Unwrap() error { return e.cause } + +func spendMutationResponseFailure(err error, opts ShowJSONOpts) error { + if opts.OutputKind != OutputResponse { + return err + } + switch opts.Operation { + case "(resource) admin.organization.spend_limit > (method) update", + "(resource) admin.organization.spend_limit > (method) delete", + "(resource) admin.organization.projects.spend_limit > (method) update", + "(resource) admin.organization.projects.spend_limit > (method) delete", + "(resource) admin.organization.spend_alerts > (method) create", + "(resource) admin.organization.spend_alerts > (method) update", + "(resource) admin.organization.spend_alerts > (method) delete", + "(resource) admin.organization.projects.spend_alerts > (method) create", + "(resource) admin.organization.projects.spend_alerts > (method) update", + "(resource) admin.organization.projects.spend_alerts > (method) delete", + "(resource) admin.organization.projects.rate_limits > (method) update_rate_limit": + default: + return err + } + var syntaxError *json.SyntaxError + var typeError *json.UnmarshalTypeError + if errors.As(err, &syntaxError) || errors.As(err, &typeError) || errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return &spendMutationResponseError{cause: err} + } + return err +} + +// A status alone cannot establish configuration at an arbitrary API address. +// Recovery uses declared command metadata, never request URLs or response prose. +func readableSpendNotFoundMessage(root *cli.Command, failure error) string { + command := commandErrorContext(root, failure) + invocation := clihelp.DisplayInvocation(root) + message := "" + switch localcommands.ResourceName(command) { + case "admin:organization:spend-limit": + if command.Name == "retrieve" || command.Name == "delete" { + message = "The organization spend limit was not returned.\n" + + "Check the API address and organization. The limit may not be configured.\n" + + "Limit options: " + invocation + " help admin organization spend-limit update" + } + case "admin:organization:projects:spend-limit": + if command.Name == "retrieve" || command.Name == "delete" { + message = "The project or its spend limit could not be found.\n" + + "Check the project ID and access first.\n" + + "Project lookup: " + invocation + " help admin organization projects retrieve\n" + + "If the project exists, the limit may not be configured.\n" + + "Limit options: " + invocation + " help admin organization projects spend-limit update" + } + case "admin:organization:spend-alerts", "admin:organization:projects:spend-alerts": + if command.Name == "retrieve" || command.Name == "update" || command.Name == "delete" { + path := "admin organization spend-alerts list" + message = "The spend alert could not be found at this API address.\nCheck the alert ID against current organization alerts." + if localcommands.ResourceName(command) == "admin:organization:projects:spend-alerts" { + path = "admin organization projects spend-alerts list" + message = "The project spend alert could not be found at this API address.\nCheck the project ID, access, and current project alerts." + } + message += "\nList options: " + invocation + " help " + path + } + } + if message != "" { + message += "\n" + spendRecoveryContext + } + return message +} + +func readableOrganizationSpendLimitMessage(root *cli.Command) string { + invocation := clihelp.DisplayInvocation(root) + return "The API reports an organization spend limit was reached.\n" + + "Ask an organization owner to inspect that limit.\n" + + "An admin API key is required for the OpenAI limit lookup.\n" + + "Limit lookup: " + invocation + " help admin organization spend-limit retrieve\n" + + "Admin key setup: " + invocation + " help setup admin\n" + + spendRecoveryContext +} + +func readableSpendMutationFailure(root, command *cli.Command) string { + path := "" + change := "spend change" + switch localcommands.ResourceName(command) { + case "admin:organization:spend-limit": + if command.Name == "update" || command.Name == "delete" { + path = "admin organization spend-limit retrieve" + } + case "admin:organization:projects:spend-limit": + if command.Name == "update" || command.Name == "delete" { + path = "admin organization projects spend-limit retrieve" + } + case "admin:organization:spend-alerts", "admin:organization:projects:spend-alerts": + prefix := "admin organization spend-alerts " + if localcommands.ResourceName(command) == "admin:organization:projects:spend-alerts" { + prefix = "admin organization projects spend-alerts " + } + switch command.Name { + case "create": + path = prefix + "list" + case "update", "delete": + path = prefix + "retrieve" + } + case "admin:organization:projects:rate-limits": + if command.Name == "update-rate-limit" { + path = "admin organization projects rate-limits list-rate-limits" + change = "rate-limit change" + } + } + if path == "" { + return "" + } + return "The API response did not confirm the " + change + ".\n" + + "The API may have received the change.\n" + + "Check its current state before repeating the command.\n" + + "Read options: " + clihelp.DisplayInvocation(root) + " help " + path + "\n" + + spendRecoveryContext +} diff --git a/pkg/custom/spend_error_guidance_test.go b/pkg/custom/spend_error_guidance_test.go new file mode 100644 index 00000000..15f5fc98 --- /dev/null +++ b/pkg/custom/spend_error_guidance_test.go @@ -0,0 +1,243 @@ +package custom + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + "syscall" + "testing" + + "github.com/openai/openai-cli/internal/commands/admin" + clioutput "github.com/openai/openai-cli/internal/output" + "github.com/openai/openai-go/v3" + "github.com/stretchr/testify/require" + "github.com/tidwall/gjson" + "github.com/urfave/cli/v3" +) + +func spendGuidanceTestCommand(resource, operation string) *cli.Command { + return &cli.Command{Name: operation, Metadata: map[string]any{ + "openai-resource-command": resource, admin.CredentialsRequiredMetadata: true, + }} +} + +func TestSpendErrorGuidanceResponseConversionProvenance(t *testing.T) { + command := spendGuidanceTestCommand("admin:organization:spend-alerts", "create") + options := ShowJSONOpts{Context: t.Context(), Format: "yaml", ExplicitFormat: true, + Operation: "(resource) admin.organization.spend_alerts > (method) create", + OutputKind: OutputResponse, Stdout: io.Discard, Stderr: io.Discard} + failure := ShowJSON(gjson.Parse(`{"synthetic-private-response": broken}`), options) + var marker *spendMutationResponseError + var syntaxFailure *json.SyntaxError + require.ErrorAs(t, failure, &marker) + require.ErrorAs(t, failure, &syntaxFailure) + require.ErrorIs(t, failure, syntaxFailure) + require.Equal(t, syntaxFailure.Error(), failure.Error()) + root := readableErrorTestCommand(t) + contextual := withCommandError(command, failure) + require.Contains(t, readableLocalErrorMessage(root, contextual), "The API may have received the change.") + require.Equal(t, "Could not decode JSON: invalid JSON syntax.", localErrorMessage(root, contextual)) + for _, cause := range []error{context.Canceled, context.DeadlineExceeded} { + joined := withCommandError(command, errors.Join(failure, cause)) + require.ErrorIs(t, joined, syntaxFailure) + require.ErrorIs(t, joined, cause) + require.Equal(t, localErrorMessage(root, joined), readableLocalErrorMessage(root, joined)) + } + writeFailure := &clioutput.WriteError{Err: syscall.EPIPE} + joined := errors.Join(failure, writeFailure) + require.ErrorIs(t, joined, syntaxFailure) + require.ErrorIs(t, joined, syscall.EPIPE) + require.False(t, clioutput.IsBrokenPipe(joined), "a closed pipe must not discard a response failure") + inputFailure := withCommandError(command, syntaxFailure) + require.Equal(t, localErrorMessage(root, inputFailure), readableLocalErrorMessage(root, inputFailure)) + for _, kind := range []string{"read response", "error presentation"} { + t.Run(kind, func(t *testing.T) { + opts := options + if kind == "read response" { + opts.Operation = "(resource) admin.organization.spend_alerts > (method) retrieve" + } else { + opts.OutputKind = OutputUnspecified + } + err := ShowJSON(gjson.Parse(`{"synthetic-private-response": broken}`), opts) + require.ErrorAs(t, err, &syntaxFailure) + require.False(t, errors.As(err, &marker)) + }) + } + options.Stdout = failOutputWriter{err: syscall.EPIPE} + failure = ShowJSON(gjson.Parse(`{"id":"alert_synthetic"}`), options) + require.ErrorIs(t, failure, syscall.EPIPE) + require.True(t, clioutput.IsBrokenPipe(failure)) + require.False(t, errors.As(failure, &marker), "sink failures must retain their output classification") + ctx, cancel := context.WithCancel(t.Context()) + cancel() + options.Context = ctx + failure = ShowJSON(gjson.Parse(`{"synthetic-private-response": broken}`), options) + require.ErrorIs(t, failure, context.Canceled) + require.False(t, errors.As(failure, &marker), "pre-conversion cancellation must retain its classification") +} + +func TestSpendErrorGuidanceAdminMetadata(t *testing.T) { + root := readableErrorTestCommand(t) + root.Metadata["help-invocation"] = "'/tmp/synthetic app/openai'" + for _, required := range []bool{false, true} { + command := &cli.Command{Name: "retrieve", Metadata: map[string]any{admin.CredentialsRequiredMetadata: required}} + failure := &openai.Error{StatusCode: http.StatusUnauthorized} + message := readableAPIErrorMessage(root, withCommandError(command, failure), failure) + want := "Key setup: '/tmp/synthetic app/openai' help setup" + if required { + want += " admin" + require.Contains(t, message, "OPENAI_ADMIN_KEY") + require.Contains(t, message, "Authorization override") + } + require.Contains(t, strings.Split(message, "\n"), want) + } +} + +func TestSpendErrorGuidanceNotFoundPreservesUncertainty(t *testing.T) { + root := readableErrorTestCommand(t) + for _, test := range []struct{ resource, operation, want string }{ + {"admin:organization:spend-limit", "retrieve", "may not be configured"}, + {"admin:organization:projects:spend-limit", "delete", "Check the project ID and access first"}, + {"admin:organization:spend-alerts", "update", "help admin organization spend-alerts list"}, + {"admin:organization:projects:spend-alerts", "retrieve", "help admin organization projects spend-alerts list"}, + } { + command := spendGuidanceTestCommand(test.resource, test.operation) + failure := &openai.Error{StatusCode: http.StatusNotFound, Message: "synthetic-private-secret\x1b[2J"} + message := readableAPIErrorMessage(root, withCommandError(command, failure), failure) + require.Contains(t, message, test.want) + require.Contains(t, message, spendRecoveryContext) + require.NotContains(t, message, "No hard spend limit configured") + require.NotContains(t, message, "synthetic-private") + require.NotContains(t, message, "\x1b") + } + for _, command := range []*cli.Command{ + spendGuidanceTestCommand("models", "retrieve"), + spendGuidanceTestCommand("admin:organization:spend-limit", "update"), + spendGuidanceTestCommand("admin:organization:spend-alerts", "create"), + } { + require.Empty(t, readableSpendNotFoundMessage(root, withCommandError(command, errors.New("synthetic")))) + } +} + +func TestSpendErrorGuidanceTransportPrecedenceAndMachineFallback(t *testing.T) { + command := spendGuidanceTestCommand("admin:organization:spend-limit", "update") + network := &url.Error{Op: "Post", URL: "https://synthetic-private.invalid/?token=synthetic-private", Err: io.EOF} + for _, test := range []struct { + name string + err error + want string + changed bool + }{ + {"lost response", network, "The API may have received the change", true}, + {"truncated response", fmt.Errorf("synthetic-private: %w", io.ErrUnexpectedEOF), "The API may have received the change", true}, + {"cancel", errors.Join(network, context.Canceled), "Request canceled.", false}, + {"timeout", errors.Join(network, context.DeadlineExceeded), "The request timed out.", false}, + {"configuration", &url.Error{Op: "Post", URL: "synthetic-private", Err: errors.New("mTLS does not support HTTPS proxies")}, "mTLS does not support HTTPS proxies", false}, + } { + t.Run(test.name, func(t *testing.T) { + failure := withCommandError(command, test.err) + root := readableErrorTestCommand(t) + baseline := localErrorMessage(root, failure) + var human bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &human)) + require.Contains(t, human.String(), test.want) + require.NotContains(t, human.String(), "synthetic-private") + if test.changed { + require.Contains(t, human.String(), "help admin organization spend-limit retrieve") + require.NotContains(t, human.String(), "Could not connect") + } else { + require.Equal(t, baseline+"\n", human.String()) + } + root = readableErrorTestCommand(t, "--format-error", "json") + var machine bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &machine)) + var value map[string]any + require.NoError(t, json.Unmarshal(machine.Bytes(), &value)) + require.Equal(t, map[string]any{"message": baseline}, value) + root = readableErrorTestCommand(t, "--format-error", "text", "--transform-error", "message") + machine.Reset() + require.NoError(t, ShowCommandError(root, failure, &machine)) + require.Equal(t, baseline+"\n", machine.String()) + }) + } + root := readableErrorTestCommand(t) + for _, command := range []*cli.Command{ + spendGuidanceTestCommand("models", "delete"), + spendGuidanceTestCommand("admin:organization:spend-limit", "retrieve"), + } { + failure := withCommandError(command, network) + require.Equal(t, localErrorMessage(root, failure), readableLocalErrorMessage(root, failure)) + } +} + +func TestSpendErrorGuidanceMutationReadRoutes(t *testing.T) { + root := readableErrorTestCommand(t) + for _, test := range []struct{ resource, operation, read string }{ + {"admin:organization:projects:spend-limit", "delete", "admin organization projects spend-limit retrieve"}, + {"admin:organization:spend-alerts", "create", "admin organization spend-alerts list"}, + {"admin:organization:projects:spend-alerts", "create", "admin organization projects spend-alerts list"}, + {"admin:organization:projects:spend-alerts", "update", "admin organization projects spend-alerts retrieve"}, + {"admin:organization:projects:rate-limits", "update-rate-limit", "admin organization projects rate-limits list-rate-limits"}, + } { + require.Contains(t, readableSpendMutationFailure(root, spendGuidanceTestCommand(test.resource, test.operation)), "help "+test.read) + } + network := &url.Error{Op: "Post", URL: "https://synthetic-private.invalid", Err: io.EOF} + for _, operation := range []string{"update-rate-limit", "list-rate-limits"} { + failure := withCommandError(spendGuidanceTestCommand("admin:organization:projects:rate-limits", operation), network) + var human, machine bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &human)) + baseline := localErrorMessage(root, failure) + if operation == "update-rate-limit" { + require.Contains(t, human.String(), "The API response did not confirm the rate-limit change.") + require.Contains(t, human.String(), "help admin organization projects rate-limits list-rate-limits") + require.Contains(t, human.String(), spendRecoveryContext) + } else { + require.Equal(t, baseline+"\n", human.String()) + } + require.NotContains(t, human.String(), "synthetic-private") + require.NoError(t, ShowCommandError(readableErrorTestCommand(t, "--format-error", "json"), failure, &machine)) + var value map[string]any + require.NoError(t, json.Unmarshal(machine.Bytes(), &value)) + require.Equal(t, map[string]any{"message": baseline}, value) + } +} + +func TestSpendErrorGuidanceLimitCodesStayScoped(t *testing.T) { + root := readableErrorTestCommand(t) + for _, test := range []struct{ code, want, absent string }{ + {"organization_spend_limit_exceeded", "admin API key", "project's billing"}, + {"organization_usage_limit_exceeded", "organization's billing", "help admin organization spend-limit"}, + {"project_spend_limit_exceeded", "project's billing", "organization spend limit"}, + {"rate_limit_exceeded", "Wait before trying again", "admin API key"}, + {"synthetic-private-code", "Wait before trying again", "spend limit"}, + } { + failure := &openai.Error{StatusCode: http.StatusTooManyRequests, Code: test.code} + message := readableAPIErrorMessage(root, failure, failure) + require.Contains(t, message, test.want) + require.NotContains(t, message, test.absent) + require.NotContains(t, message, "synthetic-private") + } +} + +func TestSpendErrorGuidanceAggregateKeepsSafeMessageAndCause(t *testing.T) { + apierr := &openai.Error{StatusCode: http.StatusForbidden, Message: "synthetic-private"} + failure := &limitsWorkflowFailure{message: "The overview is incomplete. Retry the failed source.", cause: errors.Join(apierr, io.ErrClosedPipe)} + for _, args := range [][]string{nil, {"--format-error", "json"}, {"--transform-error", "message"}} { + root := readableErrorTestCommand(t, args...) + var out bytes.Buffer + require.NoError(t, ShowCommandError(root, failure, &out)) + require.Contains(t, out.String(), failure.message) + require.NotContains(t, out.String(), "synthetic-private") + require.NotContains(t, out.String(), "HTTP 403") + require.ErrorIs(t, failure, io.ErrClosedPipe) + var preserved *openai.Error + require.ErrorAs(t, failure, &preserved) + require.Same(t, apierr, preserved) + } +} diff --git a/scripts/demos/limits-workspace/README.md b/scripts/demos/limits-workspace/README.md new file mode 100644 index 00000000..97ce1d29 --- /dev/null +++ b/scripts/demos/limits-workspace/README.md @@ -0,0 +1,99 @@ +# Limits workspace terminal replay + +This recipe drives an immutable CLI binary against a synthetic loopback API. +It makes no live API calls or billing changes. +The recorder uses the shared `capture_and_render.sh` lifecycle and existing PTY harness. +No Go build occurs during recording. + +Obtain the source-build and PTY reservations through the current coordinator. +Read `/Users/vguvvala/code/cli-work/DESIGN.md`, `STATUS.md`, and `PRE_PUSH_REVIEW.md` before presentation changes or publication. +Preserve generated ownership and obtain independent adversarial review before every feature push. + +Use existing Python, asciinema, agg, ffmpeg, ffprobe, and Menlo installations. +Supply a verified binary and its full source commit. +Keep each output directory empty and outside the repository. +Set `DEMO_SOURCE_STATE` when recording an uncommitted candidate based on that source commit. +Binary hashes and source manifests establish recording identity; they do not establish build provenance alone. + +```sh +# Run only during the granted PTY reservation. +DEMO_WIDTH=80 DEMO_NO_COLOR=1 \ + bash scripts/demos/limits-workspace/record.sh --run-authorized-pty-slot \ + /absolute/path/to/immutable/openai SOURCE_SHA /absolute/path/to/evidence/limits-80 + +DEMO_WIDTH=40 DEMO_NO_COLOR=1 \ + bash scripts/demos/limits-workspace/record.sh --run-authorized-pty-slot \ + /absolute/path/to/immutable/openai SOURCE_SHA /absolute/path/to/evidence/limits-40 +``` + +Width 80 uses a 24-row workspace. Width 40 uses a 12-row workspace. +The outer recording reserves four additional rows for the evidence label and shell prompt. +`DEMO_NO_COLOR=0` permits a separate color run; monochrome remains the default. +The isolated child environment includes only fake credentials and the fixture endpoint. + +| Scene | Scope | Confirmed fixture action | +| --- | --- | --- | +| `org-limit` | Organization | Replace the hard limit with USD 125.25 per month | +| `org-limit-delete` | Organization | Remove the hard limit and retain its confirmed receipt after a missing-record refresh | +| `project-limit` | Project | Replace the hard limit with USD 125.25 per month | +| `alert-create` | Organization | Correct an invalid amount, retain the draft through help and Back, then create a USD 75.50 alert | +| `alert-print` | Organization | Print the complete reviewed alert command after terminal restoration; perform no mutation | +| `alert-edit` | Project | Change the alert to USD 150.25 and clear its prefix to null | +| `alert-delete` | Organization | Delete the selected synthetic alert | +| `rate-update` | Project | Change requests per minute to 250; preserve all other dimensions | +| `uncertain` | Organization | Apply a synthetic write, lose its response, and verify that repetition stays blocked | +| `slow-cancel` | Organization | Cancel a held read with Ctrl+C; perform no mutation | +| `preflight-400` | Organization | Reject the preflight read, retain the draft, explain recovery, and exit with failure without mutation | + +Each mutation scene opens a form or deletion review. +The driver reads to the review's end before selecting Apply. +It checks that the fixture receives no mutation before confirmation. +The create-alert scene checks visible validation errors and complete help text. +Repeated help preserves the draft, prefix mode, and selected Review action. +Escape from review returns to the draft. Help from review clears Apply selection. +The final request verifies the retained amount, recipient, and prefix. +The uncertain scene checks one mutation, retained failure guidance, and exit status 1. +The cancellation scene waits for the fixture's request marker and requires exit status 130. +Successful mutation scenes require exit status 0 and one mutation each. +The print scene uses Ctrl+P and requires exit status 0 with no mutation. +It compares the complete printed command bytes after terminal restoration, including the final newline. +Wrapped inspection rows do not establish command copyability. +The preflight scene loads the overview before returning HTTP 400 from the confirmation read. +It retains the draft, blocks review before refresh, and checks F1 recovery at both widths. +Escape from help returns to the retained draft before another Escape opens the overview. +Normal exit preserves the failure and hides private response data. No mutation occurs. +Final recovery tells users to reopen the workspace and preserve the original request scope. +The recorder rejects workspace keyboard actions in failure diagnostics after terminal restoration. + +The fixture stores only synthetic request bodies, response hashes, and method/path/status records. +It never logs request headers or credentials. +Shutdown releases held work and joins the server worker. +The scene driver restores terminal output processing and closes every owned child PTY. + +The recorder preserves raw child bytes, child casts, outer casts, request logs, hashes, and actual process statuses. +The validator checks byte identity across all three capture forms. +Checkpoint renderers receive no stdin. The recorder logs their exact names, timestamps, and output filenames. +A second validation pass compares that inventory with source checkpoints and all PNG and frame-GIF filenames. +It rejects missing, duplicate, altered, or unexpected checkpoint artifacts. +It checks exact mutation bodies and guards against automatic retry after an uncertain write. +Terminal captures merge stdout and stderr. They do not prove separate stream routing. + +Planned screenshot evidence comes from recorded checkpoints: + +- Overview: scope, reported current-month Costs, spend units, returned enforcement, and keyboard controls. +- Form: entered values, retained optional-field semantics, and visible Review action. +- Review top: selected resource and original state. +- Review Apply: explicit final action after scrolling through the complete review. +- Review Print: the Ctrl+P action and the inspection-only command label. +- Printed command: the complete command after terminal restoration, with no mutation. +- Saved state: refreshed resource and confirmed outcome. +- Uncertain state: retained draft and blocked retry guidance. +- Preflight recovery: retained draft, required refresh, HTTP status, and field/scope guidance. +- Loading state: cancellation remains available while a read is pending. +- Final scene frame: restored shell, actual result, and no stale confirmation question. + +Inspect every full-terminal PNG and relevant GIF frame at both sizes. +Check clipping, stale rows, focus markers, controls, and final state after exit. +Generated artifacts are terminal replays, not native terminal appearance evidence. +Do not commit binary media or claim these planned scenes ran before execution. +The earlier spend-threshold recorder remains the separate before/after comparison. diff --git a/scripts/demos/limits-workspace/record.sh b/scripts/demos/limits-workspace/record.sh new file mode 100755 index 00000000..5cd383c3 --- /dev/null +++ b/scripts/demos/limits-workspace/record.sh @@ -0,0 +1,85 @@ +#!/bin/bash +set -euo pipefail + +if [ "$#" -ne 4 ] || [ "$1" != --run-authorized-pty-slot ]; then + echo 'usage: record.sh --run-authorized-pty-slot CLI_BINARY SOURCE_SHA OUTPUT_DIR' >&2 + exit 2 +fi +shift +record_source="$(cd "$(dirname "$0")" && pwd -P)" +record_root="$(cd "$record_source/../../.." && pwd -P)" +record_width="${DEMO_WIDTH:-80}" +record_no_color="${DEMO_NO_COLOR:-1}" +case "$record_width" in 80) record_height=24;; 40) record_height=12;; *) echo 'DEMO_WIDTH must be 80 or 40.' >&2; exit 2;; esac +case "$record_no_color" in 0|1) ;; *) echo 'DEMO_NO_COLOR must be 0 or 1.' >&2; exit 2;; esac +source "$record_source/../capture_and_render.sh" +# This workflow has one candidate binary. Passing it twice reuses the shared lifecycle. +# The shared cleanup also owns synthetic executable paths shown in command previews. +TMPDIR=/private/tmp demo_prepare_capture "$record_root" "$1" "$1" "$2" "$2" "$3" "$record_source/server.py" +demo_python="$(command -v python3)" +record_digest="$(shasum -a 256 "$demo_after" | cut -d ' ' -f 1)" +demo_start_api "$demo_output/requests.jsonl" "$demo_output/fixtures.json" "$demo_runtime/gates" +cat > "$demo_runtime/scene.sh" <<'SCENE' +#!/bin/bash +set -euo pipefail +test -t 0 && test -t 1 && test -t 2 || exit 99 +printf '\033[2J\033[H' +printf '%s\n' 'Terminal replay. Synthetic data.' "$DEMO_SCENE_LABEL" +exec "$DEMO_PYTHON" -I -B "$DEMO_DRIVER" +SCENE +cp "$demo_runtime/scene.sh" "$demo_output/scene.sh" +{ + echo 'feature: supported limits workspace workflows' + echo "candidate source commit: $demo_after_sha" + echo "candidate source state: ${DEMO_SOURCE_STATE:-committed source supplied by caller}" + echo 'baseline: no before comparison; this records the supplied workspace binary only' + echo 'scope: synthetic loopback API; Bash PTY replay; no native graphical terminal or live API claim' + echo "child terminal: ${record_width}x${record_height}; outer terminal: ${record_width}x$((record_height+4))" + echo "NO_COLOR: $record_no_color; GOMAXPROCS=2; isolated HOME; CI unset" + echo 'render: Menlo 18px, asciinema theme, shared capture_and_render lifecycle' + demo_capture_metadata + "$demo_python" --version +} > "$demo_output/metadata.txt" +shasum -a 256 "$record_source/record.sh" "$record_source/server.py" "$record_source/scene.py" \ + "$record_source/validate.py" "$record_source/README.md" \ + "$record_root/scripts/demos/capture_and_render.sh" "$record_root/scripts/image_picker_harness.py" \ + "$record_root/pkg/custom/limits_commands.go" "$record_root/pkg/custom/limits_service.go" \ + "$record_root/pkg/custom/limits_values.go" "$record_root/pkg/custom/limits_draft.go" \ + "$record_root/pkg/custom/limits_workspace.go" "$record_root/pkg/custom/limits_workspace_view.go" \ + "$record_root/internal/output/transform/project_limits_overview.go" > "$demo_output/source-sha256.txt" +demo_window_size="${record_width}x$((record_height+4))" +demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ + --theme asciinema --fps-cap 15 --last-frame-duration 2) +record_scenes=(org-limit org-limit-delete project-limit alert-create alert-edit alert-delete rate-update uncertain slow-cancel alert-print preflight-400) +for record_scene in "${record_scenes[@]}"; do + demo_capture_scene "$record_scene" 0 "$demo_runtime/after" "$demo_api_url/$record_scene/v1" \ + "$record_scene: ${record_width}x${record_height}" \ + "DEMO_PYTHON=$demo_python" "DEMO_ASCIINEMA=$demo_asciinema" "DEMO_DRIVER=$record_source/scene.py" \ + "DEMO_SCENE_NAME=$record_scene" "DEMO_EVIDENCE_BASE=$demo_output/$record_scene-evidence" \ + "DEMO_WIDTH=$record_width" "DEMO_HEIGHT=$record_height" "DEMO_NO_COLOR=$record_no_color" \ + "DEMO_RUNTIME_DIRECTORY=$demo_runtime" \ + "DEMO_GATE_DIRECTORY=$demo_runtime/gates" "DEMO_REQUEST_LOG=$demo_output/requests.jsonl" +done +demo_stop_api +test "$record_digest" = "$(shasum -a 256 "$demo_after" | cut -d ' ' -f 1)" +shasum -a 256 -c "$demo_output/source-sha256.txt" > "$demo_output/source-validation.txt" +"$demo_python" -I -B "$record_source/validate.py" "$demo_output" > "$demo_output/validation.txt" +demo_assemble_capture 150 "${record_scenes[@]}" +# Select checkpoint frames from the exact child captures. No UI text is reconstructed. +: > "$demo_output/checkpoint-frames.txt" +for record_scene in "${record_scenes[@]}"; do + while IFS=$'\t' read -r record_name record_seconds; do + "$demo_agg" --quiet "${demo_render_options[@]}" --select "$record_seconds" \ + "$demo_output/$record_scene-evidence.child.cast" "$demo_output/$record_scene-$record_name-frame.gif" < /dev/null + "$demo_ffmpeg" -nostdin -hide_banner -loglevel error -y -i "$demo_output/$record_scene-$record_name-frame.gif" \ + -frames:v 1 "$demo_output/$record_scene-$record_name.png" < /dev/null + printf '%s\t%s\t%s\t%s\t%s\n' "$record_scene" "$record_name" "$record_seconds" \ + "$record_scene-$record_name-frame.gif" "$record_scene-$record_name.png" >> "$demo_output/checkpoint-frames.txt" + done < <("$demo_python" -I -B -c 'import json,sys; data=json.load(open(sys.argv[1])); [print(item["name"]+"\t"+str(item["seconds"])) for item in data["checkpoints"]]' "$demo_output/$record_scene-evidence.json") +done +"$demo_python" -I -B "$record_source/validate.py" "$demo_output" --rendered-checkpoints >> "$demo_output/validation.txt" +( + cd "$demo_output" + shasum -a 256 -- *.cast *.gif *.png *.txt *.json *.jsonl *.sh > SHA256SUMS +) +printf 'Recorded limits workspace terminal replay in %s\n' "$demo_output" diff --git a/scripts/demos/limits-workspace/scene.py b/scripts/demos/limits-workspace/scene.py new file mode 100755 index 00000000..0c6af77e --- /dev/null +++ b/scripts/demos/limits-workspace/scene.py @@ -0,0 +1,466 @@ +#!/usr/bin/env python3 +"""Drive the real workspace with deterministic synthetic input and retain raw bytes.""" + +import hashlib +import importlib.util +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +import termios +import time + + +spec = importlib.util.spec_from_file_location("limits_demo_terminal", Path(__file__).resolve().parents[2] / "image_picker_harness.py") +harness = importlib.util.module_from_spec(spec) +spec.loader.exec_module(harness) + +# Trapped signals retain their default disposition in the foreground CLI child. +BASH_WORKSPACE = r'''trap ':' HUP INT TERM +"$@" +limits_scene_status=$? +printf '%s\n' "$limits_scene_status" > "$DEMO_CLI_EXIT" || exit 74 +"$DEMO_TERMIOS_PYTHON" -I -B -c ' +import json, os, sys, termios +attributes = termios.tcgetattr(0) +attributes[6] = [value.hex() if isinstance(value, bytes) else value for value in attributes[6]] +with open(sys.argv[1], "x", encoding="utf-8") as destination: + json.dump({"parent_pid": os.getppid(), "attributes": attributes}, destination) +' "$DEMO_TERMIOS_SNAPSHOT" || exit 74 +exit "$limits_scene_status" +''' + + +def write_text(path, value): + with Path(path).open("x", encoding="utf-8") as destination: + if destination.write(value) != len(value): + raise OSError("incomplete scene evidence write") + + +class RelayTerminal(harness.Terminal): + def __init__(self, *args, terminal_snapshot, **kwargs): + self.terminal_snapshot = terminal_snapshot + super().__init__(*args, **kwargs) + + def read(self, duration=0.05): + offset = len(self.raw) + super().read(duration) + chunk = self.raw[offset:] + if sys.stdout.buffer.write(chunk) != len(chunk): + raise OSError("short terminal relay write") + sys.stdout.buffer.flush() + + def send(self, value): + if os.write(self.master, value) != len(value): + raise OSError("short workspace key write") + + def finish(self, expected): + deadline = time.monotonic() + 8 + while self.child.poll() is None: + self.read() + if time.monotonic() >= deadline: + raise AssertionError("workspace did not exit") + drain(self, 0.15) + if self.child.returncode != expected: + raise AssertionError(f"workspace exit={self.child.returncode}, expected={expected}; tail={self.text()[-900:]}") + # Bash snapshots the modes before exiting its controlling PTY session. + snapshot = json.loads(self.terminal_snapshot.read_text()) + if snapshot["parent_pid"] != self.child.pid: + raise AssertionError("terminal modes were not captured under the live Bash parent") + restored, initial = snapshot["attributes"].copy(), self.initial.copy() + initial[6] = [value.hex() if isinstance(value, bytes) else value for value in initial[6]] + self.terminal_modes = {"initial": initial.copy(), "restored": restored.copy(), + "snapshot_parent_pid": snapshot["parent_pid"], "bash_pid": self.child.pid} + if sys.platform == "darwin": + restored[3] &= ~termios.PENDIN + initial[3] &= ~termios.PENDIN + if restored != initial: + raise AssertionError("workspace did not restore terminal input") + if self.raw.rfind(b"\x1b[?1049l") <= self.raw.rfind(b"\x1b[?1049h"): + raise AssertionError("workspace did not leave its alternate screen") + if self.raw.rfind(b"\x1b[?25h") <= self.raw.rfind(b"\x1b[?25l"): + raise AssertionError("workspace did not restore cursor visibility") + + +def drain(terminal, seconds): + deadline = time.monotonic() + seconds + while time.monotonic() < deadline: + terminal.read(min(0.03, max(0, deadline - time.monotonic()))) + + +def screen_text(terminal): + with tempfile.TemporaryDirectory(prefix="limits-demo-screen-") as temporary: + recording, transcript = Path(temporary) / "screen.cast", Path(temporary) / "screen.txt" + header = {"version": 2, "width": terminal.initial_size[0], "height": terminal.initial_size[1]} + write_text(recording, "\n".join(json.dumps(item) for item in [header, *terminal.events]) + "\n") + subprocess.run([os.environ["DEMO_ASCIINEMA"], "convert", "-f", "txt", str(recording), str(transcript)], + capture_output=True, check=True, timeout=10, + env={"PATH": "/usr/bin:/bin", "HOME": temporary, + "ASCIINEMA_STATE_HOME": temporary + "/state", "ASCIINEMA_CONFIG_HOME": temporary + "/config"}) + return transcript.read_text() + + +def wait_screen(terminal, required, forbidden=(), timeout=10): + deadline, screen = time.monotonic() + timeout, "" + while time.monotonic() < deadline: + drain(terminal, 0.15) + screen = screen_text(terminal) + if all(text in screen for text in required) and not any(text in screen for text in forbidden): + return screen + if terminal.child.poll() is not None: + break + raise AssertionError(f"workspace state missing {required!r}; screen={screen!r}") + + +def requests(scene): + entries = [json.loads(line) for line in Path(os.environ["DEMO_REQUEST_LOG"]).read_text().splitlines() if line] + return [entry for entry in entries if entry["scene"] == scene] + + +def mutations(scene): + return [item for item in requests(scene) if item["method"] in {"POST", "DELETE"}] + + +def checkpoint(terminal, evidence, output, name, required=()): + screen = screen_text(terminal) + if not all(text in screen for text in required): + raise AssertionError(f"checkpoint {name} omitted its required preview: {required!r}") + write_text(Path(str(output) + f".{name}.txt"), screen) + evidence["checkpoints"].append({"name": name, "seconds": time.monotonic() - terminal.started}) + drain(terminal, 0.6) + + +def key(terminal, evidence, label, value): + if terminal.child.poll() is not None: + raise AssertionError("workspace exited before its scripted action") + evidence["keys"].append({"key": label, "seconds": time.monotonic() - terminal.started}) + terminal.send(value) + drain(terminal, 0.08) + + +def replace(terminal, evidence, label, value): + key(terminal, evidence, label, b"\x01\x0b\x1b[200~" + value.encode() + b"\x1b[201~") + + +def review(terminal, evidence, scene, output): + wait_screen(terminal, ["> Back", "Enter: select"]) + checkpoint(terminal, evidence, output, "review-top") + preview_seen = False + for _ in range(40): + screen = screen_text(terminal) + if "Command printing needs" in screen or "Command preview omitted" in screen: + raise AssertionError("review omitted the real command preview") + if not preview_seen and "Command preview (inspection only)" in screen: + checkpoint(terminal, evidence, output, "review-command", + ("Command preview (inspection only)",)) + preview_seen = True + if "Apply change" in screen and "Read to the end" not in screen: + break + if not preview_seen and "Command preview" in screen: + key(terminal, evidence, "Show command preview", b"\x1b[B") + else: + key(terminal, evidence, "PageDown", b"\x1b[6~") + else: + raise AssertionError("review never exposed its final action") + if not preview_seen: + raise AssertionError("review reached Apply without a command preview checkpoint") + if mutations(scene): + raise AssertionError("fixture received a mutation before explicit confirmation") + if scene == "alert-print": + wait_screen(terminal, ["Ctrl+P"]) + checkpoint(terminal, evidence, output, "review-print", ("Ctrl+P",)) + key(terminal, evidence, "Ctrl+P print and exit", b"\x10") + return + key(terminal, evidence, "Tab to Apply", b"\t") + wait_screen(terminal, ["> Apply change"]) + if scene == "alert-create": + checkpoint(terminal, evidence, output, "review-before-help", ("> Apply change",)) + key(terminal, evidence, "Review help", b"\x1bOP") + wait_screen(terminal, ["Spend manages hard monthly"]) + checkpoint(terminal, evidence, output, "review-help") + key(terminal, evidence, "Return to review", b"\x1b") + wait_screen(terminal, ["> Back", "Apply change"], ["> Apply change"]) + checkpoint(terminal, evidence, output, "review-return", ("> Back",)) + if mutations(scene): + raise AssertionError("returning from help applied the draft") + key(terminal, evidence, "Select Apply again", b"\t") + wait_screen(terminal, ["> Apply change"]) + checkpoint(terminal, evidence, output, "review-apply") + key(terminal, evidence, "Apply", b"\r") + + +def run_scene(terminal, scene, evidence, output): + if scene == "slow-cancel": + gate = Path(os.environ["DEMO_GATE_DIRECTORY"]) / "slow-cancel.requested" + deadline = time.monotonic() + 8 + while not gate.exists(): + terminal.read() + if time.monotonic() >= deadline: + raise AssertionError("slow request did not reach fixture") + wait_screen(terminal, ["Loading", "Ctrl+C"]) + checkpoint(terminal, evidence, output, "loading") + key(terminal, evidence, "Ctrl+C", b"\x03") + terminal.finish(130) + write_text(Path(os.environ["DEMO_GATE_DIRECTORY"]) / "slow-cancel.release", "child canceled and exited\n") + if mutations(scene): + raise AssertionError("canceling a load caused a mutation") + return + wait_screen(terminal, ["Set hard spend limit", "Ctrl+C"], ["Loading"]) + checkpoint(terminal, evidence, output, "overview") + if scene.startswith("alert-"): + key(terminal, evidence, "Alerts", b"\x1b[C") + wait_screen(terminal, ["Create spend alert"]) + creating = scene in {"alert-create", "alert-print"} + if not creating: + key(terminal, evidence, "Select alert", b"\x1b[B") + if scene == "alert-delete": + key(terminal, evidence, "Delete selected alert", b"d") + else: + key(terminal, evidence, "Edit alert", b"\r") + wait_screen(terminal, ["Monthly USD", "Review change"]) + amount = "75.505" if scene == "alert-create" else "75.50" if creating else "150.25" + replace(terminal, evidence, "Monthly USD", amount) + key(terminal, evidence, "Recipients", b"\t") + if creating: + replace(terminal, evidence, "Synthetic recipient", "finance@example.test") + key(terminal, evidence, "Subject prefix", b"\t") + if creating: + prefix = "Demo team's budget $HOME; keep 'quoted' text" if scene == "alert-print" else "Demo" + replace(terminal, evidence, "Subject prefix", prefix) + key(terminal, evidence, "Prefix mode", b"\t") + if scene == "alert-edit": + key(terminal, evidence, "Replace prefix mode", b"\r") + key(terminal, evidence, "Clear prefix to null", b"\r") + key(terminal, evidence, "Review action", b"\t") + checkpoint(terminal, evidence, output, "form") + key(terminal, evidence, "Review", b"\r") + if scene == "alert-create": + wait_screen(terminal, ["Enter a USD amount", "Review change"]) + checkpoint(terminal, evidence, output, "validation", ("Enter a USD amount",)) + if mutations(scene): + raise AssertionError("an invalid amount sent a mutation") + key(terminal, evidence, "Validation help", b"\x1bOP") + wait_screen(terminal, ["Draft details", "two decimal places."]) + checkpoint(terminal, evidence, output, "validation-help", ("two decimal places.",)) + key(terminal, evidence, "Repeated help", b"\x1bOP") + key(terminal, evidence, "Return to draft", b"\x1b") + wait_screen(terminal, ["> Review change", "Demo", "Enter a USD amount"]) + checkpoint(terminal, evidence, output, "draft-return", ("> Review change", "Demo")) + for _ in range(4): + key(terminal, evidence, "Previous alert field", b"\x1b[Z") + wait_screen(terminal, ["Monthly USD", "75.505"]) + replace(terminal, evidence, "Correct Monthly USD", "75.50") + for _ in range(4): + key(terminal, evidence, "Next alert field", b"\t") + key(terminal, evidence, "Review corrected draft", b"\r") + wait_screen(terminal, ["> Back", "Enter: select"]) + key(terminal, evidence, "Escape from review", b"\x1b") + wait_screen(terminal, ["> Review change", "Demo"]) + for _ in range(4): + key(terminal, evidence, "Inspect retained alert amount", b"\x1b[Z") + wait_screen(terminal, ["Monthly USD", "75.50", "Review change"]) + checkpoint(terminal, evidence, output, "review-back", ("Monthly USD", "75.50")) + for _ in range(4): + key(terminal, evidence, "Return to review action", b"\t") + key(terminal, evidence, "Review retained draft", b"\r") + elif scene == "org-limit-delete": + key(terminal, evidence, "Select removal", b"\x1b[B") + wait_screen(terminal, ["Remove hard spend limit"]) + key(terminal, evidence, "Review removal", b"\r") + elif scene == "rate-update": + key(terminal, evidence, "Alerts", b"\x1b[C") + key(terminal, evidence, "Rates", b"\x1b[C") + wait_screen(terminal, ["rl_demo"]) + key(terminal, evidence, "Edit rate", b"\r") + wait_screen(terminal, ["Requests / minute", "Review change"]) + replace(terminal, evidence, "Requests per minute", "250") + for _ in range(6): + key(terminal, evidence, "Next rate field", b"\t") + checkpoint(terminal, evidence, output, "form") + key(terminal, evidence, "Review", b"\r") + else: + key(terminal, evidence, "Edit hard limit", b"\r") + wait_screen(terminal, ["Monthly USD", "Review change"]) + replace(terminal, evidence, "Monthly USD", "125.25") + key(terminal, evidence, "Review action", b"\t") + checkpoint(terminal, evidence, output, "form") + key(terminal, evidence, "Review", b"\r") + review(terminal, evidence, scene, output) + if scene == "alert-print": + terminal.finish(0) + scene_requests = requests(scene) + if mutations(scene) or not scene_requests or any(item["method"] != "GET" for item in scene_requests): + raise AssertionError("printing a reviewed command must perform no mutation") + channel = json.dumps({"recipients": ["finance@example.test"], + "subject_prefix": "Demo team's budget $HOME; keep 'quoted' text", + "type": "email"}, sort_keys=True, separators=(",", ":")) + command = (evidence["staged_binary"]["path"] + " '--base-url=" + evidence["request_base_url"] + + "' admin organization spend-alerts create" + " --currency USD --interval month --threshold-amount 7550 --notification-channel '" + + channel.replace("'", "'\\''") + "'") + output_flags = terminal.terminal_modes["initial"][1] + newline = "\r\n" if output_flags & termios.OPOST and output_flags & termios.ONLCR else "\n" + expected = (command + newline).encode() + restored = terminal.raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + start = len(terminal.raw) - len(expected) + if (start < restored or terminal.raw[start:] != expected + or terminal.raw[restored:].count(expected) != 1 + or terminal.raw.rfind(b"\x1b[?25h") > start): + raise AssertionError("printed command bytes differ or precede terminal restoration") + with Path(str(output) + ".printed-command.raw.txt").open("xb") as destination: + if destination.write(terminal.raw[start:]) != len(expected): + raise OSError("incomplete printed command evidence write") + evidence["printed_command"] = {"byte_offset": start, "byte_length": len(expected), + "sha256": hashlib.sha256(expected).hexdigest()} + checkpoint(terminal, evidence, output, "printed-command") + elif scene == "preflight-400": + wait_screen(terminal, ["HTTP 400", "125.25", "Review change"], ["Saving"]) + checkpoint(terminal, evidence, output, "preflight", ("HTTP 400", "125.25")) + key(terminal, evidence, "Blocked review before refresh", b"\r\r") + wait_screen(terminal, ["Refresh with r", "125.25", "Review change"]) + checkpoint(terminal, evidence, output, "refresh-required", ("Refresh with r", "125.25")) + key(terminal, evidence, "F1 recovery", b"\x1bOP") + help_screen = wait_screen(terminal, ["HTTP 400", "Recheck"]) + compact_help = "".join(help_screen.split()) + for message in ("No write sent.", "request fields and scope", "press r to refresh"): + if "".join(message.split()) not in compact_help: + raise AssertionError("preflight help omitted status or recovery guidance") + checkpoint(terminal, evidence, output, "preflight-help", ("HTTP 400",)) + key(terminal, evidence, "Return to retained draft", b"\x1b") + wait_screen(terminal, ["125.25", "Review change"]) + checkpoint(terminal, evidence, output, "preflight-draft", ("125.25", "Review change")) + key(terminal, evidence, "Back to overview", b"\x1b") + wait_screen(terminal, ["Set hard spend limit"]) + key(terminal, evidence, "Quit", b"q") + terminal.finish(1) + if mutations(scene): + raise AssertionError("failed preflight sent a mutation") + if b"synthetic-private-preflight" in terminal.raw: + raise AssertionError("preflight recovery exposed private response data") + restored = terminal.raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + diagnostic = terminal.raw[restored:] + for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", + b"Reopen the limits workspace", b"review current settings", + b"help admin organization spend-limit retrieve", + b"original connection, authentication, and scope options"): + if message not in diagnostic: + raise AssertionError("normal exit lost the preflight diagnostic") + if b"may have succeeded" in diagnostic: + raise AssertionError("preflight exit incorrectly claimed an uncertain mutation") + elif scene == "uncertain": + wait_screen(terminal, ["Save not confirmed", "Review change"]) + checkpoint(terminal, evidence, output, "uncertain") + key(terminal, evidence, "Blocked repeat Enter", b"\r\r") + if len(mutations(scene)) != 1: + raise AssertionError("uncertain write repeated automatically") + key(terminal, evidence, "Back", b"\x1b") + wait_screen(terminal, ["Writes locked"]) + key(terminal, evidence, "Quit", b"q") + terminal.finish(1) + else: + wait_screen(terminal, ["Removal confirmed" if scene == "org-limit-delete" else "Change confirmed."], ["Loading", "Saving"]) + checkpoint(terminal, evidence, output, "saved") + key(terminal, evidence, "Quit", b"q") + terminal.finish(0) + if len(mutations(scene)) != 1: + raise AssertionError("scene did not produce exactly one confirmed mutation") + + +def main(): + scene, output = os.environ["DEMO_SCENE_NAME"], Path(os.environ["DEMO_EVIDENCE_BASE"]) + width, height = int(os.environ["DEMO_WIDTH"]), int(os.environ["DEMO_HEIGHT"]) + if (width, height) not in {(80, 24), (40, 12)} or not all(os.isatty(fd) for fd in (0, 1, 2)): + raise RuntimeError("scene requires a supported terminal size") + binary = shutil.which("openai") + if binary is None: + raise RuntimeError("immutable CLI binary is unavailable") + binary = Path(binary).resolve(strict=True) + runtime = Path(os.environ["DEMO_RUNTIME_DIRECTORY"]).resolve(strict=True) + if runtime.parent != Path("/private/tmp") or not runtime.name.startswith("cli-demo."): + raise AssertionError("the recorder runtime must use an owned synthetic /private/tmp directory") + project = scene in {"project-limit", "alert-edit", "rate-update"} + args = ["projects", "limits", "--project-id", "proj_demo"] if project else ["admin", "limits"] + if scene == "alert-print": + args.insert(0, "--base-url=" + os.environ["OPENAI_BASE_URL"]) + evidence = {"scene": scene, "width": width, "height": height, "command": ["openai", *args], + "keys": [], "checkpoints": [], "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "parent_shell": "/bin/bash", "recorder_runtime": str(runtime), + "request_base_url": os.environ["OPENAI_BASE_URL"]} + print("$ openai " + " ".join(args), flush=True) + original = termios.tcgetattr(1) + relay = termios.tcgetattr(1) + relay[1] &= ~termios.OPOST + try: + termios.tcsetattr(1, termios.TCSANOW, relay) + with tempfile.TemporaryDirectory(prefix="openai-limits-demo-", dir=runtime) as temporary: + staged = Path(temporary) / "openai" + shutil.copy2(binary, staged) + staged_hash = hashlib.sha256(staged.read_bytes()).hexdigest() + if staged_hash != evidence["binary_sha256"]: + raise AssertionError("staged CLI differs from the supplied binary") + evidence["staged_binary"] = {"path": str(staged), "sha256_before": staged_hash} + home = Path(temporary) / "home" + home.mkdir(mode=0o700) + exit_receipt = Path(temporary) / "cli-exit-status" + terminal_snapshot = Path(temporary) / "terminal-modes.json" + env = {"PATH": temporary + ":/usr/bin:/bin", "HOME": str(home), "TERM": "xterm-256color", "LANG": "en_US.UTF-8", + "GOMAXPROCS": "2", "PAGER": "cat", "OPENAI_API_KEY": "synthetic-demo-key", + "OPENAI_ADMIN_KEY": "synthetic-demo-admin-key", "OPENAI_BASE_URL": os.environ["OPENAI_BASE_URL"], + "DEMO_CLI_EXIT": str(exit_receipt), "DEMO_TERMIOS_PYTHON": sys.executable, + "DEMO_TERMIOS_SNAPSHOT": str(terminal_snapshot)} + if os.environ.get("DEMO_NO_COLOR", "1") == "1": + env.update(NO_COLOR="1", FORCE_COLOR="0") + terminal = None + try: + # Post-command status handling keeps Bash as the CLI's real parent. + # Both processes inherit the existing PTY session and process group. + shell_args = ["--noprofile", "--norc", "-c", BASH_WORKSPACE, "bash", str(staged), *args] + terminal = RelayTerminal("/bin/bash", shell_args, env, width=width, height=height, + controlling_terminal=True, terminal_snapshot=terminal_snapshot) + run_scene(terminal, scene, evidence, output) + status_text = exit_receipt.read_text() + if status_text != str(terminal.child.returncode) + "\n": + raise AssertionError("Bash did not preserve the actual CLI exit status") + evidence["cli_exit_status"] = int(status_text) + evidence["terminal_modes"] = terminal.terminal_modes + evidence["staged_binary"]["sha256_after"] = hashlib.sha256(staged.read_bytes()).hexdigest() + if (evidence["staged_binary"]["sha256_after"] != evidence["binary_sha256"] + or hashlib.sha256(binary.read_bytes()).hexdigest() != evidence["binary_sha256"]): + raise AssertionError("supplied or staged CLI changed during capture") + if str(binary).encode() in terminal.raw: + raise AssertionError("capture exposed the private binary source path") + if terminal.child.returncode != 0: + restored = terminal.raw.rfind(b"\x1b[?1049l") + if restored < 0: + raise AssertionError("final diagnostic lacks terminal restoration") + diagnostic = b" ".join(terminal.raw[restored + len(b"\x1b[?1049l"):].lower().split()) + for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", + b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): + if action in diagnostic: + raise AssertionError("final diagnostic names an action from the closed workspace") + evidence.update(exit_status=terminal.child.returncode, relayed_bytes=len(terminal.raw), + relayed_sha256=hashlib.sha256(terminal.raw).hexdigest()) + with Path(str(output) + ".tty").open("xb") as destination: + if destination.write(terminal.raw) != len(terminal.raw): + raise OSError("incomplete raw terminal write") + header = {"version": 2, "width": width, "height": height, "title": "Limits workspace; synthetic terminal replay"} + write_text(Path(str(output) + ".child.cast"), "\n".join(json.dumps(item) for item in [header, *terminal.events]) + "\n") + write_text(Path(str(output) + ".exit.txt"), screen_text(terminal)) + finally: + if terminal is not None: + terminal.close() + evidence["staged_binary"]["removed"] = not Path(temporary).exists() + if not evidence["staged_binary"]["removed"]: + raise AssertionError("owned staged CLI directory remains after cleanup") + finally: + termios.tcsetattr(1, termios.TCSANOW, original) + write_text(Path(str(output) + ".json"), json.dumps(evidence, indent=2) + "\n") + print("\n$ ", end="", flush=True) + time.sleep(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/limits-workspace/server.py b/scripts/demos/limits-workspace/server.py new file mode 100755 index 00000000..0d08c552 --- /dev/null +++ b/scripts/demos/limits-workspace/server.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +"""Stateful synthetic limits API. Bind only to loopback; never log credentials.""" + +import copy +import hashlib +import http.server +import json +from pathlib import Path +import signal +import socket +import sys +import threading +import time +from urllib.parse import urlsplit, parse_qs + + +SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") + + +def write_text(path, value): + with Path(path).open("x", encoding="utf-8") as destination: + if destination.write(value) != len(value): + raise OSError("incomplete fixture evidence write") + + +def initial_state(project): + scope = "project" if project else "organization" + return { + "limit": {"object": scope + ".spend_limit", "threshold_amount": 10000, + "currency": "USD", "interval": "month", "enforcement": {"status": "enforcing"}}, + "alerts": [{"object": scope + ".spend_alert", "id": "alert_demo", "threshold_amount": 10000, + "currency": "USD", "interval": "month", "notification_channel": { + "type": "email", "recipients": ["finance@example.test"], "subject_prefix": "Demo"}}], + "rates": [{"object": "project.rate_limit", "id": "rl_demo", "model": "synthetic-model", + "max_requests_per_1_minute": 100, "max_tokens_per_1_minute": 20000, + "max_images_per_1_minute": 5, "max_requests_per_1_day": 1000, + "max_audio_megabytes_per_1_minute": 10, "batch_1_day_max_input_tokens": 50000}], + } + + +def page(records): + return {"object": "list", "data": records, "has_more": False, + "first_id": records[0]["id"] if records else None, + "last_id": records[-1]["id"] if records else None} + + +def main(): + if len(sys.argv) != 5: + raise SystemExit("usage: server.py ADDRESS_FILE REQUEST_LOG FIXTURE_METADATA GATE_DIRECTORY") + gate = Path(sys.argv[4]) + gate.mkdir() + states = {name: initial_state(name in {"project-limit", "alert-edit", "rate-update"}) for name in SCENES} + limit_reads = {name: 0 for name in SCENES} + write_text(sys.argv[3], json.dumps({"data": "synthetic only", "initial": states}, indent=2) + "\n") + stop, lock, failures = threading.Event(), threading.Lock(), [] + with open(sys.argv[2], "x", encoding="utf-8") as log: + def record(entry): + line = json.dumps(entry, separators=(",", ":")) + "\n" + with lock: + if log.write(line) != len(line): + raise OSError("incomplete request log write") + log.flush() + + class Fixture(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def handle_request(self): + if self.headers.get("Authorization") != "Bearer synthetic-demo-admin-key": + raise AssertionError("unexpected fixture authentication") + parts = urlsplit(self.path).path.strip("/").split("/") + if len(parts) < 4 or parts[0] not in states or parts[1] != "v1" or parts[2] != "organization": + raise AssertionError("unexpected fixture route") + scene, project, tail = parts[0], False, parts[3:] + if tail[:1] == ["projects"]: + if tail[1:2] != ["proj_demo"]: + raise AssertionError("unexpected synthetic project") + project, tail = True, tail[2:] + state, scope = states[scene], "project" if project else "organization" + payload = None + if self.command == "POST": + payload = json.loads(self.rfile.read(int(self.headers["Content-Length"]))) + status, result = 200, None + if self.command == "GET" and tail == ["costs"]: + query = parse_qs(urlsplit(self.path).query) + expected_project = ["proj_demo"] if scene in {"project-limit", "alert-edit", "rate-update"} else [] + if query.get("project_ids[]", []) != expected_project: + raise AssertionError("Costs scope differs from the selected workspace") + start = int(query["start_time"][0]) + result = {"object": "page", "data": [{"object": "bucket", "start_time": start, + "end_time": start + 86400, "results": [{"object": "organization.costs.result", + "amount": {"value": 37.25, "currency": "usd"}, + "project_id": "proj_demo" if scene in {"project-limit", "alert-edit", "rate-update"} else None, + "line_item": None}]}], "has_more": False, "next_page": None} + elif self.command == "GET" and not tail and project: + result = {"id": "proj_demo", "object": "organization.project", "name": "Synthetic project", "status": "active"} + elif tail == ["spend_limit"]: + if self.command == "GET": + if scene == "slow-cancel": + record({"scene": scene, "method": "GET", "resource": "spend_limit", "status": "held"}) + write_text(gate / "slow-cancel.requested", "synthetic request received\n") + deadline = time.monotonic() + 30 + while not (gate / "slow-cancel.release").exists(): + if stop.wait(0.05): + return + if time.monotonic() >= deadline: + raise TimeoutError("slow scene did not confirm cancellation") + return + limit_reads[scene] += 1 + result = state["limit"] + if scene == "preflight-400" and limit_reads[scene] > 1: + status, result = 400, {"error": { + "message": "synthetic-private-preflight-message\u001b[2J", + "param": "synthetic-private-preflight-field\nline", + "code": "synthetic-private-preflight-code\u001b]0;title\u0007"}} + if result is None: + status, result = 404, {"error": {"message": "Synthetic limit is absent"}} + elif self.command == "POST": + if set(payload) != {"threshold_amount", "currency", "interval"} or payload["currency"] != "USD" or payload["interval"] != "month": + raise AssertionError("unexpected limit mutation body") + state["limit"] = dict(payload, object=scope + ".spend_limit", enforcement={"status": "enforcing"}) + result = state["limit"] + elif self.command == "DELETE": + state["limit"] = None + result = {"object": scope + ".spend_limit.deleted", "deleted": True} + elif tail[:1] == ["spend_alerts"]: + alert_id = tail[1] if len(tail) == 2 else None + alert = next((value for value in state["alerts"] if value["id"] == alert_id), None) + if self.command == "GET": + result = page(state["alerts"]) if alert_id is None else alert + elif self.command == "POST": + if set(payload) != {"threshold_amount", "currency", "interval", "notification_channel"} or payload["currency"] != "USD" or payload["interval"] != "month": + raise AssertionError("unexpected alert mutation body") + result = dict(payload, object=scope + ".spend_alert", id=alert_id or "alert_created") + if alert_id: + if alert is None: + raise AssertionError("edit targeted an unknown synthetic alert") + state["alerts"][state["alerts"].index(alert)] = result + else: + state["alerts"].append(result) + elif self.command == "DELETE" and alert is not None: + state["alerts"].remove(alert) + result = {"object": scope + ".spend_alert.deleted", "id": alert_id, "deleted": True} + elif tail[:1] == ["rate_limits"] and project: + if self.command == "GET": + result = page(state["rates"]) + elif self.command == "POST" and tail[1:] == ["rl_demo"]: + if payload != {"max_requests_per_1_minute": 250}: + raise AssertionError("rate update must change only the selected dimension") + state["rates"][0].update(payload) + result = state["rates"][0] + if result is None: + raise AssertionError("unsupported synthetic request") + body = json.dumps(copy.deepcopy(result), separators=(",", ":")).encode() + entry = {"scene": scene, "method": self.command, "path": urlsplit(self.path).path, + "status": status, "response_sha256": hashlib.sha256(body).hexdigest()} + if payload is not None: + entry["body"] = payload + if scene == "uncertain" and self.command == "POST": + entry["status"] = "response_lost_after_write" + record(entry) + self.connection.shutdown(socket.SHUT_RDWR) + self.connection.close() + self.close_connection = True + return + self.send_response(status) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.send_header("x-should-retry", "false") + self.end_headers() + if self.wfile.write(body) != len(body): + raise OSError("incomplete fixture response") + self.wfile.flush() + record(entry) + + do_GET = handle_request + do_POST = handle_request + do_DELETE = handle_request + + class Server(http.server.ThreadingHTTPServer): + daemon_threads = False + + def handle_error(self, *_): + with lock: + failures.append("fixture handler failed") + + server = Server(("127.0.0.1", 0), Fixture) + signal.signal(signal.SIGTERM, lambda *_: stop.set()) + signal.signal(signal.SIGINT, lambda *_: stop.set()) + worker = threading.Thread(target=server.serve_forever, kwargs={"poll_interval": 0.05}) + worker.start() + try: + write_text(sys.argv[1], f"http://127.0.0.1:{server.server_port}\n") + stop.wait() + finally: + stop.set() + server.shutdown() + server.server_close() + worker.join() + if failures: + raise RuntimeError("synthetic fixture failed; evidence is incomplete") + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/limits-workspace/validate.py b/scripts/demos/limits-workspace/validate.py new file mode 100755 index 00000000..8dfe0c13 --- /dev/null +++ b/scripts/demos/limits-workspace/validate.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +"""Check recorded bytes, actual child statuses, and synthetic mutation bodies.""" + +import hashlib +import json +import math +from pathlib import Path +import sys +import termios +from urllib.parse import urlsplit + + +SCENES = ("org-limit", "org-limit-delete", "project-limit", "alert-create", "alert-edit", + "alert-delete", "rate-update", "uncertain", "slow-cancel", "alert-print", "preflight-400") + + +def validate_rendered_checkpoints(root): + expected_rows = [] + expected_pngs = {scene + ".png" for scene in SCENES} + expected_frames = {scene + "-frame.gif" for scene in SCENES} + for scene in SCENES: + evidence = json.loads((root / (scene + "-evidence.json")).read_text()) + assert evidence["scene"] == scene + names = set() + for point in evidence["checkpoints"]: + name, seconds = point["name"], point["seconds"] + assert name and all(character in "abcdefghijklmnopqrstuvwxyz0123456789-" for character in name) + assert name not in names, (scene, "duplicate checkpoint name", name) + names.add(name) + assert type(seconds) in {int, float} and math.isfinite(seconds) and seconds >= 0 + frame, png = scene + "-" + name + "-frame.gif", scene + "-" + name + ".png" + expected_rows.append("\t".join((scene, name, str(seconds), frame, png))) + expected_frames.add(frame) + expected_pngs.add(png) + assert names, (scene, "missing checkpoints") + actual_rows = (root / "checkpoint-frames.txt").read_text().splitlines() + assert actual_rows == expected_rows, "rendered checkpoint names, times, or filenames differ from source evidence" + actual_pngs = {item.name for item in root.glob("*.png")} + actual_frames = {item.name for item in root.glob("*-frame.gif")} + assert actual_pngs == expected_pngs, ("PNG inventory mismatch", sorted(actual_pngs ^ expected_pngs)) + assert actual_frames == expected_frames, ("checkpoint GIF inventory mismatch", sorted(actual_frames ^ expected_frames)) + for name in expected_pngs | expected_frames: + artifact = root / name + assert artifact.is_file() and not artifact.is_symlink() and artifact.stat().st_size > 0, name + print("PASS: rendered checkpoint names, exact times, PNGs, and frame GIFs match source evidence without missing or extra files.") + + +def main(): + if len(sys.argv) not in {2, 3} or len(sys.argv) == 3 and sys.argv[2] != "--rendered-checkpoints": + raise SystemExit("usage: validate.py OUTPUT_DIRECTORY [--rendered-checkpoints]") + root = Path(sys.argv[1]) + if len(sys.argv) == 3: + validate_rendered_checkpoints(root) + return + requests = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] + for scene in SCENES: + evidence = json.loads((root / (scene + "-evidence.json")).read_text()) + expected_status = 130 if scene == "slow-cancel" else 1 if scene in {"uncertain", "preflight-400"} else 0 + assert evidence["exit_status"] == expected_status, (scene, "incorrect child exit") + assert evidence["cli_exit_status"] == evidence["exit_status"], (scene, "Bash changed the CLI exit status") + assert evidence["parent_shell"] == "/bin/bash", (scene, "missing native Bash parent") + modes = evidence["terminal_modes"] + assert modes["snapshot_parent_pid"] == modes["bash_pid"], (scene, "terminal snapshot lacks its live Bash parent") + initial, restored = modes["initial"].copy(), modes["restored"].copy() + assert len(initial) == len(restored) == 7, (scene, "incomplete terminal mode snapshot") + if sys.platform == "darwin": + initial[3] &= ~termios.PENDIN + restored[3] &= ~termios.PENDIN + assert initial == restored, (scene, "terminal input mode was not restored") + staged = evidence["staged_binary"] + staged_path = Path(staged["path"]) + runtime = Path(evidence["recorder_runtime"]) + assert runtime.parent == Path("/private/tmp") and runtime.name.startswith("cli-demo.") + assert staged_path.name == "openai" and staged_path.parent.parent == runtime + assert staged_path.parent.name.startswith("openai-limits-demo-") + assert staged["sha256_before"] == staged["sha256_after"] == evidence["binary_sha256"] + assert staged["removed"] is True and not staged_path.parent.exists(), (scene, "staged CLI cleanup incomplete") + assert (evidence["width"], evidence["height"]) in {(80, 24), (40, 12)} + raw = (root / (scene + "-evidence.tty")).read_bytes() + child = [json.loads(line) for line in (root / (scene + "-evidence.child.cast")).read_text().splitlines()] + child_bytes = "".join(event[2] for event in child[1:] if event[1] == "o").encode() + outer = [json.loads(line) for line in (root / (scene + ".cast")).read_text().splitlines()] + outer_bytes = "".join(event[2] for event in outer[1:] if event[1] == "o").encode() + marker = ("$ " + " ".join(evidence["command"]) + "\r\n").encode() + assert outer_bytes.count(marker) == 1, (scene, "ambiguous command boundary") + relay = outer_bytes.split(marker, 1)[1] + prompt = b"\r\n$ " + assert relay.endswith(prompt), (scene, "missing restored prompt") + assert relay[:-len(prompt)] == raw == child_bytes, (scene, "capture changed CLI bytes") + assert hashlib.sha256(raw).hexdigest() == evidence["relayed_sha256"] + if expected_status != 0: + restored = raw.rfind(b"\x1b[?1049l") + assert restored >= 0, (scene, "final diagnostic lacks terminal restoration") + diagnostic = b" ".join(raw[restored + len(b"\x1b[?1049l"):].lower().split()) + for action in (b"press r", b"use r to", b"refresh with r", b"press ?", b"press f1", b"use f1", + b"f1 for", b"f1:", b"ctrl+p", b"tab to apply"): + assert action not in diagnostic, (scene, "final diagnostic names an action from the closed workspace") + mutations = [entry for entry in requests if entry["scene"] == scene and entry["method"] in {"POST", "DELETE"}] + assert len(mutations) == (0 if scene in {"slow-cancel", "alert-print", "preflight-400"} else 1), (scene, "unexpected mutation count") + if scene == "slow-cancel": + assert any(entry["scene"] == scene and entry["status"] == "held" for entry in requests) + continue + checkpoints = {point["name"]: point for point in evidence["checkpoints"]} + assert "review-command" in checkpoints + preview = (root / (scene + "-evidence.review-command.txt")).read_text() + assert "Command preview (inspection only)" in preview, (scene, "missing inspection preview") + assert "Command printing needs" not in preview and "Command preview omitted" not in preview + final_screen = (root / (scene + "-evidence.exit.txt")).read_text() + assert "> Apply change" not in final_screen, (scene, "stale confirmation after exit") + if scene == "alert-print": + scene_requests = [entry for entry in requests if entry["scene"] == scene] + assert scene_requests and all(entry["method"] == "GET" for entry in scene_requests) + assert "review-print" in checkpoints and "printed-command" in checkpoints + assert "review-apply" not in checkpoints + assert checkpoints["review-command"]["seconds"] < checkpoints["review-print"]["seconds"] + keys = [event["key"] for event in evidence["keys"]] + assert keys.count("Ctrl+P print and exit") == 1 and "Apply" not in keys + endpoint = urlsplit(evidence["request_base_url"]) + assert endpoint.scheme == "http" and endpoint.hostname == "127.0.0.1" and endpoint.port + assert evidence["request_base_url"] == f"http://127.0.0.1:{endpoint.port}/alert-print/v1" + channel = json.dumps({"recipients": ["finance@example.test"], + "subject_prefix": "Demo team's budget $HOME; keep 'quoted' text", + "type": "email"}, sort_keys=True, separators=(",", ":")) + command = (staged["path"] + " '--base-url=" + evidence["request_base_url"] + + "' admin organization spend-alerts create" + " --currency USD --interval month --threshold-amount 7550 --notification-channel '" + + channel.replace("'", "'\\''") + "'") + newline = "\r\n" if initial[1] & termios.OPOST and initial[1] & termios.ONLCR else "\n" + expected = (command + newline).encode() + printed = (root / (scene + "-evidence.printed-command.raw.txt")).read_bytes() + start = len(raw) - len(expected) + restored_end = raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + assert raw.rfind(b"\x1b[?1049l") > raw.rfind(b"\x1b[?1049h") >= 0 + assert raw.rfind(b"\x1b[?25h") > raw.rfind(b"\x1b[?25l") >= 0 + assert restored_end <= start and raw.rfind(b"\x1b[?25h") < start + assert printed == expected == raw[start:], (scene, "printed command bytes changed") + assert raw[restored_end:].count(expected) == 1 + assert evidence["printed_command"] == {"byte_offset": start, "byte_length": len(expected), + "sha256": hashlib.sha256(expected).hexdigest()} + continue + if scene == "preflight-400": + scene_requests = [entry for entry in requests if entry["scene"] == scene] + assert scene_requests and all(entry["method"] == "GET" for entry in scene_requests) + limit_reads = [entry for entry in scene_requests if entry["path"].endswith("/spend_limit")] + assert [entry["status"] for entry in limit_reads] == [200, 400], (scene, "preflight did not follow a successful overview") + assert all(entry["status"] == 200 for entry in scene_requests if not entry["path"].endswith("/spend_limit")) + required = ("review-command", "review-apply", "preflight", "refresh-required", "preflight-help", "preflight-draft") + assert all(name in checkpoints for name in required) + assert all(checkpoints[left]["seconds"] < checkpoints[right]["seconds"] for left, right in zip(required, required[1:])) + keys = [event["key"] for event in evidence["keys"]] + assert keys.count("Apply") == keys.count("Blocked review before refresh") == keys.count("F1 recovery") == 1 + for name, messages in (("preflight", ("HTTP 400", "125.25", "Review change")), + ("refresh-required", ("Refresh with r", "125.25", "Review change")), + ("preflight-help", ("HTTP 400", "No write sent.", "request fields and scope", "press r to refresh")), + ("preflight-draft", ("125.25", "Review change"))): + screen = (root / (scene + "-evidence." + name + ".txt")).read_text() + compact = "".join(screen.split()) + assert all("".join(message.split()) in compact for message in messages), (scene, name, "missing recovery evidence") + assert b"synthetic-private-preflight" not in raw, (scene, "private response leaked") + restored_end = raw.rfind(b"\x1b[?1049l") + len(b"\x1b[?1049l") + diagnostic = raw[restored_end:] + assert all(message in diagnostic for message in (b"HTTP 400", b"No write sent.", b"request fields and scope", + b"Reopen the limits workspace", b"review current settings", + b"help admin organization spend-limit retrieve", + b"original connection, authentication, and scope options")) + assert b"may have succeeded" not in diagnostic + continue + if scene == "alert-create": + required = ("validation", "validation-help", "draft-return", "review-back", + "review-top", "review-before-help", "review-help", "review-return", "review-apply", "saved") + assert all(name in checkpoints for name in required) + assert all(checkpoints[left]["seconds"] < checkpoints[right]["seconds"] for left, right in zip(required, required[1:])) + for name, messages in (("validation", ("Enter a USD amount", "Review change")), + ("validation-help", ("Draft details", "two decimal places.")), + ("draft-return", ("> Review change", "Demo")), + ("review-back", ("Monthly USD", "75.50")), + ("review-return", ("> Back", "Apply change"))): + screen = (root / (scene + "-evidence." + name + ".txt")).read_text() + assert all(message in screen for message in messages), (scene, name, "missing draft recovery evidence") + request = mutations[0] + if scene in {"org-limit", "project-limit", "uncertain"}: + assert request["body"] == {"currency": "USD", "interval": "month", "threshold_amount": 12525} + elif scene in {"alert-create", "alert-edit"}: + assert request["body"] == {"currency": "USD", "interval": "month", + "threshold_amount": 7550 if scene == "alert-create" else 15025, + "notification_channel": {"type": "email", "recipients": ["finance@example.test"], + "subject_prefix": "Demo" if scene == "alert-create" else None}} + elif scene == "org-limit-delete": + assert request["method"] == "DELETE" and request["path"].endswith("/spend_limit") + assert request.get("body") is None + elif scene == "alert-delete": + assert request["method"] == "DELETE" and request["path"].endswith("/spend_alerts/alert_demo") + elif scene == "rate-update": + assert request["body"] == {"max_requests_per_1_minute": 250} + if scene == "uncertain": + assert request["status"] == "response_lost_after_write" + else: + assert request["status"] == 200 + assert "review-command" in checkpoints and "review-apply" in checkpoints + assert checkpoints["review-command"]["seconds"] < checkpoints["review-apply"]["seconds"] + assert [event["key"] for event in evidence["keys"]].count("Apply") == 1 + print("PASS: eleven real workspace processes; exact child/raw/outer bytes; expected exits and mutation bodies.") + print("PASS: one mutation per confirmed action; no retry after uncertain outcome; slow load canceled without mutation.") + print("PASS: native Bash parents preserve CLI exits; staged binary hashes match; ten inspection previews; staged files removed.") + print("PASS: all seven terminal mode fields restored before Bash exits; only Darwin PENDIN normalized.") + print("PASS: Ctrl+P exits without mutation; the complete command prints byte-for-byte after terminal restoration.") + print("PASS: preflight HTTP 400 retains the draft; safe F1 recovery; zero mutations; normal exit preserves failure.") + print("LIMIT: inspect all checkpoint images and GIF frames before claiming visual acceptance.") + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/record-spend-thresholds.sh b/scripts/demos/record-spend-thresholds.sh new file mode 100755 index 00000000..32460f02 --- /dev/null +++ b/scripts/demos/record-spend-thresholds.sh @@ -0,0 +1,67 @@ +#!/bin/bash +set -euo pipefail + +if [ "$#" -ne 6 ] || [ "$1" != --run-authorized-pty-slot ]; then + echo 'usage: record-spend-thresholds.sh --run-authorized-pty-slot BEFORE_BINARY AFTER_BINARY BEFORE_SHA AFTER_SHA OUTPUT_DIR' >&2 + exit 2 +fi +shift +record_source="$(cd "$(dirname "$0")" && pwd -P)" +record_repo="$(cd "$record_source/../.." && pwd -P)" +record_recipe="$record_source/spend-thresholds" +source "$record_source/capture_and_render.sh" +demo_prepare_capture "$record_repo" "$1" "$2" "$3" "$4" "$5" "$record_recipe/server.py" +demo_python="$(command -v python3)" +demo_start_api "$demo_output/requests.jsonl" "$demo_output/fixtures.json" +cp "$record_recipe/scene.sh" "$demo_runtime/scene.sh" +cp "$record_recipe/scene.sh" "$demo_output/scene.sh" +{ + echo 'feature: spend threshold units and notification semantics' + echo "before commit: $demo_before_sha" + echo "after commit: $demo_after_sha" + echo 'scope: Bash PTY terminal replay; synthetic loopback API; no native graphical terminal validation' + echo 'render: 100x22 and 40x24; Menlo 18px; asciinema theme; NO_COLOR=1' + echo 'data: synthetic USD amounts in integer cents; fake admin credentials; isolated HOME' + demo_capture_metadata + "$demo_python" --version + shasum -a 256 "$demo_python" "$record_source/record-spend-thresholds.sh" \ + "$record_recipe/scene.sh" "$record_recipe/validate.py" "$record_recipe/README.md" \ + "$demo_output/fixtures.json" +} > "$demo_output/metadata.txt" +if [ -n "${DEMO_SOURCE_MANIFEST:-}" ]; then + cp "$DEMO_SOURCE_MANIFEST" "$demo_output/source-manifest.txt" + shasum -a 256 "$demo_output/source-manifest.txt" >> "$demo_output/metadata.txt" +fi +demo_render_options=(--renderer resvg --font-family Menlo --font-size 18 --line-height 1.2 \ + --theme asciinema --fps-cap 15 --last-frame-duration 2) +for record_width in 100 40; do + demo_window_size="${record_width}x22" + if [ "$record_width" -eq 40 ]; then demo_window_size=40x24; fi + record_scenes=() + for record_case in limit alerts json missing; do + for record_phase in before after; do + record_scene="$record_phase-$record_case-$record_width" + record_scenes+=("$record_scene") + record_endpoint="$demo_api_url/v1" + if [ "$record_case" = missing ]; then record_endpoint="$demo_api_url/missing/v1"; fi + mkdir -p "$demo_runtime/$record_scene-home" + demo_capture_scene "$record_scene" 0 "$demo_runtime/$record_phase" "$record_endpoint" \ + "$record_phase: $record_case ($record_width columns)" \ + "HOME=$demo_runtime/$record_scene-home" "DEMO_CASE=$record_case" \ + OPENAI_ADMIN_KEY=synthetic-demo-admin-key NO_COLOR=1 FORCE_COLOR=0 GOMAXPROCS=2 + done + done + demo_assemble_capture 200 "${record_scenes[@]}" + for record_suffix in cast gif txt; do + mv "$demo_output/comparison.$record_suffix" "$demo_output/comparison-$record_width.$record_suffix" + done + mv "$demo_output/comparison-scenes.txt" "$demo_output/comparison-scenes-$record_width.txt" + mv "$demo_output/media.json" "$demo_output/media-$record_width.json" +done +demo_stop_api +"$demo_python" "$record_recipe/validate.py" "$demo_output" > "$demo_output/validation.txt" +( + cd "$demo_output" + shasum -a 256 -- *.cast *.gif *.png *.txt *.json *.jsonl *.sh > SHA256SUMS +) +printf 'Recorded spend threshold terminal replays in %s\n' "$demo_output" diff --git a/scripts/demos/spend-thresholds/README.md b/scripts/demos/spend-thresholds/README.md new file mode 100644 index 00000000..d1310cb9 --- /dev/null +++ b/scripts/demos/spend-thresholds/README.md @@ -0,0 +1,62 @@ +# Spend threshold terminal replays + +This recipe compares real CLI binaries against fixed synthetic responses. +It uses the shared `capture_and_render.sh` lifecycle and a Python loopback fixture. +It requires no fixture build, installation, live API request, or billing change. + +The recordings are Bash PTY terminal replays, not native graphical terminal validation. +Each command uses fake admin credentials and an isolated configuration directory. +The fixture checks request authentication without logging credentials or request headers. + +## Record + +Reserve a PTY execution slot before running this recipe. +Prepare independently verified binaries for the baseline and candidate commits. +Provide their full 40-character commit IDs. +The recorder records binary hashes but cannot establish build provenance. + +Required tools are Python 3, Bash, asciinema, agg, ffmpeg, ffprobe, and shasum. +Use an empty output directory outside the repository. + +```sh +bash scripts/demos/record-spend-thresholds.sh --run-authorized-pty-slot \ + /path/to/before/openai /path/to/after/openai \ + "$BEFORE_SHA" "$AFTER_SHA" /path/to/evidence/spend-thresholds +``` + +Optionally set `DEMO_SOURCE_MANIFEST` to a verified source/build manifest. +The recorder copies and hashes that manifest alongside the evidence. +The authorization flag records the operator's decision; it does not reserve a slot. + +The initial comparison base is `e68939820415144d769ed02de6aa72d5b7d32948`. +The batch coordinator verified a shared baseline binary with this SHA-256: + +```text +23414722af83f1d4acfd1e80b1723448fc711083d7b76b9bcf36ee7813ff3df2 +``` + +Verify any reused binary against its own build record and digest. + +## Scenes and assertions + +Both 100-column and 40-column captures include these before/after scenes: + +- Organization limit: raw `10000` becomes `USD 100.00 per month`; reported enforcement stays visible. +- Project alert: raw `20000` becomes `USD 200.00 per month`; readable output identifies notifications without caps. +- Explicit JSON: original cents and response fields remain unchanged. +- Missing enforcement: readable output states `not reported in this response`. + +The fixture serves one alert with `has_more: false`. +This replay does not establish multi-page behavior; dedicated public tests cover pagination. +Commands use existing canonical routes and mutation-free GET requests. + +The recorder checks all command exit statuses and fixture shutdown. +The validator checks every transcript, PTY width, request path, and response hash. +Fixture request, response-write, flush, and close failures stop the recording. +SIGTERM closes the fixture cleanly with status zero. + +Each scene produces an asciicast, GIF, PNG, and plain transcript. +`comparison-100.gif` and `comparison-40.gif` contain the respective complete comparisons. +Metadata records commits, binary/tool hashes, environment, and command exit statuses. +`fixtures.json`, `requests.jsonl`, `validation.txt`, and `SHA256SUMS` retain reproducible evidence. +Review screenshots before publication. Keep generated media outside Git. diff --git a/scripts/demos/spend-thresholds/scene.sh b/scripts/demos/spend-thresholds/scene.sh new file mode 100755 index 00000000..c8b14772 --- /dev/null +++ b/scripts/demos/spend-thresholds/scene.sh @@ -0,0 +1,23 @@ +#!/bin/bash +set -euo pipefail +test -t 0 && test -t 1 && test -t 2 || exit 99 +printf '\033[2J\033[H' +printf '%s\n' "$DEMO_SCENE_LABEL" 'Terminal replay. Synthetic data.' +printf '\n' +case "$DEMO_CASE" in + limit|missing) + printf '%s\n' '$ openai admin organization spend-limit retrieve' + openai admin organization spend-limit retrieve + ;; + alerts) + printf '%s\n' '$ openai admin organization projects spend-alerts list --project-id proj_demo --max-items 2' + openai admin organization projects spend-alerts list --project-id proj_demo --max-items 2 + ;; + json) + printf '%s\n' '$ openai admin organization spend-limit retrieve --format json' + openai admin organization spend-limit retrieve --format json + ;; + *) echo 'Unknown spend demo scene.' >&2; exit 2;; +esac +printf '\n$ ' +sleep 2 diff --git a/scripts/demos/spend-thresholds/server.py b/scripts/demos/spend-thresholds/server.py new file mode 100755 index 00000000..6d1e18d8 --- /dev/null +++ b/scripts/demos/spend-thresholds/server.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Serve fixed spend responses on loopback and fail on fixture I/O errors.""" + +import hashlib +import http.server +import json +import os +from pathlib import Path +import signal +import sys +from urllib.parse import urlsplit + + +def write_text(path, text): + with Path(path).open("x", encoding="utf-8") as destination: + if destination.write(text) != len(text): + raise OSError("Incomplete fixture file write") + + +def main(): + if len(sys.argv) != 4: + raise SystemExit("usage: server.py ADDRESS_FILE REQUEST_LOG FIXTURE_METADATA") + limit = {"object": "organization.spend_limit", "threshold_amount": 10000, + "currency": "USD", "interval": "month", "enforcement": {"status": "enforcing"}} + missing = {key: value for key, value in limit.items() if key != "enforcement"} + alerts = [{"id": "alert_demo", "object": "project.spend_alert", "threshold_amount": 20000, + "currency": "USD", "interval": "month", + "notification_channel": {"type": "email", "recipients": ["finance@example.test"]}}] + fixtures = { + "/v1/organization/spend_limit": limit, + "/missing/v1/organization/spend_limit": missing, + "/v1/organization/projects/proj_demo/spend_alerts": { + "object": "list", "data": alerts, "first_id": "alert_demo", "last_id": "alert_demo", "has_more": False}, + } + bodies = {path: json.dumps(value, separators=(",", ":")).encode() for path, value in fixtures.items()} + write_text(sys.argv[3], json.dumps(fixtures, indent=2) + "\n") + running = True + + def stop(*_): + nonlocal running + running = False + + signal.signal(signal.SIGTERM, stop) + signal.signal(signal.SIGINT, stop) + with open(sys.argv[2], "x", encoding="utf-8") as request_log: + class Fixture(http.server.BaseHTTPRequestHandler): + def log_message(self, *_): + pass + + def do_GET(self): + path = urlsplit(self.path).path + if self.headers.get("Authorization") != "Bearer synthetic-demo-admin-key": + raise ValueError("Unexpected synthetic fixture authentication") + body = bodies[path] + self.send_response(200) + self.send_header("Content-Type", "application/json") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + if self.wfile.write(body) != len(body): + raise OSError("Incomplete fixture response write") + self.wfile.flush() + entry = json.dumps({"method": "GET", "path": path, "status": 200, + "response_sha256": hashlib.sha256(body).hexdigest()}) + "\n" + if request_log.write(entry) != len(entry): + raise OSError("Incomplete fixture request-log write") + request_log.flush() + + class Server(http.server.HTTPServer): + failed = False + + def get_request(self): + connection, address = super().get_request() + connection.settimeout(2) + return connection, address + + def handle_error(self, *_): + self.failed = True + print("Synthetic fixture request or write failed.", file=sys.stderr) + + with Server(("127.0.0.1", 0), Fixture) as server: + server.timeout = 0.05 + address_file = Path(sys.argv[1]) + pending_address = address_file.with_suffix(".tmp") + write_text(pending_address, f"http://127.0.0.1:{server.server_port}\n") + os.replace(pending_address, address_file) + while running and not server.failed: + server.handle_request() + if server.failed: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/scripts/demos/spend-thresholds/validate.py b/scripts/demos/spend-thresholds/validate.py new file mode 100755 index 00000000..ae5967cb --- /dev/null +++ b/scripts/demos/spend-thresholds/validate.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python3 +"""Verify every replay, command status, request, and unchanged JSON response.""" + +import hashlib +import json +from pathlib import Path +import sys + + +def check(condition, message): + if not condition: + raise SystemExit(message) + + +def main(): + check(len(sys.argv) == 2, "usage: validate.py CAPTURE_DIRECTORY") + root = Path(sys.argv[1]) + fixtures = json.loads((root / "fixtures.json").read_text()) + metadata = (root / "metadata.txt").read_text() + requests = [] + for width in (100, 40): + for case in ("limit", "alerts", "json", "missing"): + for phase in ("before", "after"): + scene = f"{phase}-{case}-{width}" + text = (root / f"{scene}.txt").read_text() + compact = "".join(text.split()) + header = json.loads((root / f"{scene}.cast").read_text().splitlines()[0]) + check(header["width"] == width, f"{scene}: incorrect PTY width") + check(header["height"] == (24 if width == 40 else 22), f"{scene}: incorrect PTY height") + check("Terminalreplay.Syntheticdata." in compact, f"{scene}: missing evidence label") + check(f"{scene} exit status: 0 (expected 0)" in metadata, f"{scene}: missing successful exit") + path = "/v1/organization/spend_limit" + if case == "alerts": + path = "/v1/organization/projects/proj_demo/spend_alerts" + check("alert_demo" in text and "finance@example.test" in text, f"{scene}: lost alert fields") + elif case == "missing": + path = "/missing/v1/organization/spend_limit" + if case == "json": + value, _ = json.JSONDecoder().raw_decode(text[text.index("{"):]) + check(value == fixtures[path], f"{scene}: explicit JSON changed") + check("Spendthreshold:" not in compact, f"{scene}: projection leaked into JSON") + elif phase == "before": + amount = "20000" if case == "alerts" else "10000" + check(f"Thresholdamount:{amount}" in compact, f"{scene}: baseline cents are missing") + check("Spendthreshold:" not in compact, f"{scene}: baseline already contains the projection") + else: + amount = "200.00" if case == "alerts" else "100.00" + check(f"Spendthreshold:USD{amount}permonth" in compact, f"{scene}: missing units") + if case == "alerts": + check("Alertsnotify;theyarenotspendingcaps." in compact, f"{scene}: missing alert semantics") + elif case == "missing": + check("Enforcement:notreportedinthisresponse" in compact, f"{scene}: missing enforcement caveat") + else: + check("Status:enforcing" in compact, f"{scene}: reported enforcement changed") + body = json.dumps(fixtures[path], separators=(",", ":")).encode() + requests.append({"method": "GET", "path": path, "status": 200, + "response_sha256": hashlib.sha256(body).hexdigest()}) + actual = [json.loads(line) for line in (root / "requests.jsonl").read_text().splitlines()] + check(actual == requests, "Unexpected fixture requests, order, or response hashes") + print("PASS: 16 successful scenes; 100/40-column PTYs; exact fixture requests and hashes.") + print("PASS: baseline cents; readable USD/month; preserved JSON; alert semantics; missing enforcement.") + + +if __name__ == "__main__": + main()